unsigned char *smf_get_mask( ThrWorkForce *wf, smf_modeltype mtype,
AstKeyMap *config, smfDIMMData *dat, int flags,
int *status ) {
/* Local Variables: */
AstCircle *circle; /* AST Region used to mask a circular area */
AstKeyMap *akm; /* KeyMap holding AST config values */
AstKeyMap *subkm; /* KeyMap holding model config values */
char refparam[ DAT__SZNAM ];/* Name for reference NDF parameter */
char words[100]; /* Buffer for variable message words */
const char *cval; /* The ZERO_MASK string value */
const char *modname; /* The name of the model being masked */
const char *skyrefis; /* Pointer to SkyRefIs attribute value */
dim_t i; /* Pixel index */
double *pd; /* Pointer to next element of map data */
double *predef; /* Pointer to mask defined by previous run */
double *ptr; /* Pointer to NDF Data array */
double *pv; /* Pointer to next element of map variance */
double centre[ 2 ]; /* Coords of circle centre in radians */
double meanhits; /* Mean hits in the map */
double radius[ 1 ]; /* Radius of circle in radians */
double zero_circle[ 3 ]; /* LON/LAT/Radius of circular mask */
double zero_lowhits; /* Fraction of mean hits at which to threshold */
double zero_snr; /* Higher SNR at which to threshold */
double zero_snrlo; /* Lower SNR at which to threshold */
int *ph; /* Pointer to next hits value */
int have_mask; /* Did a mask already exist on entry? */
int imask; /* Index of next mask type */
int indf1; /* Id. for supplied reference NDF */
int indf2; /* Id. for used section of reference NDF */
int isstatic; /* Are all used masks static? */
int lbnd_grid[ 2 ]; /* Lower bounds of map in GRID coords */
int mask_types[ NTYPE ]; /* Identifier for the types of mask to use */
int munion; /* Use union of supplied masks */
int nel; /* Number of mapped NDF pixels */
int nmask; /* The number of masks to be combined */
int nsource; /* No. of source pixels in final mask */
int skip; /* No. of iters for which AST is not subtracted */
int thresh; /* Absolute threshold on hits */
int ubnd_grid[ 2 ]; /* Upper bounds of map in GRID coords */
int zero_c_n; /* Number of zero circle parameters read */
int zero_mask; /* Use the reference NDF as a mask? */
int zero_niter; /* Only mask for the first "niter" iterations. */
int zero_notlast; /* Don't zero on last iteration? */
size_t ngood; /* Number good samples for stats */
smf_qual_t *pq; /* Pinter to map quality */
unsigned char **mask; /* Address of model's mask pointer */
unsigned char *newmask; /* Individual mask work space */
unsigned char *pm; /* Pointer to next returned mask pixel */
unsigned char *pn; /* Pointer to next new mask pixel */
unsigned char *result; /* Returned mask pointer */
/* Initialise returned values */
result = NULL;
/* Check inherited status. Also check that a map is being created. */
if( *status != SAI__OK || !dat || !dat->map ) return result;
/* Begin an AST context. */
astBegin;
/* Get the sub-keymap containing the configuration parameters for the
requested model. Also get a pointer to the mask array to use (there is
one for each maskable model)*/
if( mtype == SMF__COM ) {
modname = "COM";
mask = &(dat->com_mask);
} else if( mtype == SMF__AST ) {
modname = "AST";
mask = &(dat->ast_mask);
} else if( mtype == SMF__FLT ) {
modname = "FLT";
mask = &(dat->flt_mask);
} else {
modname = NULL;
mask = NULL;
*status = SAI__ERROR;
errRepf( " ", "smf_get_mask: Unsupported model type %d supplied - "
"must be COM, FLT or AST.", status, mtype );
}
subkm = NULL;
astMapGet0A( config, modname, &subkm );
/* Get the "ast.skip" value - when considering "zero_niter" and
"zero_freeze", we only count iterations for which the AST model
is subtracted (i.e. the ones following the initial "ast.skip"
iterations). */
astMapGet0A( config, "AST", &akm );
astMapGet0I( akm, "SKIP", &skip );
akm = astAnnul( akm );
/* Get the number of iterations over which the mask is to be applied. Zero
means all. Return with no mask if this number of iterations has
already been performed. */
zero_niter = 0;
astMapGet0I( subkm, "ZERO_NITER", &zero_niter );
if( zero_niter == 0 || dat->iter < zero_niter + skip ) {
/* Only return a mask if this is not the last iteration, or if ZERO_NOTLAST
is unset. */
zero_notlast = 0;
astMapGet0I( subkm, "ZERO_NOTLAST", &zero_notlast );
if( !( flags & SMF__DIMM_LASTITER ) || !zero_notlast ) {
/* Create a list of the mask types to be combined to get the final mask by
looking for non-default values for the corresponding configuration
parameters in the supplied KeyMap. Static masks (predefined, circles
or external NDFs) may be used on any iteration, but dynamic masks
(lowhits, snr) will only be avialable once the map has been determined
at the end of the first iteration. This means that when masking anything
but the AST model (which is determined after the map), the dynamic masks
cannot be used on the first iteration. Make a note if all masks being
used are static. */
isstatic = 1;
nmask = 0;
zero_lowhits = 0.0;
astMapGet0D( subkm, "ZERO_LOWHITS", &zero_lowhits );
if( zero_lowhits > 0.0 ) {
if( mtype == SMF__AST || !( flags & SMF__DIMM_FIRSTITER ) ) {
mask_types[ nmask++] = LOWHITS;
isstatic = 0;
}
} else if( zero_lowhits < 0.0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter %s.ZERO_LOWHITS (%g) - "
"it must not be negative.", status, modname, zero_lowhits );
}
if( astMapGet1D( subkm, "ZERO_CIRCLE", 3, &zero_c_n, zero_circle ) ) {
if( zero_c_n == 1 || zero_c_n == 3 ) {
mask_types[ nmask++] = CIRCLE;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad number of values (%d) for config parameter "
"%s.ZERO_CIRCLE - must be 1 or 3.", status, zero_c_n,
modname );
}
}
cval = NULL;
astMapGet0C( subkm, "ZERO_MASK", &cval );
if( cval ) {
if( !astChrMatch( cval, "REF" ) &&
!astChrMatch( cval, "MASK2" ) &&
!astChrMatch( cval, "MASK3" ) ) {
astMapGet0I( subkm, "ZERO_MASK", &zero_mask );
cval = ( zero_mask > 0 ) ? "REF" : NULL;
}
if( cval ) {
strcpy( refparam, cval );
astChrCase( NULL, refparam, 1, 0 );
mask_types[ nmask++] = REFNDF;
}
}
zero_snr = 0.0;
astMapGet0D( subkm, "ZERO_SNR", &zero_snr );
if( zero_snr > 0.0 ) {
if( mtype == SMF__AST || !( flags & SMF__DIMM_FIRSTITER ) ) {
mask_types[ nmask++] = SNR;
isstatic = 0;
}
} else if( zero_snr < 0.0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter %s.ZERO_SNR (%g) - "
"it must not be negative.", status, modname, zero_snr );
}
if( astMapHasKey( subkm, "ZERO_MASK_POINTER" ) ) {
astMapGet0P( subkm, "ZERO_MASK_POINTER", (void **) &predef );
if( predef ) mask_types[ nmask++] = PREDEFINED;
}
/* No need to create a mask if no masking was requested or possible. */
if( nmask > 0 ) {
/* Decide if we are using the union or intersection of the masks. */
astMapGet0I( subkm, "ZERO_UNION", &munion );
/* Note if a mask existed on entry. If not, create a mask now, and
initialise it to hold the mask defined by the initial sky map. */
if( *mask == NULL ) {
have_mask = 0;
if( dat->initqual ) {
*mask = astMalloc( dat->msize*sizeof( **mask ) );
if( *mask ) {
pm = *mask;
pq = dat->initqual;
for( i = 0; i < dat->msize; i++ ) {
*(pm++) = ( *(pq++) != 0 );
}
}
} else{
*mask = astCalloc( dat->msize, sizeof( **mask ) );
}
} else {
have_mask = 1;
}
/* If we are combining more than one mask, we need work space to hold
an individual mask independently of the total mask. If we are using
only one mask, then just use the main mask array. */
if( nmask > 1 ) {
newmask = astMalloc( dat->msize*sizeof( *newmask ) );
} else {
newmask = *mask;
}
/* Get the number of iterations after which the mask is to be frozen.
Zero means "never freeze the mask". */
int zero_freeze = 0;
astMapGet0I( subkm, "ZERO_FREEZE", &zero_freeze );
/* Loop round each type of mask to be used. */
for( imask = 0; imask < nmask && *status == SAI__OK; imask++ ){
/* If the mask is now frozen, we just return the existing mask. So leave the
loop. */
if( zero_freeze != 0 && dat->iter > zero_freeze + skip ) {
break;
/* Low hits masking... */
} else if( mask_types[ imask ] == LOWHITS ) {
/* Set hits pixels with 0 hits to VAL__BADI so that stats1 ignores them */
ph = dat->hitsmap;
for( i = 0; i < dat->msize; i++,ph++ ) {
if( *ph == 0 ) *ph = VAL__BADI;
}
/* Find the mean hits in the map */
smf_stats1I( dat->hitsmap, 1, dat->msize, NULL, 0, 0, &meanhits,
NULL, NULL, &ngood, status );
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: mean hits = %lf, ngood "
"= %zd", status, meanhits, ngood );
/* Create the mask */
thresh = meanhits*zero_lowhits;
ph = dat->hitsmap;
pn = newmask;
for( i = 0; i < dat->msize; i++,ph++ ) {
*(pn++) = ( *ph != VAL__BADI && *ph < thresh ) ? 1 : 0;
}
/* Report masking info. */
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: masking %s "
"model at hits = %d.", status, modname, thresh );
/* Circle masking... */
} else if( mask_types[ imask ] == CIRCLE ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the circle
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* If only one parameter supplied it is radius, assume reference
LON/LAT from the frameset to get the centre. If the SkyFrame
represents offsets from the reference position (i.e. the source is
moving), assume the circle is to be centred on the origin. */
if( zero_c_n == 1 ) {
zero_circle[ 2 ] = zero_circle[ 0 ];
skyrefis = astGetC( dat->outfset, "SkyRefIs" );
if( skyrefis && !strcmp( skyrefis, "Origin" ) ) {
zero_circle[ 0 ] = 0.0;
zero_circle[ 1 ] = 0.0;
} else {
zero_circle[ 0 ] = astGetD( dat->outfset, "SkyRef(1)" );
zero_circle[ 1 ] = astGetD( dat->outfset, "SkyRef(2)" );
}
zero_circle[ 0 ] *= AST__DR2D;
zero_circle[ 1 ] *= AST__DR2D;
}
/* The supplied bounds are for pixel coordinates... we need bounds for grid
coordinates which have an offset */
lbnd_grid[ 0 ] = 1;
lbnd_grid[ 1 ] = 1;
ubnd_grid[ 0 ] = dat->ubnd_out[ 0 ] - dat->lbnd_out[ 0 ] + 1;
ubnd_grid[ 1 ] = dat->ubnd_out[ 1 ] - dat->lbnd_out[ 1 ] + 1;
/* Coordinates & radius of the circular region converted from degrees
to radians */
centre[ 0 ] = zero_circle[ 0 ]*AST__DD2R;
centre[ 1 ] = zero_circle[ 1 ]*AST__DD2R;
radius[ 0 ] = zero_circle[ 2 ]*AST__DD2R;
/* Create the Circle, defined in the current Frame of the FrameSet (i.e.
the sky frame). */
circle = astCircle( astGetFrame( dat->outfset, AST__CURRENT), 1,
centre, radius, NULL, " " );
/* Fill the mask with zeros. */
memset( newmask, 0, sizeof( *newmask )*dat->msize );
/* Get the mapping from the sky frame (current) to the grid frame (base),
and then set the mask to 1 for all of the values outside this circle */
astMaskUB( circle, astGetMapping( dat->outfset, AST__CURRENT,
AST__BASE ),
0, 2, lbnd_grid, ubnd_grid, newmask, 1 );
/* Report masking info. */
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model will"
" be masked %s using a circle of "
"radius %g arc-secs, centred at %s=%s, %s=%s.",
status, modname, words, radius[0]*AST__DR2D*3600,
astGetC( dat->outfset, "Symbol(1)" ),
astFormat( dat->outfset, 1, centre[ 0 ] ),
astGetC( dat->outfset, "Symbol(2)" ),
astFormat( dat->outfset, 2, centre[ 1 ] ) );
}
/* Reference NDF masking... */
} else if( mask_types[ imask ] == REFNDF ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the NDF
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* Begin an NDF context. */
ndfBegin();
/* Get an identifier for the NDF using the associated ADAM parameter. */
ndfAssoc( refparam, "READ", &indf1, status );
/* Get a section from this NDF that matches the bounds of the map. */
ndfSect( indf1, 2, dat->lbnd_out, dat->ubnd_out, &indf2,
status );
/* Map the section. */
ndfMap( indf2, "DATA", "_DOUBLE", "READ", (void **) &ptr,
&nel, status );
/* Check we can use the pointer safely. */
if( *status == SAI__OK ) {
/* Find bad pixels in the NDF and set those pixels to 1 in the mask. */
pn = newmask;
for( i = 0; i < dat->msize; i++ ) {
*(pn++) = ( *(ptr++) == VAL__BADD ) ? 1 : 0;
}
/* Report masking info. */
ndfMsg( "N", indf2 );
msgSetc( "M", modname );
if( zero_niter == 0 ) {
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The ^M "
"model will be masked on each iteration "
"using the bad pixels in NDF '^N'.",
status );
} else {
msgSeti( "I", zero_niter );
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The ^M "
"model will be masked for ^I iterations "
"using the bad pixels in NDF '^N'.",
status );
}
}
/* End the NDF context. */
ndfEnd( status );
}
/* SNR masking... */
} else if( mask_types[ imask ] == SNR ) {
/* Get the lower SNR limit. */
zero_snrlo = 0.0;
astMapGet0D( subkm, "ZERO_SNRLO", &zero_snrlo );
if( zero_snrlo <= 0.0 ) {
zero_snrlo = zero_snr;
} else if( zero_snrlo > zero_snr && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter "
"%s.ZERO_SNRLO (%g) - it must not be higher "
"than %s.ZERO_SNR (%g).", status, modname,
zero_snrlo, modname, zero_snr );
}
/* If the higher and lower SNR limits are equal, just do a simple
threshold on the SNR values to get the mask. */
if( zero_snr == zero_snrlo ) {
pd = dat->map;
pv = dat->mapvar;
pn = newmask;
for( i = 0; i < dat->msize; i++,pd++,pv++ ) {
*(pn++) = ( *pd != VAL__BADD && *pv != VAL__BADD &&
*pv >= 0.0 && *pd < zero_snr*sqrt( *pv ) ) ? 1 : 0;
}
/* Report masking info. */
if( !have_mask ) {
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using an SNR limit of %g.",
status, modname, words, zero_snr );
}
/* If the higher and lower SNR limits are different, create an initial
mask by thresholding at the ZERO_SNR value, and then extend the source
areas within the mask down to an SNR limit of ZERO_SNRLO. */
} else {
smf_snrmask( wf, dat->map, dat->mapvar, dat->mdims,
zero_snr, zero_snrlo, newmask, status );
/* Report masking info. */
if( !have_mask ) {
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using an SNR limit of %g "
"extended down to %g.", status, modname,
words, zero_snr, zero_snrlo );
}
}
/* Predefined masking... */
} else if( mask_types[ imask ] == PREDEFINED ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* Find bad pixels in the predefined array and set those pixels to 1 in
the mask. */
pn = newmask;
for( i = 0; i < dat->msize; i++ ) {
*(pn++) = ( *(predef++) == VAL__BADD ) ? 1 : 0;
}
/* Report masking info. */
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using a smoothed form of "
"the final mask created with the previous map.",
status, modname, words );
}
}
/* If required, add the new mask into the returned mask. If this is the
first mask, we just copy the new mask to form the returned mask.
Otherwise, we combine it with the existing returned mask. */
if( ! have_mask || ! isstatic ) {
if( nmask > 1 ) {
if( imask == 0 ) {
memcpy( *mask, newmask, dat->msize*sizeof(*newmask));
} else {
pm = *mask;
pn = newmask;
if( munion ) {
for( i = 0; i < dat->msize; i++,pm++ ) {
if( *(pn++) == 0 ) *pm = 0;
}
} else {
for( i = 0; i < dat->msize; i++,pm++ ) {
if( *(pn++) == 1 ) *pm = 1;
}
}
}
}
}
}
/* Free the individual mask work array if it was used. */
if( nmask > 1 ) newmask = astFree( newmask );
/* Check that the mask has some source pixels (i.e. pixels that have non-bad data values -
we do not also check variance values since they are not available until the second
iteration). */
if( *status == SAI__OK ) {
nsource = 0;
pm = *mask;
pd = dat->map;
for( i = 0; i < dat->msize; i++,pd++,pv++,pm++ ) {
if( *pd != VAL__BADD && *pm == 0 ) nsource++;
}
if( nsource < 5 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", "The %s mask being used has fewer than 5 "
"source pixels.", status, modname );
if( zero_snr > 0.0 ) {
errRepf( "", "Maybe your zero_snr value (%g) is too high?",
status, zero_snr );
}
}
}
/* Return the mask pointer if all has gone well. */
if( *status == SAI__OK ) result = *mask;
}
}
}
/* End the AST context, annulling all AST Objects created in the context. */
astEnd;
/* Return the pointer to the boolean mask. */
return result;
}
void smurf_extinction( int * status ) {
/* Local Variables */
smfArray *bbms = NULL; /* Bad bolometer masks */
smfArray *darks = NULL; /* Dark data */
AstKeyMap *extpars = NULL; /* Tau relation keymap */
Grp *fgrp = NULL; /* Filtered group, no darks */
smfArray *flatramps = NULL;/* Flatfield ramps */
int has_been_sky_removed = 0;/* Data are sky-removed */
AstKeyMap *heateffmap = NULL; /* Heater efficiency data */
size_t i; /* Loop counter */
Grp *igrp = NULL; /* Input group */
AstKeyMap *keymap=NULL; /* Keymap for storing parameters */
smf_tausrc tausrc; /* enum value of optical depth source */
smf_extmeth extmeth; /* Extinction correction method */
char tausource[LEN__METHOD]; /* String for optical depth source */
char method[LEN__METHOD]; /* String for extinction airmass method */
smfData *odata = NULL; /* Output data struct */
Grp *ogrp = NULL; /* Output group */
size_t outsize; /* Total number of NDF names in the output group */
size_t size; /* Number of files in input group */
double tau = 0.0; /* Zenith tau at this wavelength */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
if (*status != SAI__OK) return;
/* Main routine */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Read the input file */
kpg1Rgndf( "IN", 0, 1, "", &igrp, &size, status );
/* Filter out darks */
smf_find_science( igrp, &fgrp, 0, NULL, NULL, 1, 1, SMF__NULL, &darks,
&flatramps, &heateffmap, NULL, status );
/* input group is now the filtered group so we can use that and
free the old input group */
size = grpGrpsz( fgrp, status );
grpDelet( &igrp, status);
igrp = fgrp;
fgrp = NULL;
if (size > 0) {
/* Get output file(s) */
kpg1Wgndf( "OUT", igrp, size, size, "More output files required...",
&ogrp, &outsize, status );
} else {
msgOutif(MSG__NORM, " ","All supplied input frames were DARK,"
" nothing to extinction correct", status );
}
/* Get group of pixel masks and read them into a smfArray */
smf_request_mask( "BBM", &bbms, status );
/* Read the tau relations from config file or group. We do not
allow sub instrument overloading because these are all values
based on filter name. */
keymap = kpg1Config( "TAUREL", "$SMURF_DIR/smurf_extinction.def", NULL,
1, status );
/* and we need to use the EXT entry */
astMapGet0A( keymap, "EXT", &extpars );
keymap = astAnnul( keymap );
/* Get tau source */
parChoic( "TAUSRC", "Auto",
"Auto,CSOtau,CSOFit, Filtertau, WVMraw", 1,
tausource, sizeof(tausource), status);
/* Decide how the correction is to be applied - convert to flag */
parChoic( "METHOD", "ADAPTIVE",
"Adaptive,Quick,Full,", 1, method, sizeof(method), status);
/* Place parameters into a keymap and extract values */
if( *status == SAI__OK ) {
keymap = astKeyMap( " " );
if( astOK ) {
astMapPut0C( keymap, "TAUSRC", tausource, NULL );
astMapPut0C( keymap, "TAUMETHOD", method, NULL );
smf_get_extpar( keymap, &tausrc, &extmeth, NULL, status );
}
}
for (i=1; i<=size && ( *status == SAI__OK ); i++) {
/* Flatfield - if necessary */
smf_open_and_flatfield( igrp, ogrp, i, darks, flatramps, heateffmap,
&odata, status );
if (*status != SAI__OK) {
/* Error flatfielding: tell the user which file it was */
msgSeti("I",i);
errRep(TASK_NAME, "Unable to open the ^I th file", status);
}
/* Mask out bad pixels - mask data array not quality array */
smf_apply_mask( odata, bbms, SMF__BBM_DATA, 0, status );
/* Now check that the data are sky-subtracted */
if ( !smf_history_check( odata, "smf_subtract_plane", status ) ) {
/* Should we override remsky check? */
parGet0l("HASSKYREM", &has_been_sky_removed, status);
if ( !has_been_sky_removed && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti("I",i);
errRep("", "Input data from file ^I are not sky-subtracted", status);
}
}
/* If status is OK, make decisions on source keywords the first
time through. */
if ( *status == SAI__OK && i == 1 ) {
if (tausrc == SMF__TAUSRC_CSOTAU ||
tausrc == SMF__TAUSRC_AUTO ||
tausrc == SMF__TAUSRC_TAU) {
double deftau;
const char * param = NULL;
smfHead *ohdr = odata->hdr;
/* get default CSO tau -- this could be calculated from CSO fits */
deftau = smf_calc_meantau( ohdr, status );
/* Now ask for desired CSO tau */
if ( tausrc == SMF__TAUSRC_CSOTAU || tausrc == SMF__TAUSRC_AUTO) {
param = "CSOTAU";
} else if (tausrc == SMF__TAUSRC_TAU) {
param = "FILTERTAU";
deftau = smf_cso2filt_tau( ohdr, deftau, extpars, status );
}
parGdr0d( param, deftau, 0.0,1.0, 1, &tau, status );
} else if ( tausrc == SMF__TAUSRC_CSOFIT || tausrc == SMF__TAUSRC_WVMRAW ) {
/* Defer a message until after extinction correction */
} else {
*status = SAI__ERROR;
errRep("", "Unsupported opacity source. Possible programming error.",
status);
}
}
/* Apply extinction correction - note that a check is made to
determine whether the data have already been extinction
corrected */
smf_correct_extinction( wf, odata, &tausrc, extmeth, extpars, tau, NULL, NULL, status );
if ( tausrc == SMF__TAUSRC_WVMRAW ) {
msgOutif(MSG__VERB," ", "Used Raw WVM data for extinction correction", status);
} else if ( tausrc == SMF__TAUSRC_CSOFIT ) {
msgOutif(MSG__VERB," ", "Used fit to CSO data for extinction correction", status);
} else if ( tausrc == SMF__TAUSRC_CSOTAU ) {
msgOutif(MSG__VERB," ", "Used an explicit CSO tau value for extinction correction", status);
} else if ( tausrc == SMF__TAUSRC_TAU ) {
msgOutif(MSG__VERB," ", "Used an explicit filter tau value for extinction correction", status);
} else {
if (*status == SAI__OK) {
const char * taustr = smf_tausrc_str( tausrc, status );
*status = SAI__ERROR;
errRepf( "", "Unexpected opacity source used for extinction correction of %s."
" Possible programming error.", status, taustr );
}
}
/* Set character labels */
smf_set_clabels( "Extinction corrected",NULL, NULL, odata->hdr, status);
smf_write_clabels( odata, status );
/* Free resources for output data */
smf_close_file( &odata, status );
}
/* Write out the list of output NDF names, annulling the error if a null
parameter value is supplied. */
if( *status == SAI__OK && ogrp ) {
grpList( "OUTFILES", 0, 0, NULL, ogrp, status );
if( *status == PAR__NULL ) errAnnul( status );
}
/* Tidy up after ourselves: release the resources used by the grp routines */
if (darks) smf_close_related( &darks, status );
if (bbms) smf_close_related( &bbms, status );
if( flatramps ) smf_close_related( &flatramps, status );
if (heateffmap) heateffmap = smf_free_effmap( heateffmap, status );
grpDelet( &igrp, status);
grpDelet( &ogrp, status);
if( keymap ) keymap = astAnnul( keymap );
if (extpars) extpars = astAnnul( extpars );
ndfEnd( status );
}
void smurf_calcqu( int *status ) {
/* Local Variables: */
AstFitsChan *fc; /* Holds FITS headers for output NDFs */
AstKeyMap *config; /* Holds all cleaning parameters */
AstKeyMap *dkpars; /* Holds dark squid cleaning parameters */
AstKeyMap *heateffmap = NULL; /* Heater efficiency data */
AstKeyMap *sub_instruments;/* Indicates which instrument is being used */
Grp *bgrp = NULL; /* Group of base names for each chunk */
Grp *igrp = NULL; /* Group of input files */
Grp *ogrp = NULL; /* Group of output files */
Grp *sgrp = NULL; /* Group of science files */
HDSLoc *loci = NULL; /* Locator for output I container file */
HDSLoc *locq = NULL; /* Locator for output Q container file */
HDSLoc *locu = NULL; /* Locator for output U container file */
NdgProvenance *oprov; /* Provenance to store in each output NDF */
ThrWorkForce *wf; /* Pointer to a pool of worker threads */
char headval[ 81 ]; /* FITS header value */
char ndfname[ 30 ]; /* Name of output Q or U NDF */
char polcrd[ 81 ]; /* FITS 'POL_CRD' header value */
char subarray[ 10 ]; /* Subarray name (e.g. "s4a", etc) */
double angrot; /* Angle from focal plane X axis to fixed analyser */
double paoff; /* WPLATE value corresponding to POL_ANG=0.0 */
float arcerror; /* Max acceptable error (arcsec) in one block */
int block_end; /* Index of last time slice in block */
int block_start; /* Index of first time slice in block */
int dkclean; /* Clean dark squids? */
int fix; /* Fix the POL-2 triggering issue? */
int iblock; /* Index of current block */
int iplace; /* NDF placeholder for current block's I image */
int ipolcrd; /* Reference direction for waveplate angles */
int maxsize; /* Max no. of time slices in a block */
int minsize; /* Min no. of time slices in a block */
int nc; /* Number of characters written to a string */
int pasign; /* +1 or -1 indicating sense of POL_ANG value */
int qplace; /* NDF placeholder for current block's Q image */
int submean; /* Subtract mean value from each time slice? */
int uplace; /* NDF placeholder for current block's U image */
size_t ichunk; /* Continuous chunk counter */
size_t idx; /* Subarray counter */
size_t igroup; /* Index for group of related input NDFs */
size_t inidx; /* Index into group of science input NDFs */
size_t nchunk; /* Number continuous chunks outside iter loop */
size_t ssize; /* Number of science files in input group */
smfArray *concat = NULL; /* Pointer to smfArray holding bolometer data */
smfArray *darks = NULL; /* dark frames */
smfArray *dkarray = NULL; /* Pointer to smfArray holding dark squid data */
smfArray *flatramps = NULL;/* Flatfield ramps */
smfData *data = NULL; /* Concatenated data for one subarray */
smfData *dkdata = NULL; /* Concatenated dark squid data for one subarray */
smfGroup *sgroup = NULL; /* smfGroup corresponding to sgrp */
/* Check inhereited status */
if( *status != SAI__OK ) return;
/* Start new AST and NDF contexts. */
astBegin;
ndfBegin();
/* Find the number of cores/processors available and create a work force
holding the same number of threads. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get a group of input files */
kpg1Rgndf( "IN", 0, 1, " Give more NDFs...", &igrp, &ssize, status );
/* Get a group containing just the files holding science data. */
smf_find_science( igrp, &sgrp, 0, NULL, NULL, 1, 1, SMF__NULL, &darks,
&flatramps, &heateffmap, NULL, status );
/* Check we have at least once science file. */
ssize = grpGrpsz( sgrp, status );
if( ssize == 0 ) {
msgOutif( MSG__NORM, " ", "All supplied input frames were DARK.",
status );
} else {
/* See if a correction should be made for the POL2 triggering issue. */
parGet0l( "FIX", &fix, status );
/* Create HDS container files to hold the output NDFs. */
datCreat( "OUTQ", "CALCQU", 0, 0, status );
datCreat( "OUTU", "CALCQU", 0, 0, status );
/* Associate the locators with the structures. */
datAssoc( "OUTQ", "WRITE", &locq, status );
datAssoc( "OUTU", "WRITE", &locu, status );
/* The I images are optional. */
if( *status == SAI__OK ) {
datCreat( "OUTI", "CALCQU", 0, 0, status );
datAssoc( "OUTI", "WRITE", &loci, status );
if( *status == PAR__NULL ) {
errAnnul( status );
loci = NULL;
}
}
/* Group the input files so that all files within a single group have the
same wavelength and belong to the same subscan of the same observation.
Also identify chunks of data that are contiguous in time, and
determine to which such chunk each group belongs. All this information
is returned in a smfGroup structure ("*sgroup"). */
smf_grp_related( sgrp, ssize, 1, 1, 0, NULL, NULL, NULL,
NULL, &sgroup, &bgrp, NULL, status );
/* Obtain the number of contiguous chunks. */
if( *status == SAI__OK ) {
nchunk = sgroup->chunk[ sgroup->ngroups - 1 ] + 1;
} else {
nchunk = 0;
}
/* Indicate we have not yet found a value for the ARCERROR parameter. */
arcerror = 0.0;
/* Loop over all contiguous chunks */
for( ichunk = 0; ichunk < nchunk && *status == SAI__OK; ichunk++ ) {
/* Display the chunk number. */
if( nchunk > 1 ) {
msgOutiff( MSG__VERB, "", " Doing chunk %d of %d.",
status, (int) ichunk + 1, (int) nchunk );
}
/* Concatenate the data within this contiguous chunk. This produces a
smfArray ("concat") containing a smfData for each subarray present in
the chunk. Each smfData holds the concatenated data for a single
subarray. */
smf_concat_smfGroup( wf, NULL, sgroup, darks, NULL, flatramps,
heateffmap, ichunk, 1, 1, NULL, 0, NULL, NULL,
0, 0, 0, &concat, NULL, status );
/* Get a KeyMap holding values for the configuration parameters. Since we
sorted by wavelength when calling smf_grp_related, we know that all
smfDatas in the current smfArray (i.e. chunk) will relate to the same
wavelength. Therefore we can use the same parameters for all smfDatas in
the current smfArray. */
sub_instruments = smf_subinst_keymap( SMF__SUBINST_NONE,
concat->sdata[ 0 ], NULL,
0, status );
config = kpg1Config( "CONFIG", "$SMURF_DIR/smurf_calcqu.def",
sub_instruments, status );
sub_instruments = astAnnul( sub_instruments );
/* Get the CALCQU specific parameters. */
if( !astMapGet0I( config, "PASIGN", &pasign ) ) pasign = 1;
msgOutiff( MSG__VERB, "", "PASIGN=%d", status, pasign );
if( !astMapGet0D( config, "PAOFF", &paoff ) ) paoff = 0.0;
msgOutiff( MSG__VERB, "", "PAOFF=%g", status, paoff );
if( !astMapGet0D( config, "ANGROT", &angrot ) ) angrot = 90.0;
msgOutiff( MSG__VERB, "", "ANGROT=%g", status, angrot );
if( !astMapGet0I( config, "SUBMEAN", &submean ) ) submean = 0;
msgOutiff( MSG__VERB, "", "SUBMEAN=%d", status, submean );
/* See if the dark squids should be cleaned. */
if( !astMapGet0I( config, "DKCLEAN", &dkclean ) ) dkclean = 0;
/* If required, clean the dark squids now since we might need to use them to
clean the bolometer data. */
if( dkclean ) {
/* Create a smfArray containing the dark squid data. For each one, store
a pointer to the main header so that smf_clean_smfArray can get at the
JCMTState information. */
dkarray = smf_create_smfArray( status );
for( idx = 0; idx < concat->ndat && *status == SAI__OK; idx++ ) {
data = concat->sdata[ idx ];
if( data && data->da && data->da->dksquid ) {
dkdata = data->da->dksquid;
dkdata->hdr = data->hdr;
smf_addto_smfArray( dkarray, dkdata, status );
}
}
/* Clean the smfArray containing the dark squid data. Use the "CLEANDK.*"
parameters. */
(void) astMapGet0A( config, "CLEANDK", &dkpars );
smf_clean_smfArray( wf, dkarray, NULL, NULL, NULL, dkpars, status );
dkpars = astAnnul( dkpars );
/* Nullify the header pointers so that we don't accidentally close any. */
if( dkarray ) {
for( idx = 0; idx < dkarray->ndat; idx++ ) {
dkdata = dkarray->sdata[ idx ];
dkdata->hdr = NULL;
}
/* Free the smfArray holding the dark squid data, but do not free the
individual smfDatas within it. */
dkarray->owndata = 0;
smf_close_related( &dkarray, status );
}
}
/* Now clean the bolometer data */
smf_clean_smfArray( wf, concat, NULL, NULL, NULL, config, status );
/* If required correct for the POL2 triggering issue. */
if( fix ) smf_fix_pol2( wf, concat, status );
/* Loop round each sub-array in the current contiguous chunk of data. */
for( idx = 0; idx < concat->ndat && *status == SAI__OK; idx++ ) {
data = concat->sdata[ idx ];
/* Find the name of the subarray that generated the data. */
smf_find_subarray( data->hdr, subarray, sizeof(subarray), NULL,
status );
/* Display the sub-array. */
if( concat->ndat > 1 ) {
msgOutiff( MSG__VERB, "", " Doing sub-array %s.",
status, subarray );
}
/* Create an empty provenance structure. Each input NDF that contributes
to the current chunk and array will be added as an ancestor to this
structure, which will later be stored in each output NDF created for
this chunk and array. */
oprov = ndgReadProv( NDF__NOID, "SMURF:CALCQU", status );
/* Indicate we do not yet have any FITS headers for the output NDFs */
fc = NULL;
/* Indicate we do not yet know the coordinate reference frame for the
half-waveplate angle. */
polcrd[ 0 ] = 0;
ipolcrd = 0;
/* Go through the smfGroup looking for groups of related input NDFs that
contribute to the current chunk. */
for( igroup = 0; igroup < sgroup->ngroups; igroup++ ) {
if( sgroup->chunk[ igroup ] == ichunk ) {
/* Get the integer index into the GRP group (sgrp) that holds the input NDFs.
This index identifies the input NDF that provides the data for the current
chunk and subarray. This assumes that the order in which smf_concat_smfGroup
stores arrays in the "concat" smfArray matches the order in which
smf_grp_related stores arrays within the sgroup->subgroups. */
inidx = sgroup->subgroups[ igroup ][ idx ];
/* Add this input NDF as an ancestor into the output provenance structure. */
smf_accumulate_prov( NULL, sgrp, inidx, NDF__NOID,
"SMURF:CALCQU", &oprov, status );
/* Merge the FITS headers from the current input NDF into the FitsChan
that holds headers for the output NDFs. The merging retains only those
headers which have the same value in all input NDFs. */
smf_fits_outhdr( data->hdr->fitshdr, &fc, status );
/* Get the polarimetry related FITS headers and check that all input NDFs
have usabie values. */
headval[ 0 ] = 0;
smf_getfitss( data->hdr, "POL_MODE", headval,
sizeof(headval), status );
if( strcmp( headval, "CONSTANT" ) && *status == SAI__OK ) {
*status = SAI__ERROR;
grpMsg( "N", sgrp, inidx );
errRep( " ", "Input NDF ^N does not contain "
"polarimetry data obtained with a spinning "
"half-waveplate.", status );
}
headval[ 0 ] = 0;
smf_getfitss( data->hdr, "POLWAVIN", headval,
sizeof(headval), status );
if( strcmp( headval, "Y" ) && *status == SAI__OK ) {
*status = SAI__ERROR;
grpMsg( "N", sgrp, inidx );
errRep( " ", "Half-waveplate was not in the beam for "
"input NDF ^N.", status );
}
headval[ 0 ] = 0;
smf_getfitss( data->hdr, "POLANLIN", headval,
sizeof(headval), status );
if( strcmp( headval, "Y" ) && *status == SAI__OK ) {
*status = SAI__ERROR;
grpMsg( "N", sgrp, inidx );
errRep( " ", "Analyser was not in the beam for input "
"NDF ^N.", status );
}
if( polcrd[ 0 ] ) {
headval[ 0 ] = 0;
smf_getfitss( data->hdr, "POL_CRD", headval,
sizeof(headval), status );
if( strcmp( headval, polcrd ) && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( " ", "Input NDFs have differing values for "
"FITS header 'POL_CRD'.", status );
}
} else {
smf_getfitss( data->hdr, "POL_CRD", polcrd,
sizeof(polcrd), status );
if( !strcmp( polcrd, "FPLANE" ) ) {
ipolcrd = 0;
} else if( !strcmp( polcrd, "AZEL" ) ) {
ipolcrd = 1;
} else if( !strcmp( polcrd, "TRACKING" ) ) {
ipolcrd = 2;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
msgSetc( "N", data->file->name );
msgSetc( "V", polcrd );
errRep( " ", "Input NDF ^N contains unknown value "
"'^V' for FITS header 'POL_CRD'.", status );
}
}
}
}
/* If not already done, get the maximum spatial drift (in arc-seconds) that
can be tolerated whilst creating a single I/Q/U image. The default value is
half the makemap default pixel size. Also get limits on the number of
time slices in any block. */
if( arcerror == 0.0 ) {
parDef0d( "ARCERROR", 0.5*smf_calc_telres( data->hdr->fitshdr,
status ), status );
parGet0r( "ARCERROR", &arcerror, status );
parGet0i( "MAXSIZE", &maxsize, status );
parGet0i( "MINSIZE", &minsize, status );
if( maxsize > 0 && maxsize < minsize && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", "Value of parameter MAXSIZE (%d) is less "
"than value of parameter MINSIZE (%d)", status,
maxsize, minsize );
}
}
/* The algorithm that calculates I, Q and U assumes that all samples for a
single bolometer measure flux from the same point on the sky. Due to
sky rotation, this will not be the case - each bolometer will drift
slowly across the sky. However, since the drift is (or should be)
slow we can apply the I/Q/U algorithm to blocks of contiguous data over
which the bolometers do not move significantly. We produce a separate
I, Q and U image for each such block. The first block starts at the first
time slice in the smfData. */
block_start = 0;
/* Find the time slice at which the corner bolometers have moved
a critical distance (given by parameter ARCERROR) from their
positions at the start of the block. Then back off some time slices
to ensure that the block holds an integral number of half-waveplate
rotations. */
block_end = smf_block_end( data, block_start, ipolcrd, arcerror,
maxsize, status );
/* Loop round creating I/Q/U images for each block. Count them. */
iblock = 0;
while( block_end >= 0 && *status == SAI__OK ) {
/* Skip very short blocks. */
if( block_end - block_start > minsize ) {
/* Display the start and end of the block. */
msgOutiff( MSG__VERB, "", " Doing time slice block %d "
"-> %d", status, (int) block_start,
(int) block_end );
/* Get the name for the Q NDF for this block. Start of with "Q" followed by
the block index. */
iblock++;
nc = sprintf( ndfname, "Q%d", iblock );
/* Append the subarray name to the NDF name. */
nc += sprintf( ndfname + nc, "_%s", subarray );
/* Append the chunk index to the NDF name. */
nc += sprintf( ndfname + nc, "_%d", (int) ichunk );
/* Get NDF placeholder for the Q NDF. The NDFs are created inside the
output container file. */
ndfPlace( locq, ndfname, &qplace, status );
/* The name of the U NDF is the same except the initial "Q" is changed to
"U". */
ndfname[ 0 ] = 'U';
ndfPlace( locu, ndfname, &uplace, status );
/* The name of the I NDF is the same except the initial "Q" is changed to
"I". */
if( loci ) {
ndfname[ 0 ] = 'I';
ndfPlace( loci, ndfname, &iplace, status );
} else {
iplace = NDF__NOPL;
}
/* Store the chunk and block numbers as FITS headers. */
atlPtfti( fc, "POLCHUNK", (int) ichunk, "Chunk index used by CALCQU", status );
atlPtfti( fc, "POLBLOCK", iblock, "Block index used by CALCQU", status );
/* Create the Q and U images for the current block of time slices from
the subarray given by "idx", storing them in the output container
file. */
smf_calc_iqu( wf, data, block_start, block_end, ipolcrd,
qplace, uplace, iplace, oprov, fc,
pasign, AST__DD2R*paoff, AST__DD2R*angrot,
submean, status );
/* Warn about short blocks. */
} else {
msgOutiff( MSG__VERB, "", " Skipping short block of %d "
"time slices (parameter MINSIZE=%d).", status,
block_end - block_start - 1, minsize );
}
/* The next block starts at the first time slice following the previous
block. */
block_start = block_end + 1;
/* Find the time slice at which the corner bolometers have moved
a critical distance (given by parameter ARCERROR) from their
positions at the start of the block. Then back off some time slices
to ensure that the block holds an integral number of half-waveplate
rotations. This returns -1 if all time slices have been used. */
block_end = smf_block_end( data, block_start, ipolcrd,
arcerror, maxsize, status );
}
/* Free resources */
oprov = ndgFreeProv( oprov, status );
fc = astAnnul( fc );
}
config = astAnnul( config );
/* Close the smfArray. */
smf_close_related( &concat, status );
}
/* Annul the locators for the output container files. */
datAnnul( &locq, status );
datAnnul( &locu, status );
if( loci ) datAnnul( &loci, status );
/* The parameter system hangs onto a primary locator for each container
file, so cancel the parameters to annul these locators. */
datCancl( "OUTQ", status );
datCancl( "OUTU", status );
datCancl( "OUTI", status );
}
/* Free resources. */
smf_close_related( &darks, status );
smf_close_related( &flatramps, status );
if( igrp ) grpDelet( &igrp, status);
if( sgrp ) grpDelet( &sgrp, status);
if( bgrp ) grpDelet( &bgrp, status );
if( ogrp ) grpDelet( &ogrp, status );
if( sgroup ) smf_close_smfGroup( &sgroup, status );
if (heateffmap) heateffmap = smf_free_effmap( heateffmap, status );
/* End the NDF and AST contexts. */
ndfEnd( status );
astEnd;
/* Issue a status indication.*/
if( *status == SAI__OK ) {
msgOutif( MSG__VERB, " ", "CALCQU succeeded.", status);
} else {
msgOutif( MSG__VERB, " ", "CALCQU failed.", status);
}
}
void smf_obsmap_fill( const smfData * data, AstKeyMap * obsmap, AstKeyMap * objmap,
int * status ) {
double dateobs = 0.0; /* MJD UTC of start of observation */
char object[SZFITSTR]; /* Object name */
char obsid[SZFITSTR]; /* Observation ID */
char instrume[SZFITSTR]; /* instrument name */
AstKeyMap * obsinfo = NULL; /* observation information */
if (*status != SAI__OK) return;
/* if there is no hdr we can not index it in the reported summary */
if (data->hdr && data->hdr->fitshdr) {
/* Get observation ID and see if we already have an entry in the map */
(void)smf_getobsidss( data->hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status );
/* As of 20080718 OBSID is not unique per obs so chop off date*/
obsid[22] = '\0';
if (!astMapHasKey(obsmap, obsid )) {
int itemp;
/* Create a sub keymap and populate it */
obsinfo = astKeyMap( " " );
/* fill with default value in case undefined */
one_strlcpy( object, "<undefined>", sizeof(object), status );
smf_getfitss( data->hdr, "OBJECT", object, sizeof(object),
status );
astMapPut0C( obsinfo, "OBJECT", object, NULL );
astMapPut0I( obsinfo, "OBSMODE", data->hdr->obsmode, NULL );
astMapPut0I( obsinfo, "OBSTYPE", data->hdr->obstype, NULL );
astMapPut0I( obsinfo, "SWMODE", data->hdr->swmode, NULL );
astMapPut0I( obsinfo, "INBEAM", data->hdr->inbeam, NULL );
smf_fits_getI( data->hdr, "OBSNUM", &itemp, status );
astMapPut0I( obsinfo, "OBSNUM", itemp, NULL );
smf_fits_getI( data->hdr, "UTDATE", &itemp, status );
astMapPut0I( obsinfo, "UTDATE", itemp, NULL );
smf_fits_getL( data->hdr, "SIMULATE", &itemp, status );
astMapPut0I( obsinfo, "SIMULATE", itemp, NULL );
smf_getfitss( data->hdr, "INSTRUME", instrume, sizeof(instrume),
status);
astMapPut0C( obsinfo, "INSTRUME", instrume, NULL );
/* store the MJD of observation for sorting purposes */
smf_find_dateobs( data->hdr, &dateobs, NULL, status );
astMapPut0D( obsinfo, "MJD-OBS", dateobs, NULL );
/* store information in the global observation map
and also track how many distinct objects we have */
astMapPut0A( obsmap, obsid, obsinfo, NULL );
astMapPut0I( objmap, object, 0, NULL );
obsinfo = astAnnul( obsinfo );
} else {
int curbeam = 0;
astMapGet0A( obsmap, obsid, &obsinfo );
/* INBEAM is interesting since technically each sequence can involve
different hardware being in the beam so we have to retrieve the
current value from the keymap and OR it with the current value
from the data header. */
/* we know that the value has been filled in previously so no
need to check */
astMapGet0I( obsinfo, "INBEAM", &curbeam );
curbeam |= data->hdr->inbeam;
astMapPut0I( obsinfo, "INBEAM", curbeam, NULL );
obsinfo = astAnnul( obsinfo );
}
}
return;
}
void smf_calcmodel_noi( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel, int flags,
int *status) {
/* Local Variables */
dim_t bolostep; /* Number of bolos per thread */
dim_t boxsize; /* No. of time slices in each noise box */
smfData *box = NULL; /* SmfData holding one box of input data */
size_t bstride; /* bolometer stride */
int calcfirst=0; /* Were bolo noises already measured? */
int dclimcorr; /* Min number of correlated steps */
int dcmaxsteps; /* Maximum allowed number of dc jumps */
dim_t dcfitbox; /* Width of box for DC step detection */
double dcthresh; /* Threshold for DC step detection */
dim_t dcsmooth; /* Width of median filter in DC step detection*/
double *din; /* Pointer to next input value */
double *dout; /* Pointer to next output value */
int fillgaps; /* If set perform gap filling */
dim_t i; /* Loop counter */
dim_t ibolo; /* Bolometer index */
int ibox; /* Index of current noise box */
dim_t itime; /* Time slice index */
dim_t idx=0; /* Index within subgroup */
JCMTState *instate=NULL; /* Pointer to input JCMTState */
int iw; /* Thread index */
dim_t j; /* Loop counter */
AstKeyMap *kmap=NULL; /* Local keymap */
size_t mbstride; /* model bolometer stride */
dim_t mntslice; /* Number of model time slices */
size_t mtstride; /* model time slice stride */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
dim_t nbolo; /* Number of bolometers */
int nbox = 0; /* Number of noise boxes */
size_t nchisq; /* Number of data points in chisq calc */
dim_t nelbox; /* Number of data points in a noise box */
dim_t ndata; /* Total number of data points */
size_t nflag; /* Number of new flags */
int nleft; /* Number of samples not in a noise box */
dim_t ntslice; /* Number of time slices */
int nw; /* Number of worker threads */
size_t pend; /* Last non-PAD sample */
size_t pstart; /* First non-PAD sample */
smf_qual_t *qin; /* Pointer to next input quality value */
smf_qual_t *qout; /* Pointer to next output quality value */
smfArray *qua=NULL; /* Pointer to RES at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to RES at chunk */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
dim_t spikebox=0; /* Box size for spike detection */
double spikethresh=0; /* Threshold for spike detection */
size_t tend; /* Last input sample to copy */
size_t tstart; /* First input sample to copy */
size_t tstride; /* time slice stride */
double *var=NULL; /* Sample variance */
size_t xbstride; /* Box bolometer stride */
int zeropad; /* Pad with zeros? */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointer to sub-keymap containing NOI parameters */
astMapGet0A( keymap, "NOI", &kmap );
/* Assert bolo-ordered data */
smf_model_dataOrder( dat, allmodel, chunk, SMF__RES|SMF__QUA, 0, status );
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
model = allmodel[chunk];
/* Obtain parameters for NOI */
/* Data-cleaning parameters */
smf_get_cleanpar( kmap, res->sdata[0], NULL, &dcfitbox, &dcmaxsteps, &dcthresh,
&dcsmooth, &dclimcorr, NULL, &fillgaps, &zeropad, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
&spikethresh, &spikebox, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, status );
/* Did we already calculate the noise on each detector? */
astMapGet0I( kmap, "CALCFIRST", &calcfirst );
/* Initialize chisquared */
dat->chisquared[chunk] = 0;
nchisq = 0;
/* Loop over index in subgrp (subarray) */
for( idx=0; idx<res->ndat; idx++ ) {
/* Get pointers to DATA components */
res_data = (res->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Get the raw data dimensions */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice, &ndata,
&bstride, &tstride, status );
/* NOI model dimensions */
smf_get_dims( model->sdata[idx], NULL, NULL, NULL, &mntslice, NULL,
&mbstride, &mtstride, status );
/* Only estimate the white noise level once at the beginning - the
reason for this is to make measurements of the convergence
easier. We either do it prior to the start of iterations (in which
case the relative weights will be influeced by low-frequency noise,
this is initialized in smf_model_create), or else we calculate
the noise after the first iteration. */
if( (flags & SMF__DIMM_FIRSTITER) && (!calcfirst) ) {
/* There are two forms for the NOI model: one constant noise value
for each bolometer, or "ntslice" noise values for each bolometer.
Handle the first case now. */
if( mntslice == 1 ) {
var = astMalloc( nbolo*sizeof(*var) );
if (var) {
/* Measure the noise from power spectra */
smf_bolonoise( wf, res->sdata[idx], 0, 0.5, SMF__F_WHITELO,
SMF__F_WHITEHI, 0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT,
var, NULL, NULL, status );
for( i=0; i<nbolo; i++ ) if( !(qua_data[i*bstride]&SMF__Q_BADB) ) {
/* Loop over time and store the variance for each sample */
for( j=0; j<mntslice; j++ ) {
model_data[i*mbstride+(j%mntslice)*mtstride] = var[i];
}
}
var = astFree( var );
}
/* If the NOI model is of the second form, the noise is estimated
in boxes of samples lasting "NOI.BOX_SIZE" seconds, and then the
noise level in the box is assigned to all samples in the box. */
} else if( mntslice == ntslice ) {
/* If not already done, get NOI.BOX_SIZE and convert from seconds to
samples. */
if( idx == 0 ) {
boxsize = 0;
smf_get_nsamp( kmap, "BOX_SIZE", res->sdata[0], &boxsize, status );
msgOutf( "", FUNC_NAME ": Calculating a NOI variance for each "
"box of %d samples.", status, (int) boxsize );
/* Find the indices of the first and last non-PAD sample. */
smf_get_goodrange( qua_data, ntslice, tstride, SMF__Q_PAD,
&pstart, &pend, status );
/* How many whole boxes fit into this range? */
nbox = ( pend - pstart + 1 ) / boxsize;
if( nbox == 0 ) nbox = 1;
/* How many samples would be left over at the end if we used this
many boxes? */
nleft = ( pend - pstart + 1 ) - nbox*boxsize;
/* Increase "boxsize" to reduce this number as far as possible.
Any samples that are left over after this increase of boxsize
will not be used when calculating the noise levels in each
bolometer. */
boxsize += nleft/nbox;
/* Create a smfData to hold one box-worth of input data. We
do not need to copy jcmtstate information. */
if( res->sdata[idx]->hdr ) {
instate = res->sdata[idx]->hdr->allState;
res->sdata[idx]->hdr->allState = NULL;
}
box = smf_deepcopy_smfData( res->sdata[idx], 0,
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY,
0, 0, status );
if( instate ) res->sdata[idx]->hdr->allState = instate;
/* Set the length of the time axis to the box size plus padding,
and create empty data and quality arrays for it. */
if( *status == SAI__OK ) {
box->dims[ box->isTordered?2:0 ] = boxsize + pstart + (ntslice - pend - 1);
smf_get_dims( box, NULL, NULL, NULL, NULL, &nelbox,
&xbstride, NULL, status );
box->pntr[0] = astMalloc( sizeof( double )*nelbox );
box->qual = astMalloc( sizeof( smf_qual_t )*nelbox );
/* For every bolometer, flag the start and end of the quality
array as padding, and store zeros in the data array. */
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
dout = ((double *) box->pntr[0]) + xbstride*ibolo;
qout = box->qual + xbstride*ibolo;
for( itime = 0; itime < pstart; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
dout = ((double *) box->pntr[0]) + xbstride*ibolo + pstart + boxsize;;
qout = box->qual + xbstride*ibolo + pstart + boxsize;
for( itime = pend + 1; itime < ntslice; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
}
}
}
/* Work space to hold the variance for each bolometer in a box */
var = astMalloc( nbolo*sizeof(*var) );
if( *status == SAI__OK ) {
/* Index of the first time slice within the input smfData
that is included in the first box. */
tstart = pstart;
/* Loop round each noise box */
for( ibox = 0; ibox < nbox; ibox++ ) {
/* Copy the data and quality values for this box from the
input smfData into "box", leaving room for padding at
both ends of box. Note, data is bolo-ordered so we
can assume that "tstride" is 1. */
din = ((double *)(res->sdata[idx]->pntr[0])) + tstart;
dout = ((double *)(box->pntr[0])) + pstart;
qin = qua_data + tstart;
qout = box->qual + pstart;
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
memcpy( dout, din, boxsize*sizeof( *din ) );
memcpy( qout, qin, boxsize*sizeof( *qin ) );
din += bstride;
dout += xbstride;
qin += bstride;
qout += xbstride;
}
/* Measure the noise from power spectra in the box. */
smf_bolonoise( wf, box, 0, 0.5, SMF__F_WHITELO, SMF__F_WHITEHI,
0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT, var,
NULL, NULL, status );
/* Loop over time and store the variance for each sample in
the NOI model. On the last box, pick up any left over time
slices. */
if( ibox < nbox - 1 ) {
tend = tstart + boxsize - 1;
} else {
tend = pend;
}
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
if( !( qua_data[ ibolo*bstride ] & SMF__Q_BADB ) ) {
dout = model_data + ibolo*bstride + tstart;
for( itime = tstart; itime <= tend; itime++ ) {
*(dout++) = var[ ibolo ];
}
}
}
/* Update the index of the first time slice within the input
smfData that is included in the next box. */
tstart += boxsize;
}
var = astFree( var );
}
/* Report an error if the number of samples for each bolometer in
the NOI model is not 1 or "ntslice". */
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": NOI model has %d samples - should be "
"%d or 1.", status, (int) mntslice, (int) ntslice);
}
}
if( kmap ) {
/* Flag spikes in the residual after first iteration */
if( spikethresh && !(flags&SMF__DIMM_FIRSTITER) ) {
/* Now re-flag */
smf_flag_spikes( wf, res->sdata[idx], SMF__Q_MOD,
spikethresh, spikebox, &nflag, status );
msgOutiff(MSG__VERB," ", " flagged %zu new %lf-sig spikes",
status, nflag, spikethresh );
}
if( dcthresh && dcfitbox ) {
smf_fix_steps( wf, res->sdata[idx], dcthresh, dcsmooth,
dcfitbox, dcmaxsteps, dclimcorr, 1, &nflag, NULL,
NULL, status );
msgOutiff(MSG__VERB, ""," detected %zu bolos with DC steps\n",
status, nflag);
}
}
/* Now calculate contribution to chi^2. This bit takes along time
if there is a lot of data so share the work out amongst the available
worker threads. How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many bolometers to process in each worker thread. */
bolostep = nbolo/nw;
if( bolostep == 0 ) bolostep = 1;
/* Allocate job data for threads, and store the range of bolos to be
processed by each one. Ensure that the last thread picks up any
left-over bolos. */
SmfCalcModelNoiData *job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
SmfCalcModelNoiData *pdata;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->b1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->b2 = pdata->b1 + bolostep - 1;
} else {
pdata->b2 = nbolo - 1 ;
}
/* Store other values common to all jobs. */
pdata->ntslice = ntslice;
pdata->mntslice = mntslice;
pdata->qua_data = qua_data;
pdata->model_data = model_data;
pdata->res_data = res_data;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->mbstride = mbstride;
pdata->mtstride = mtstride;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_calcmodel_noi, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Accumulate the results from all the worker threads. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
dat->chisquared[chunk] += pdata->chisquared;
nchisq += pdata->nchisq;
}
/* Free the job data. */
job_data = astFree( job_data );
}
}
}
/* Free resources */
if( box ) {
box->pntr[0] = astFree( box->pntr[0] );
box->qual = astFree( box->qual );
smf_close_file( &box, status );
}
/* Normalize chisquared for this chunk */
if( (*status == SAI__OK) && (nchisq >0) ) {
dat->chisquared[chunk] /= (double) nchisq;
}
/* Clean Up */
if( kmap ) kmap = astAnnul( kmap );
}
void atlKy2hd( AstKeyMap *keymap, HDSLoc *loc, int *status ) {
/*
* Name:
* atlKy2hd
* Purpose:
* Copies values from an AST KeyMap to a primitive HDS object.
* Language:
* C.
* Invocation:
* void atlKy2hd( AstKeyMap *keymap, HDSLoc *loc, int *status )
* Description:
* This routine copies the contents of an AST KeyMap into a supplied
* HDS structure.
* Arguments:
* keymap
* An AST pointer to the KeyMap.
* loc
* A locator for the HDS object into which the KeyMap contents
* are to be copied. A new component is added to the HDS object for
* each entry in the KeyMap.
* status
* The inherited status.
* Copyright:
* Copyright (C) 2008, 2010, 2012 Science & Technology Facilities Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful,but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* TIMJ: Tim Jenness (JAC, Hawaii)
* {enter_new_authors_here}
* History:
* 29-APR-2008 (DSB):
* Original version.
* 2010-09-23 (TIMJ):
* Fix arrays of strings.
* 2010-09-30 (TIMJ):
* Make sure that we are using the correct status pointer in AST.
* 2010-10-01 (TIMJ):
* Sort the keys when writing to HDS structured.
* 2010-10-04 (TIMJ):
* Support Short ints in keymap
* 14-SEP-2012 (DSB):
* Moved from kaplibs to atl.
* 17-SEP-2012 (DSB):
* Add support for undefined values.
* {enter_further_changes_here}
* Bugs:
* {note_any_bugs_here}
*/
/* Local Varianles: */
AstObject **objArray = NULL;
AstObject *obj = NULL;
HDSLoc *cloc = NULL;
HDSLoc *dloc = NULL;
const char *cval = NULL;
const char *key;
double dval;
float fval;
int i;
int ival;
int j;
int lenc;
int nval;
char *oldsortby;
int *oldstat = NULL;
int size;
int type;
int veclen;
size_t el;
void *pntr = NULL;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Make sure that we are checking AST status */
oldstat = astWatch( status );
/* If set, save the old SortBy value and then ensure alphabetical sorting.
We need to take a copy of the original string since the buffer in which
the string is stored may be re-used by subsequent incocations of astGetC. */
if( astTest( keymap, "SortBy" ) ) {
int nc = 0;
oldsortby = astAppendString( NULL, &nc, astGetC( keymap, "SortBy" ) );
} else {
oldsortby = NULL;
}
astSet( keymap, "SortBy=KeyUp" );
/* Loop round each entry in the KeyMap. */
size = astMapSize( keymap );
for( i = 0; i < size; i++ ) {
if (*status != SAI__OK) break;
/* Get the key. the data type and the vector length for the current
KeyMap entry. */
key = astMapKey( keymap, i );
type = astMapType( keymap, key );
veclen = astMapLength( keymap, key );
/* If the current entry holds one or more nested KeyMaps, then we call
this function recursively to add them into a new HDS component. */
if( type == AST__OBJECTTYPE ) {
/* First deal with scalar entries holding a single KeyMap. */
if( veclen == 1 ) {
datNew( loc, key, "KEYMAP_ENTRY", 0, NULL, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0A( keymap, key, &obj );
if( astIsAKeyMap( obj ) ) {
atlKy2hd( (AstKeyMap *) obj, cloc, status );
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "atlKy2hd: Supplied KeyMap contains unusable AST "
"objects (programming error).", status );
}
datAnnul( &cloc, status );
/* Now deal with vector entries holding multiple KeyMaps. */
} else {
datNew( loc, key, "KEYMAP_ENTRY", 1, &veclen, status );
datFind( loc, key, &cloc, status );
objArray = astMalloc( sizeof( AstObject *) * (size_t)veclen );
if( objArray ) {
(void) astMapGet1A( keymap, key, veclen, &nval, objArray );
for( j = 1; j <= veclen; j++ ) {
datCell( cloc, 1, &j, &dloc, status );
if( astIsAKeyMap( objArray[ j - 1 ] ) ) {
atlKy2hd( (AstKeyMap *) objArray[ j - 1 ], dloc, status );
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "atlKy2hd: Supplied KeyMap contains unusable AST "
"objects (programming error).", status );
}
datAnnul( &dloc, status );
}
objArray = astFree( objArray );
}
datAnnul( &cloc, status );
}
/* For primitive types... */
} else if( type == AST__INTTYPE ) {
if( veclen == 1 ) {
datNew0I( loc, key, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0I( keymap, key, &ival );
datPut0I( cloc, ival, status );
datAnnul( &cloc, status );
} else {
datNew1I( loc, key, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_INTEGER", "WRITE", &pntr, &el, status );
(void) astMapGet1I( keymap, key, veclen, &nval, (int *) pntr );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
} else if( type == AST__SINTTYPE ) {
short sval = 0;
if( veclen == 1 ) {
datNew0W( loc, key, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0S( keymap, key, &sval );
datPut0W( cloc, sval, status );
datAnnul( &cloc, status );
} else {
datNew1W( loc, key, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_WORD", "WRITE", &pntr, &el, status );
(void) astMapGet1S( keymap, key, veclen, &nval, (short *) pntr );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
} else if( type == AST__DOUBLETYPE ) {
if( veclen == 1 ) {
datNew0D( loc, key, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0D( keymap, key, &dval );
datPut0D( cloc, dval, status );
datAnnul( &cloc, status );
} else {
datNew1D( loc, key, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_DOUBLE", "WRITE", &pntr, &el, status );
(void) astMapGet1D( keymap, key, veclen, &nval, (double *) pntr );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
} else if( type == AST__FLOATTYPE ) {
if( veclen == 1 ) {
datNew0R( loc, key, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0F( keymap, key, &fval );
datPut0R( cloc, fval, status );
datAnnul( &cloc, status );
} else {
datNew1R( loc, key, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_REAL", "WRITE", &pntr, &el, status );
(void) astMapGet1F( keymap, key, veclen, &nval, (float *) pntr );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
} else if( type == AST__STRINGTYPE ) {
lenc = astMapLenC( keymap, key );
if( veclen == 1 ) {
datNew0C( loc, key, lenc, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0C( keymap, key, &cval );
datPut0C( cloc, cval, status );
datAnnul( &cloc, status );
} else {
datNew1C( loc, key, lenc, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_CHAR", "WRITE", &pntr, &el, status );
(void) atlMapGet1C( keymap, key, veclen*lenc, lenc, &nval,
(char *) pntr, status );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
/* KeyMap "UNDEF" values are always scalar and have no corresponding HDS
data type. So arbitrarily use an "_INTEGER" primitive with no defined
value to represent a KeyMap UNDEF value. */
} else if( type == AST__UNDEFTYPE ) {
datNew0L( loc, key, status );
/* Unknown or unsupported data types. */
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "T", type );
errRep( "", "atlKy2hd: Supplied KeyMap contains entries with "
"unusable data type (^T) (programming error).", status );
}
}
/* If it was originally set, re-instate the old SortBy value in the KeyMap,
then free the memory. Otherwise, clear the SortBy attribute. */
if( oldsortby ) {
astSetC( keymap, "SortBy", oldsortby );
oldsortby = astFree( oldsortby );
} else {
astClear( keymap, "SortBy" );
}
/* Reset AST status */
astWatch( oldstat );
}
static void DisplayKeyMap( AstKeyMap *km, int sort, const char *prefix,
AstKeyMap *refkm, int *status ){
/*
* Name:
* DisplayKeyMap
* Purpose:
* Display the contents of a keymap.
* Synopsis:
* void DisplayKeyMap( AstKeyMap *km, int sort, const char *prefix,
* int *status )
* Arguments:
* km
* Pointer to the KeyMaps containing the values to display.
* sort
* If non-zero, sort the values alphabetically by their keys.
* prefix
* A string to prepend to eack key.
* refkm
* Reference key map (e.g. values from the supplied configuration
* rather than the NDF history), or null if not required.
* status
* Inherited status pointer.
* Description:
* This function displays the contents of a supplied KeyMap as
* a series of "key = value" strings, one per line. It calls itself
* recursively if a nested KeyMap is found, adding a suitable
* prefix to the nested keys. If a reference key map is supplied then
* the output shows how the main key map differs from it.
*/
/* Local Variables: */
AstObject *avalue;
AstObject *refavalue;
char cbuffer[ 255 ];
char newpref[ 255 ];
const char *cvalue;
const char *refcvalue;
const char *key;
int ikey;
int ival;
int nc;
int nkey;
int nval;
/* Check the inherited status */
if( *status != SAI__OK ) return;
if (refkm) astClear(refkm, "KeyError");
/* Sort the supplied KeyMap is required. */
if( sort ) astSetC( km, "SortBy", "KeyUp" );
/* Loop round all keys in the supplied KeyMap. */
nkey = astMapSize( km );
for( ikey = 0; ikey < nkey; ikey++ ) {
key = astMapKey( km, ikey );
/* If the current entry is a nest KeyMap, get a pointer to it and call
this function recurisvely to display it, modifying the prefix to add
to each key so that it includes the key associated with the nest keymap. */
if( astMapType( km, key ) == AST__OBJECTTYPE ) {
astMapGet0A( km, key, &avalue );
if (refkm) {
if (! astMapGet0A(refkm, key, &refavalue)) {
refavalue = (AstObject*) astKeyMap("");
}
}
else {
refavalue = NULL;
}
sprintf( newpref, "%s%s.", prefix, key );
DisplayKeyMap( (AstKeyMap *) avalue, sort, newpref,
(AstKeyMap *) refavalue, status );
avalue = astAnnul( avalue );
if (refavalue) refavalue = astAnnul(refavalue);
/* If the current entry is not a nested keymap, we display it now. */
} else {
/* Get the vector length of the entry. */
nval = astMapLength( km, key );
/* If it is a scalar, just get its value as a character string using
the automatic type conversion provided by the KeyMap class, and
display it, putting the supplied prefix at the start of the key. */
if( nval <= 1 ) {
cvalue = "<undef>";
astMapGet0C( km, key, &cvalue );
if (refkm) {
refcvalue = "<undef>";
if (astMapGet0C(refkm, key, &refcvalue)
&& ! strcmp(cvalue, refcvalue)) {
msgOutf("", "- %s%s = %s", status, prefix, key, cvalue);
}
else {
msgOutf("", "+ %s%s = %s", status, prefix, key, cvalue);
}
}
else {
msgOutf( "", "%s%s = %s", status, prefix, key, cvalue );
}
/* If it is a vector, we construct a string containing a comma-separated
list of elements, enclosed in parentheses. */
} else {
nc = 0;
cvalue = astAppendString( NULL, &nc, "(" );
for( ival = 0; ival < nval; ival++ ) {
if( astMapGetElemC( km, key, sizeof( cbuffer) - 1, ival,
cbuffer ) ) {
cvalue = astAppendString( (char *) cvalue, &nc, cbuffer );
}
if( ival < nval - 1 ) cvalue = astAppendString( (char *) cvalue,
&nc, "," );
}
cvalue = astAppendString( (char *) cvalue, &nc, ")" );
/* Do the same for the reference KeyMap. */
if (refkm && (nval = astMapLength(refkm, key))) {
nc = 0;
refcvalue = astAppendString(NULL, &nc, "(");
for (ival = 0; ival < nval; ival++) {
if (ival) {
refcvalue = astAppendString((char*) refcvalue, &nc, "," );
}
if (astMapGetElemC(refkm, key, sizeof(cbuffer) - 1, ival,
cbuffer)) {
refcvalue = astAppendString((char*) refcvalue, &nc,
cbuffer);
}
}
refcvalue = astAppendString((char*) refcvalue, &nc, ")");
}
else {
refcvalue = NULL;
}
/* Display the total string, with the current key prefix, and free the memory
used to store it. */
if (refkm) {
if (refcvalue && ! strcmp(cvalue, refcvalue)) {
msgOutf("", "- %s%s = %s", status, prefix, key, cvalue);
}
else {
msgOutf("", "+ %s%s = %s", status, prefix, key, cvalue);
}
}
else {
msgOutf( "", "%s%s = %s", status, prefix, key, cvalue );
}
cvalue = astFree( (void *) cvalue );
if (refcvalue) refcvalue = astFree((void*) refcvalue);
}
}
}
}
void kpg1Kygp1( AstKeyMap *keymap, Grp **igrp, const char *prefix,
int *status ){
/*
*+
* Name:
* kpg1Kygp1
* Purpose:
* Creates a GRP group holding keyword/value pairs read from an AST KeyMap.
* Language:
* C.
* Invocation:
* void kpg1Kygp1( AstKeyMap *keymap, Grp **igrp, const char *prefix,
* int *status )
* Description:
* This function is the inverse of kpg1Kymp1. It extracts the values
* from the supplied AST KeyMap and creates a set of "name=value" strings
* which it appends to a supplied group (or creates a new group). If
* the KeyMap contains nested KeyMaps, then the "name" associated with
* each primitive value stored in the returned group is a hierarchical
* list of component names separated by dots.
* Arguments:
* keymap
* A pointer to the KeyMap. Numerical entries which have bad values
* (VAL__BADI for integer entries or VAL__BADD for floating point
* entries) are not copied into the group.
* igrp
* A location at which is stored a pointer to the Grp structure
* to which the name=value strings are to be appended. A new group is
* created and a pointer to it is returned if the supplied Grp
* structure is not valid.
* prefix
* A string to append to the start of each key extracted from the
* supplied KeyMap. If NULL, no prefix is used.
* status
* Pointer to the inherited status value.
* Notes:
* - This function provides a private implementation for the public
* KPG1_KYGRP Fortran routine and kpg1Kygrp C function.
* - Entries will be stored in the group in alphabetical order
* Copyright:
* Copyright (C) 2008,2010 Science & Technology Facilities Council.
* Copyright (C) 2005 Particle Physics & Astronomy Research Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful,but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* TIMJ: Tim Jenness (JAC, Hawaii)
* {enter_new_authors_here}
* History:
* 7-NOV-2005 (DSB):
* Original version.
* 15-JUL-2008 (TIMJ):
* Tweak to GRP C API.
* 2010-07-19 (TIMJ):
* Handle vector keymap entries.
* 2010-08-12 (TIMJ):
* Store entries in group in alphabetical order.
* 13-AUG-2010 (DSB):
* Re-instate the original SortBy value before exiting.
* {enter_further_changes_here}
* Bugs:
* {note_any_bugs_here}
*-
*/
/* Local Variables: */
AstObject *obj; /* Pointer to nested AST Object */
char *oldsortby; /* The old value of the KeyMap's SortBy attribute */
char *text; /* Sum of concatenated strings */
const char *key; /* Key string for current entry in KeyMap */
const char *value; /* Value of current entry in KeyMap */
double dval; /* Double value */
int *old_status; /* Pointer to original status variable */
int bad; /* Is the numerical entry value bad? */
int i; /* Index into supplied KeyMap */
int ival; /* Integer value */
int n; /* Number of entries in the KeyMap */
int nc; /* Length of "text" excluding trailing null */
int type; /* Data type of current entry in KeyMap */
int valid; /* Is the supplied GRP structure valid? */
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Make AST use the Fortran status variable. */
old_status = astWatch( status );
/* Create a new GRP group if required. */
valid = grpValid( *igrp, status );
if( !valid ) *igrp = grpNew( "Created by kpg1_Kygp1", status );
/* If set, save the old SortBy value and then ensure alphabetical sorting.
We need to take a copy of the original string since the buffer in which
the string is stored may be re-used by subsequent incocations of astGetC. */
if( astTest( keymap, "SortBy" ) ) {
nc = 0;
oldsortby = astAppendString( NULL, &nc, astGetC( keymap, "SortBy" ) );
} else {
oldsortby = NULL;
}
astSet( keymap, "SortBy=KeyUp" );
/* Get the number of entries in the KeyMap. */
n = astMapSize( keymap );
/* Loop round all the entries in the KeyMap.*/
for( i = 0; i < n; i++ ) {
/* Get the name and type of the current KeyMap entry. */
key = astMapKey( keymap, i );
type = astMapType( keymap, key );
/* If the entry is an AST Object, get a pointer to it.*/
if( type == AST__OBJECTTYPE && astMapGet0A( keymap, key, &obj ) ) {
/* If it is a nested KeyMap, update the prefix and call this function
recursively. We ignore other forms of AST Objects. */
if( astIsAKeyMap( obj ) ) {
nc = 0;
text = astAppendString( NULL, &nc, prefix );
text = astAppendString( text, &nc, key );
text = astAppendString( text, &nc, "." );
kpg1Kygp1( (AstKeyMap *) obj, igrp, text, status );
text = astFree( text );
}
/* If it is a primitive, format it and add it to the group. */
} else {
/* If it is a numerical type, see if it has a bad value. */
bad = 0;
if( type == AST__INTTYPE && astMapGet0I( keymap, key, &ival ) ){
if( ival == VAL__BADI ) bad = 1;
} else if( type == AST__DOUBLETYPE && astMapGet0D( keymap, key, &dval ) ){
if( dval == VAL__BADD ) bad = 1;
}
/* If it not bad, get its formatted value. We also make sure that astMapGet0C returns
true because we intend to skip undefined values. */
if( !bad && astMapGet0C( keymap, key, &value ) ) {
size_t length;
/* Write the key and equals sign to a buffer */
nc = 0;
text = astAppendString( NULL, &nc, prefix );
text = astAppendString( text, &nc, key );
text = astAppendString( text, &nc, "=" );
length = astMapLength( keymap, key );
if( length > 1 ) {
/* Vector so we need to use (a,b,c) syntax */
char thiselem[GRP__SZNAM+1];
size_t l;
text = astAppendString( text, &nc, "(");
for ( l = 0; l < length; l++) {
if( astMapGetElemC( keymap, key, sizeof(thiselem), l, thiselem ) ) {
text = astAppendString( text, &nc, thiselem );
}
/* always deal with the comma. Even if value was undef we need to put in the comma */
if( l < (length - 1) ) {
text = astAppendString( text, &nc, "," );
}
}
text = astAppendString( text, &nc, ")");
} else {
/* Scalar */
text = astAppendString( text, &nc, value );
}
/* Put it in the group. */
grpPut1( *igrp, text, 0, status );
text = astFree( text );
}
}
}
/* If it was originally set, re-instate the old SortBy value in the KeyMap,
then free the memory. Otherwise, clear the SortBy attribute. */
if( oldsortby ) {
astSetC( keymap, "SortBy", oldsortby );
oldsortby = astFree( oldsortby );
} else {
astClear( keymap, "SortBy" );
}
/* Make AST use its original status variable. */
astWatch( old_status );
}
void smf_calcmodel_pln( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel,
int flags __attribute__((unused)),
int *status) {
/* Local Variables */
size_t bstride; /* bolo stride */
size_t i; /* Loop counter */
dim_t idx=0; /* Index within subgroup */
AstKeyMap *kmap=NULL; /* Pointer to PLN-specific keys */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
double *model_data_copy=NULL; /* Copy of model_data for one bolo */
int *lut_data = NULL; /* Lut data for one subarray */
dim_t nbolo=0; /* Number of bolometers */
dim_t ndata=0; /* Total number of data points */
int notfirst=0; /* flag for delaying until after 1st iter */
dim_t ntslice=0; /* Number of time slices */
smfArray *qua=NULL; /* Pointer to QUA at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to quality data */
smfArray *res=NULL; /* Pointer to RES at chunk */
smfArray *lut=NULL; /* Pointer to LUT at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
size_t tstride; /* Time slice stride in data array */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointer to sub-keymap containing PLN parameters. Something will
always be available.*/
astMapGet0A( keymap, "PLN", &kmap );
/* Are we skipping the first iteration? */
astMapGet0I(kmap, "NOTFIRST", ¬first);
if( notfirst && (flags & SMF__DIMM_FIRSTITER) ) {
msgOutif( MSG__VERB, "", FUNC_NAME
": skipping PLN this iteration", status );
return;
}
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
lut = dat->lut[chunk];
/* Assert ICD-ordered data */
smf_model_dataOrder( wf, dat, allmodel, chunk, SMF__RES|SMF__QUA|SMF__LUT,
1, status );
smf_get_dims( res->sdata[0], NULL, NULL, NULL, NULL,
&ndata, NULL, NULL, status);
model = allmodel[chunk];
/* Loop over index in subgrp (subarray) and put the previous iteration
of the filtered component back into the residual before calculating
and removing the new filtered component */
for( idx=0; (*status==SAI__OK)&&(idx<res->ndat); idx++ ) {
/* Obtain dimensions of the data */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice,
&ndata, &bstride, &tstride, status);
/* Get pointers to data/quality/model */
res_data = (res->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
lut_data = (lut->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
if( *status == SAI__OK ) {
/* Place last iteration of plane signal back into residual */
for (i=0; i< nbolo*ntslice; i++) {
if ( !(qua_data[i]&SMF__Q_MOD) && res_data[i] != VAL__BADD && model_data[i] != VAL__BADD ) {
res_data[i] += model_data[i];
}
}
/* Copy the residual+old model into model_data where it will be
fitted again in this iteration. */
memcpy( model_data, res_data,
ndata*smf_dtype_size(res->sdata[idx],status) );
}
/* Calculate the fit and subtract it*/
smf_subtract_plane3( wf, res->sdata[idx], dat->mdims, lut_data, status );
/* Store the difference between the plane-subtracted signal and the residual
in the model container */
if( *status == SAI__OK ) {
for (i=0; i< nbolo*ntslice; i++) {
if ( !(qua_data[i]&SMF__Q_MOD) && res_data[i] != VAL__BADD && model_data[i] != VAL__BADD ) {
model_data[i] -= res_data[i];
} else {
model_data[i] = VAL__BADD;
}
}
}
}
}
if( kmap ) kmap = astAnnul( kmap );
model_data_copy = astFree( model_data_copy );
}
void smf_updateprov( int ondf, const smfData *data, int indf,
const char *creator, NdgProvenance **modprov, int *status ){
/* Local Variables */
AstFitsChan *fc = NULL; /* AST FitsChan holding input FITS header */
AstKeyMap *anc = NULL; /* KeyMap holding ancestor info */
AstKeyMap *tkm = NULL; /* KeyMap holding contents of MORE */
NdgProvenance *prov = NULL; /* Pointer to input provenance structure */
char *obsidss = NULL; /* Pointer to OBSIDSS buffer */
char obsidssbuf[SZFITSTR]; /* OBSIDSS value in input file */
const char *vptr = NULL; /* Pointer to OBSIDSS string */
int found; /* Was OBSIDSS value found in an input ancestor? */
int ianc; /* Ancestor index */
int isroot; /* Ignore any ancestors in the input NDF? */
/* Check the inherited status */
if( *status != SAI__OK ) return;
/* Get a FitsChan holding the contents of the input NDF FITS extension. */
if( !data ) {
kpgGtfts( indf, &fc, status );
} else if (data->hdr) {
fc = data->hdr->fitshdr;
}
/* Get the input NDF identifier. */
if( data && data->file &&
data->file->ndfid != NDF__NOID) indf = data->file->ndfid;
/* Get a structure holding provenance information from indf. */
prov = ndgReadProv( indf, " ", status );
/* Initially, assume that we should include details of ancestor NDFs. */
isroot = 0;
/* Get the OBSIDSS keyword value from the input FITS header if we have a FITS
header. There can be cases (eg in STACKFRAMES) where we do not have a
FITS header so handle that gracefully. */
if (fc) obsidss = smf_getobsidss( fc, NULL, 0, obsidssbuf,
sizeof(obsidssbuf), status );
if( obsidss ) {
/* Search through all the ancestors of the input NDF (including the input
NDF itself) to see if any of them refer to the same OBSIDSS value. */
found = 0;
ianc = 0;
anc = ndgGetProv( prov, ianc, status );
while( anc ) {
if( astMapGet0A( anc, "MORE", &tkm ) ) {
if( astMapGet0C( tkm, "OBSIDSS", &vptr ) ) {
found = !strcmp( obsidss, vptr );
}
tkm = astAnnul( tkm );
}
anc = astAnnul( anc );
if( ! found ) anc = ndgGetProv( prov, ++ianc, status );
}
/* If the OBSIDSS value was not found in any ancestor, then we add it
now. So put the OBSIDSS keyword value in an AST KeyMap that will be
stored with the output provenance information. */
if( ! found ) {
tkm = astKeyMap( " " );
astMapPut0C( tkm, "OBSIDSS", obsidss, NULL );
/* Ignore ancestor NDFs if none of them referred to the correct OBSIDSS. */
isroot = 1;
}
}
/* Update the provenance for the output NDF to include the input NDF as
an ancestor. Indicate that each input NDF is a root NDF (i.e. has no
parents). Do nothing if we have no provenance to propagate, unless
we have root information and are propagating that. */
if ( ndgCountProv( prov, status ) > 0 || tkm) {
NdgProvenance *oprov = NULL;
if (modprov && *modprov) {
oprov = *modprov;
} else {
oprov = ndgReadProv( ondf, creator, status );
if (modprov) *modprov = oprov;
}
ndgPutProv( oprov, indf, tkm, isroot, status );
/* do not update the file or free provenance if we
are using an external provenance struct or returning
it to the caller */
if (!modprov) {
if( ondf != NDF__NOID ) {
ndgWriteProv( oprov, ondf, 1, status );
oprov = ndgFreeProv( oprov, status );
} else if( *status == SAI__OK ){
*status = SAI__ERROR;
errRep( " ", "smf_updateprov: Provenance is to be stored in the "
"output NDF but no output NDF identifier was supplied "
"(programming error).", status );
}
}
}
/* Free resources. */
if( prov ) prov = ndgFreeProv( prov, status );
if( tkm ) tkm = astAnnul( tkm );
if( !data && fc ) fc = astAnnul( fc );
}
double cupidConfigD( AstKeyMap *config, const char *name, double def,
int *status ){
/*
*+
* Name:
* cupidConfigD
* Purpose:
* Get the value of a configuration parameter.
* Language:
* Starlink C
* Synopsis:
* double cupidConfigD( AstKeyMap *config, const char *name, double def,
* int *status )
* Description:
* This function returns a named value from the supplied KeyMap. If
* the KeyMap does not contain the named value, an attempt is made to
* obtain a value from a secondary KeyMap which should be stored
* within the supplied KeyMap, using a key equal to the constant
* string CUPID__CONFIG. If the secondary KeyMap does not contain a
* value, then the supplied default value is returned. In either case,
* the returned value is stored in the KeyMap.
* Parameters:
* config
* An AST KeyMap holding the configuration parameters. If NULL is
* supplied, the default value is returned without error.
* name
* The name of the value to extract from the KeyMap.
* def
* The default value to use.
* status
* Pointer to the inherited status value.
* Returned Value:
* The required value. The supplied default value is returned if an
* error occurs.
* Copyright:
* Copyright (C) 2005 Particle Physics & Astronomy Research Council.
* Copyright (C) 2008 Science & Technology Facilities Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* {enter_new_authors_here}
* History:
* 5-OCT-2005 (DSB):
* Original version.
* 26-MAR-2008 (DSB):
* Re-report a more friendly message iof the supplied text could
* not be interpreted as a numerical value.
* {enter_further_changes_here}
* Bugs:
* {note_any_bugs_here}
*-
*/
/* Local Variables: */
AstObject *sconfig; /* Object pointer obtained from KeyMap */
const char *text; /* Pointer to uninterprted text value */
double ret; /* The returned value */
/* Initialise */
ret = def;
/* Abort if an error has already occurred, or if no KeyMap was supplied. */
if( *status != SAI__OK || !config ) return ret;
/* Attempt to extract the named value from the supplied KeyMap. */
if( !astMapGet0D( config, name, &ret ) ) {
/* If the value was not found in the KeyMap, see if the KeyMap contains a
secondary KeyMap. */
if( astMapGet0A( config, CUPID__CONFIG, &sconfig ) ) {
/* If it does, see if the secondary KayMap contains a value for the named
entry. If it does, remove the value from the KeyMap so it does not
appear in the CUPID NDF extension. */
if( astMapGet0D( (AstKeyMap *) sconfig, name, &ret ) ) {
astMapRemove( (AstKeyMap *) sconfig, name );
/* If the text supplied by the user could not be interpreted as a
floating point value, re-report the error. */
} else if( *status == AST__MPGER ) {
ret = def;
errAnnul( status );
if( astMapGet0C( config, name, &text ) ) {
*status = SAI__ERROR;
msgSetc( "T", text );
msgSetc( "N", name );
errRep( "", "Illegal value \"^T\" supplied for configuration "
"parameter ^N.", status );
}
/* If the value was not found in either KeyMap, return the default value. */
} else {
ret = def;
}
/* Free the pointer to the secondary KeyMap. */
sconfig = astAnnul( sconfig );
/* If no secondary KeyMap was found, return the default value. */
} else {
ret = def;
}
/* Store the returned value in the supplied KeyMap if it is good. */
if( ret != VAL__BADD ) astMapPut0D( config, name, ret, NULL );
/* If the text supplied by the user could not be interpreted as a
floating point value, re-report the error. */
} else if( *status == AST__MPGER ) {
ret = def;
errAnnul( status );
if( astMapGet0C( config, name, &text ) ) {
*status = SAI__ERROR;
msgSetc( "T", text );
msgSetc( "N", name );
errRep( "", "Illegal value \"^T\" supplied for configuration "
"parameter ^N.", status );
}
}
/* Return the result. */
return ret;
}
void smf_checkmem_dimm( dim_t maxlen, inst_t instrument, int nrelated,
smf_modeltype *modeltyps, dim_t nmodels, dim_t msize,
AstKeyMap *keymap, size_t available, dim_t maxfilelen,
size_t *necessary, int *status ) {
/* Local Variables */
int dofft=0; /* flag if we need temp space for FFTs */
dim_t i; /* Loop counter */
dim_t gain_box; /* Length of blocks for GAI/COM model */
AstKeyMap *kmap=NULL; /* Local keymap */
dim_t nblock; /* Number of blocks for GAI/COM model */
size_t ncol; /* Number of columns */
size_t ndet; /* Number of detectors each time step */
size_t ndks; /* dksquid samples in a subarray, ncol*maxlen */
size_t nrow; /* Number of rows */
size_t nsamp; /* bolo samples in a subarray, ndet*maxlen */
const char *tempstr=NULL; /* Temporary pointer to static char buffer */
size_t total = 0; /* Total bytes required */
/* Main routine */
if (*status != SAI__OK) return;
/* Check inputs */
if( maxlen < 1 ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": maxlen cannot be < 1", status);
return;
}
if( (nrelated < 1) || (nrelated > SMF__MXSMF) ) {
msgSeti("NREL",nrelated);
msgSeti("MAXREL",SMF__MXSMF);
*status = SAI__ERROR;
errRep("", FUNC_NAME ": nrelated, ^NREL, must be in the range [1,^MAXREL]",
status);
return;
}
if( modeltyps ) {
if( nmodels < 1 ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": modeltyps specified, mmodels cannot be < 1",
status);
return;
}
}
if( *status == SAI__OK ) {
/* Work out the data dimensions */
switch( instrument ) {
case INST__SCUBA2:
/* Kludgey, but at least we check SC2STORE__COL_INDEX so this will help
us catch possible future problems if order is changed */
if( SC2STORE__COL_INDEX ) {
ncol = 32;
nrow = 40;
} else {
ncol = 40;
nrow = 32;
}
ndet = ncol*nrow;
break;
default:
*status = SAI__ERROR;
errRep("", FUNC_NAME ": Invalid instrument given.", status);
}
}
/* Number of samples in a full data cube for one subarray */
nsamp = ndet*maxlen;
/* Check to see if we need to do filtering as part of
pre-processing. If so, we need an extra nsamp-sized buffer to
store the FFT. */
smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
&dofft, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
status );
/* Calculate memory usage of static model components: -------------------- */
if( *status == SAI__OK ) {
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated; /* RES */
total += nsamp*smf_dtype_sz(SMF__INTEGER,status)*nrelated; /* LUT */
total += nsamp*smf_dtype_sz(SMF__QUALTYPE,status)*nrelated; /* QUA */
}
/* Add on memory usage for the JCMTState (one per time slice per array) */
if( *status == SAI__OK ) {
total += maxlen*sizeof(JCMTState)*nrelated;
}
/* Add on space for dark squids */
ndks = ncol*maxlen;
total += ndks*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
/* Add on space for fakemap */
astMapGet0C( keymap, "FAKEMAP", &tempstr );
if( tempstr ) {
total += msize*sizeof(double);
}
/* Apply fudge factor */
total *= CHECKMEM_FUDGE;
/* Calculate memory usage of dynamic model components: ------------------- */
if( *status == SAI__OK ) {
/* Most of these will have data arrays associated with each
subarray (hence the multiplication by nrelated). An exception
is SMF__COM for which a single-common mode is used across all
subarrays. */
if( modeltyps ) {
for( i=0; i<nmodels; i++ ) {
switch( modeltyps[i] ) {
case SMF__NOI:
/* SMF__NOI also estimates the noise in each detector from the
power spectra, requiring a temporary buffer to store an FFT.
Currently we just store one variance per detector */
dofft = 1;
total += ndet*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__COM:
CHECK_MASK("COM")
total += maxlen*smf_dtype_sz(SMF__DOUBLE,status);
break;
case SMF__EXT:
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__DKS:
total += (maxlen + nrow*3)*ncol*smf_dtype_sz(SMF__DOUBLE,status) *
nrelated;
break;
case SMF__GAI:
/* Every COM.GAIN_BOX samples there are 3 planes of data
corresponding to each bolometer in the subarray. The
conversion of COM.GAIN_BOX from seconds to samples within
smf_get_nsamp assumes a sample rate of 200 Hz. Later
downsampling may result in a lower sample rate, but at least
we are erring on the conservative side by assuming 200 Hz. */
if( astMapGet0A( keymap, "COM", &kmap ) ) {
smf_get_nsamp( kmap, "GAIN_BOX", NULL, &gain_box, status );
nblock = maxlen/gain_box;
if( nblock == 0 ) nblock = 1;
total += nblock*3*nrow*ncol*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
kmap = astAnnul( kmap );
}
break;
case SMF__FLT:
/* Presently the filter temporarily transforms the entire
data cube into a second array. We therefore need to
ensure enough memory to temporarily store the data cube
twice. */
CHECK_MASK("FLT")
dofft = 1;
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__PLN:
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__SMO:
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__TMP:
/* An externally supplied template. The model just stores the gain,
offset and correlation coefficient for each bolometer, similar
to SMF__GAI except with only one chunk considered */
total += 3*nrow*ncol*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__TWO:
/* two common-mode time series and coefficients for all the
detectors */
total += 2*(ndet+maxlen)*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__AST:
/* Mostly accounted for as static memory usage above, but add space
for mask if required */
CHECK_MASK("AST")
break;
default:
*status = SAI__ERROR;
errRep("", FUNC_NAME ": Invalid smf_modeltype given.", status);
}
/* Exit on bad status */
if( *status != SAI__OK ) {
i = nmodels;
}
}
}
/* Calculate temporary space here -------------------------------------- */
if( *status == SAI__OK ) {
size_t temp=0; /* current temp memory required */
size_t maxtemp=0; /* max temp memory required */
/* Some temp space required when we initially read in the
data. Normally smf_concat_smfGroup will
smf_open_and_flatfield one file at a time before copying into
the concatenated data array. If there are N time slices in a
file, smf_open_and_flatfield will usually require space both
for the raw, and double-precision flatfielded data
simultaneously. This is an upper limit, since if previously
flatfielded data are provided no extra space for the raw
data will be required. We use the supplied maxfilelen to figure
out this maximum temporary buffer size. Remember to account
for bolo data, dark squids, and JCMTState. Also, the way
smf_iteratemap is currently written, a concatenated memory-mapped
pointing LUT is initially created, and then copied to a new
malloc'd array, so we need extra temp space for the full LUT. */
temp = (ndet + ncol)*( smf_dtype_sz(SMF__INTEGER,status)+
smf_dtype_sz(SMF__DOUBLE,status) )*maxfilelen +
2*maxfilelen*sizeof(JCMTState);
temp += nsamp*smf_dtype_sz(SMF__INTEGER,status)*nrelated;
if( temp > maxtemp ) maxtemp = temp;
/* If we are doing FFTs */
if( dofft ) {
temp = nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
if( temp > maxtemp ) maxtemp = temp;
}
/* Data cleaning... seems to be a lot. Just putting in a rough
estimate of 3 extra full sized arrays for the standard
options. */
temp = 3.*nsamp*smf_dtype_sz(SMF__DOUBLE,status);
if( temp > maxtemp ) maxtemp = temp;
/* Add on the maximum chunk of temporary memory that is required */
total += maxtemp;
}
/* Set bad status if too big */
if( (*status == SAI__OK) && (total > available) ) {
*status = SMF__NOMEM;
msgSeti("REQ",total/SMF__MIB);
msgSeti("AVAIL",available/SMF__MIB);
errRep("", FUNC_NAME
": Requested memory ^REQ MiB for map exceeds available ^AVAIL MiB",
status);
}
/* Return the required space */
if( necessary ) {
*necessary = total;
}
}
}
dim_t smf_get_padding( AstKeyMap *keymap, int report, const smfHead *hdr,
double steptime, int *status ) {
/* Local Variables: */
AstObject *obj;
const char *key;
dim_t pad;
dim_t result;
double f_low;
double filt_edgehigh;
double filt_edgelarge;
double filt_edgelow;
double filt_edgesmall;
double filt_notchlow[ SMF__MXNOTCH ];
int f_nnotch;
int iel;
int nel;
int temp;
double scalelen;
double downsampscale;
double downsampfreq;
/* Initialise */
result = 0;
/* Main routine */
if (*status != SAI__OK) return result;
/* If the keymap contains a PAD value, return it. */
if( astMapGet0I( keymap, "PAD", &temp ) ) {
if( temp < 0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "PAD cannot be < 0.", status );
} else {
result = (dim_t) temp;
}
/* If the keymap contains no PAD value, calculate a default on the basis
of the FILT_ values. */
} else {
/* If the keymap does not contain a PAD value, not even an "<undef>"
value, an error will have been reported by astMapget0I above. In this
case, annull the error and continue since the KeyMap may contain FILT_
values that can be used to calculate a default PAD value. */
if( *status == AST__MPKER ) errAnnul( status );
/* Search for filtering parameters in the keymap. None of these parameters
represent a number of time clies, so we can set the smfData (the 2nd
argument) to NULL. */
f_nnotch = 0;
smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, &filt_edgelow, &filt_edgehigh, &filt_edgesmall,
&filt_edgelarge, filt_notchlow, NULL, &f_nnotch, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, &downsampscale, &downsampfreq, NULL,
status );
if( downsampscale && downsampfreq ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": both downsampscale and downsampfreq are set",
status );
}
/* If any were not found, annul the error and continue with those that
were found (plus defaults for the others). */
if( *status == AST__MPKER ) errAnnul( status );
if( *status == SAI__OK ) {
/* Modify edge filters if spatial scales were requested */
smf_scale2freq( filt_edgesmall, filt_edgelarge, hdr, &filt_edgelow,
&filt_edgehigh, status );
if( *status == SMF__TELSTAT ) {
errAnnul( status );
}
/* Find the lowest of these frequencies. The lowest frequency will give
the greatest padding. */
f_low = ( filt_edgehigh > 0.0 ) ? filt_edgehigh : VAL__MAXD;
if( filt_edgelow > 0.0 && filt_edgelow < f_low ) f_low = filt_edgelow;
for( iel = 0; iel < f_nnotch; iel++ ) {
if( filt_notchlow[ iel ] > 0.0 && filt_notchlow[ iel ] < f_low ) f_low = filt_notchlow[ iel ];
}
/* If any down-sampling is occurring, set the step time to value implied by
the requested downsampling factor. This should match the calculation in
smf_grp_related */
if( downsampscale || downsampfreq ) {
if( downsampscale ) {
scalelen = hdr->steptime * hdr->scanvel / downsampscale;
} else {
if( hdr->steptime ) {
scalelen = downsampfreq / (1./hdr->steptime);
} else {
*status = SAI__ERROR;
scalelen = VAL__BADD;
errRep( "", FUNC_NAME ": can't resample because smfData has "
"unknown sample rate", status );
}
}
if( (*status==SAI__OK) && (scalelen<=SMF__DOWNSAMPLIMIT) )
steptime = hdr->steptime/scalelen;
/* If no downsampling is happening, use the steptime in the header unless
another steptime was supplied. */
} else if( steptime == VAL__BADD ) {
steptime = hdr->steptime;
}
/* Find the corresponding padding. */
if( f_low != VAL__MAXD ) {
result = 1.0/( steptime * f_low );
} else {
result = 0;
}
/* Now check the supplied keymap for any nested keymaps. Assumes that each
entry in the supplied KeyMap contain either a primitive value or a scalar
KeyMap pointer. */
nel = astMapSize( keymap );
for( iel = 0; iel < nel; iel++ ) {
key = astMapKey( keymap, iel );
/* If this entry is a KeyMap (assuming no other class of AST object is
stored in the KeyMap)... */
if( astMapType( keymap, key ) == AST__OBJECTTYPE ) {
/* Get a pointer to the KeyMap. */
(void) astMapGet0A( keymap, key, &obj );
/* Call this function to get the padding implied by the sub-KeyMap. */
pad = smf_get_padding( (AstKeyMap *) obj, report, hdr,
steptime, status );
/* Use the larger of the two paddings. */
if( pad > result ) result = pad;
}
}
/* If required, report the default value. */
if( report ) {
msgSeti( "P", (int) result );
msgOutif( MSG__VERB, "", "Padding each time stream with ^P zero "
"values at start and end.", status );
}
}
}
/* Return the result. */
return result;
}
void smf_grp_related( const Grp *igrp, const size_t grpsize,
const int grouping, const int checksubinst,
double maxlen_s, double *srate_maxlen,
AstKeyMap *keymap, dim_t *maxconcatlen,
dim_t *maxfilelen, smfGroup **group,
Grp **basegrp, dim_t *pad, int *status ) {
/* Local variables */
size_t *chunk=NULL; /* Array of flags for continuous chunks */
dim_t * chunklen = NULL; /* Length of continuous chunk */
size_t currentindex = 0; /* Counter */
char cwave[10]; /* String containing wavelength */
smfData *data = NULL; /* Current smfData */
double downsampscale=0; /* Angular scale downsampling size */
double downsampfreq=0; /* Target downsampling frequency */
AstKeyMap * grouped = NULL; /* Primary AstKeyMap for grouping */
size_t i; /* Loop counter for index into Grp */
int isFFT=0; /* Set if data are 4d FFT */
size_t j; /* Loop counter */
int *keepchunk=NULL; /* Flag for chunks that will be kept */
dim_t maxconcat=0; /* Longest continuous chunk length */
dim_t maxflen=0; /* Max file length in time steps */
dim_t maxlen=0; /* Maximum concat length in samples */
int maxlen_scaled=0; /* Set once maxlen has been scaled, if needed */
dim_t maxpad=0; /* Maximum padding neeed for any input file */
size_t maxrelated = 0; /* Keep track of max number of related items */
size_t *new_chunk=NULL; /* keeper chunks associated with subgroups */
dim_t *new_tlen=NULL; /* tlens for new_subgroup */
size_t ngroups = 0; /* Counter for subgroups to be stored */
size_t nkeep = 0; /* Number of chunks to keep */
dim_t * piecelen = NULL; /* Length of single file */
smf_subinst_t refsubinst; /* Subinst of first file */
size_t **subgroups = NULL; /* Array containing index arrays to parent Grp */
smf_subinst_t subinst; /* Subinst of current file */
if ( *status != SAI__OK ) return;
if( maxlen_s < 0 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": maxlen_s cannot be < 0!", status );
return;
}
/* Get downsampling parameters */
if( keymap ) {
smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, &downsampscale, &downsampfreq,
NULL, NULL, NULL, NULL, status );
if( downsampscale && downsampfreq ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": both downsampscale and downsampfreq are set",
status );
return;
}
}
/* Initialize refcwave */
refsubinst = SMF__SUBINST_NONE;
/* Loop over files in input Grp: remember Grps are indexed from 1 */
grouped = astKeyMap( "SortBy=KeyUp" );
for (i=1; i<=grpsize; i++) {
char newkey[128];
char dateobs[81];
char subarray[10];
size_t nrelated = 0;
AstKeyMap * filemap = NULL;
AstKeyMap * indexmap = NULL;
/* First step: open file and harvest metadata */
smf_open_file( NULL, igrp, i, "READ", SMF__NOCREATE_DATA, &data, status );
if (*status != SAI__OK) break;
if( i==1 ) {
isFFT = smf_isfft( data, NULL, NULL, NULL, NULL, NULL, status );
} else if( smf_isfft(data, NULL, NULL, NULL, NULL, NULL, status) != isFFT ){
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": mixture of time-series and FFT data encountered!",
status );
break;
}
/* If maxlen has not been set, do it here */
if( !maxlen && maxlen_s && data->hdr->steptime) {
maxlen = (dim_t) (maxlen_s / data->hdr->steptime );
}
/* Return srate_maxlen if requested: may want to know this number
even if maxlen_s is not set. Only calculate once, although it
gets overwritten once later if down-sampling. */
if( (i==1) && srate_maxlen && data->hdr->steptime ) {
*srate_maxlen = 1. / (double) data->hdr->steptime;
}
/* If requested check to see if we are mixing wavelengths */
if( checksubinst ) {
if( refsubinst == SMF__SUBINST_NONE ) {
refsubinst = smf_calc_subinst( data->hdr, status );
}
subinst = smf_calc_subinst( data->hdr, status );
if( subinst != refsubinst ) {
const char *refsubstr = smf_subinst_str( refsubinst, status );
const char *substr = smf_subinst_str( subinst, status );
*status = SAI__ERROR;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
msgSetc( "REFSUB", refsubstr );
msgSetc( "SUB", substr );
errRep( "", FUNC_NAME
": ^FILE uses sub-instrument ^SUB which doesn't match "
"reference ^REFSUB", status );
}
}
/* Want to form a key that will be unique for a particular subscan
We know that DATE-OBS will be set for SCUBA-2 files and be the same
for a single set. Prefix by wavelength if we are grouping by wavelength.
*/
newkey[0] = '\0';
smf_find_subarray( data->hdr, subarray, sizeof(subarray), NULL, status );
if( grouping == 1 ) {
/* Group different wavelengths separately */
smf_fits_getS( data->hdr, "WAVELEN", cwave, sizeof(cwave), status);
one_strlcat( newkey, cwave, sizeof(newkey), status );
one_strlcat( newkey, "_", sizeof(newkey), status );
}
if( grouping == 2 ) {
/* Group different subarrays separately */
one_strlcat( newkey, subarray, sizeof(newkey), status );
}
smf_fits_getS( data->hdr, "DATE-OBS", dateobs, sizeof(dateobs), status );
one_strlcat( newkey, dateobs, sizeof(newkey), status );
/* Include the dimentionality of the time series in the primary key
so that we do not end up doing something confusing like relating
a truncated file with a full length file */
if (*status == SAI__OK) {
dim_t dims[3];
char formatted[32];
smf_get_dims( data, &dims[0], &dims[1], NULL, &dims[2], NULL, NULL, NULL,
status );
sprintf(formatted, "_%" DIM_T_FMT "_%" DIM_T_FMT "_%" DIM_T_FMT, dims[0], dims[1], dims[2]);
one_strlcat( newkey, formatted, sizeof(newkey), status );
}
/* May want to read the dimensionality of the file outside of loop
so that we can compare values when storing in the keymap */
/* Now we want to create a keymap based on this key */
if (!astMapGet0A( grouped, newkey, &filemap ) ) {
int itemp = 0;
double steptime = data->hdr->steptime;
dim_t ntslice = 0;
dim_t thispad; /* Padding neeed for current input file */
filemap = astKeyMap( " " );
astMapPut0A( grouped, newkey, filemap, NULL );
/* Fill up filemap with general information on this file */
smf_find_seqcount( data->hdr, &itemp, status );
astMapPut0I( filemap, "SEQCOUNT", itemp, NULL );
smf_fits_getI( data->hdr, "NSUBSCAN", &itemp, status );
astMapPut0I( filemap, "NSUBSCAN", itemp, NULL );
/* Number of time slices */
smf_get_dims( data, NULL, NULL, NULL, &ntslice, NULL, NULL, NULL,
status );
/* Find length of down-sampled data, new steptime and maxlen */
if( (downsampscale || downsampfreq) && data->hdr && (*status==SAI__OK) ) {
double scalelen;
if( downsampscale ) {
if( data->hdr->scanvel != VAL__BADD ) {
double oldscale = steptime * data->hdr->scanvel;
scalelen = oldscale / downsampscale;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
scalelen = VAL__BADD;
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep( "", FUNC_NAME ": can't resample ^FILE because it has "
"unknown scan velocity", status );
}
} else {
if( steptime ) {
double oldsampfreq = 1./steptime;
scalelen = downsampfreq / oldsampfreq;
} else {
*status = SAI__ERROR;
scalelen = VAL__BADD;
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep( "", FUNC_NAME ": can't resample ^FILE because it has "
"unknown sample rate", status );
}
}
/* only down-sample if it will be a reasonable factor */
if( (*status==SAI__OK) && (scalelen <= SMF__DOWNSAMPLIMIT) ) {
smf_smfFile_msg(data->file, "FILE", 1, "" );
msgOutiff( MSG__VERB, "", FUNC_NAME
": will down-sample file ^FILE from %5.1lf Hz to "
"%5.1lf Hz", status, (1./steptime), (scalelen/steptime) );
ntslice = round(ntslice * scalelen);
/* If maxlen has been requested, and we have not already worked
out a scaled version (just uses the sample rates for the first
file... should be close enough -- the alternative is a 2-pass
system). */
if( !maxlen_scaled ) {
maxlen = round(maxlen*scalelen);
maxlen_scaled = 1;
msgOutiff( MSG__VERB, "", FUNC_NAME
": requested maxlen %g seconds = %" DIM_T_FMT " down-sampled "
"time-slices", status, maxlen_s, maxlen );
/* Return updated srate_maxlen for down-sampling if requested */
if( srate_maxlen ) {
*srate_maxlen = scalelen/steptime;
}
}
}
}
/* Check that an individual file is too long (we assume related
files are all the same) */
if( maxlen && (ntslice > maxlen) && *status == SAI__OK) {
*status = SAI__ERROR;
msgSeti("NTSLICE",ntslice);
msgSeti("MAXLEN",maxlen);
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep(FUNC_NAME,
"Number of time steps in file ^FILE time exceeds maximum "
"(^NTSLICE>^MAXLEN)", status);
}
/* Scaled values of ntslice and maximum length */
astMapPut0I( filemap, "NTSLICE", ntslice, NULL );
/* Work out the padding needed for this file including downsampling. */
if( keymap ) {
thispad = smf_get_padding( keymap, 0, data->hdr, VAL__BADD, status );
if( thispad > maxpad ) maxpad = thispad;
} else {
thispad = 0;
}
astMapPut0I( filemap, "PADDING", thispad, NULL );
/* Update maxflen */
if( ntslice > maxflen ) {
maxflen = ntslice;
}
/* Store OBSID or OBSIDSS depending on whether we are grouping by wavelength */
if (grouping) {
astMapPut0C( filemap, "OBSID", data->hdr->obsidss, NULL );
} else {
char obsid[81];
smf_getobsidss( data->hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status );
astMapPut0C( filemap, "OBSID", obsid, NULL );
}
}
/* Store the file index in another keymap indexed by subarray */
if ( !astMapGet0A( filemap, "GRPINDICES", &indexmap ) ) {
indexmap = astKeyMap( "SortBy=KeyUp" );
astMapPut0A( filemap, "GRPINDICES", indexmap, NULL );
}
astMapPut0I( indexmap, subarray, i, NULL );
/* Need to track the largest number of related subarrays in a single slot */
nrelated = astMapSize( indexmap );
if (nrelated > maxrelated) maxrelated = nrelated;
/* Free resources */
filemap = astAnnul( filemap );
indexmap = astAnnul( indexmap );
smf_close_file( NULL, &data, status );
}
/* We now know how many groups there are */
ngroups = astMapSize( grouped );
/* Sort out chunking. The items are sorted by date and then by wavelength.
We define a continuous chunk if it has the same OBSID, the same SEQCOUNT
and NSUBSCAN increments by one from the previous entry.
Also count number of related items in each slot.
*/
if (*status == SAI__OK) {
typedef struct { /* somewhere to store the values easily */
char obsid[81];
char related[81]; /* for concatenated subarrays */
int nsubscan;
int seqcount;
} smfCompareSeq;
smfCompareSeq current;
smfCompareSeq previous;
dim_t totlen = 0;
size_t thischunk;
/* Get the chunk flags and also store the size of the chunk */
chunk = astCalloc( ngroups, sizeof(*chunk) );
chunklen = astCalloc( ngroups, sizeof(*chunklen) );
piecelen = astCalloc( ngroups, sizeof(*piecelen) );
thischunk = 0; /* The current chunk */
for (i=0; i<ngroups; i++) {
AstKeyMap * thismap = NULL;
AstKeyMap * grpindices = NULL;
const char * tempstr = NULL;
int thistlen = 0;
size_t nsubarrays = 0;
/* Get the keymap entry for this slot */
astMapGet0A( grouped, astMapKey(grouped, i), &thismap );
/* Get info for length limits */
astMapGet0I( thismap, "NTSLICE", &thistlen );
piecelen[i] = thistlen;
if (isFFT) {
/* Never concatenate FFT data */
thismap = astAnnul(thismap);
chunk[i] = i;
chunklen[i] = thistlen;
continue;
}
/* Get indices information and retrieve the sub-instrument names
in sort order to concatenate for comparison. We only store in
a continuous chunk if we have the same subarrays for the whole
chunk. */
astMapGet0A( thismap, "GRPINDICES", &grpindices );
nsubarrays = astMapSize( grpindices );
(current.related)[0] = '\0';
for (j = 0; j < nsubarrays; j++ ) {
one_strlcat( current.related, astMapKey(grpindices, j), sizeof(current.related), status );
}
grpindices = astAnnul( grpindices );
/* Fill in the current struct */
astMapGet0I( thismap, "SEQCOUNT", &(current.seqcount) );
astMapGet0I( thismap, "NSUBSCAN", &(current.nsubscan) );
astMapGet0C( thismap, "OBSID", &tempstr );
one_strlcpy( current.obsid, tempstr, sizeof(current.obsid), status );
/* First chunk is special, else compare */
if (i == 0) {
totlen = thistlen;
} else {
if ( ( current.seqcount == previous.seqcount ) &&
( current.nsubscan - previous.nsubscan == 1 ) &&
( strcmp( current.obsid, previous.obsid ) == 0 ) &&
( strcmp( current.related, previous.related ) == 0 ) ) {
/* continuous - check length */
totlen += thistlen;
if ( maxlen && totlen > maxlen ) {
thischunk++;
totlen = thistlen; /* reset length */
} else {
/* Continuous */
}
} else {
/* discontinuity */
thischunk++;
totlen = thistlen; /* Update length of current chunk */
}
}
chunklen[thischunk] = totlen;
chunk[i] = thischunk;
memcpy( &previous, ¤t, sizeof(current) );
thismap = astAnnul( thismap );
}
}
/* Decide if we are keeping a chunk by looking at the length. */
maxconcat = 0;
nkeep = 0;
keepchunk = astMalloc( ngroups*sizeof(*keepchunk) );
for (i=0; i<ngroups; i++) {
size_t thischunk;
thischunk = chunk[i];
if ( chunklen[thischunk] < SMF__MINCHUNKSAMP ) {
/* Warning message */
msgSeti("LEN",chunklen[thischunk]);
msgSeti("MIN",SMF__MINCHUNKSAMP);
msgOut( " ", "SMF_GRP_RELATED: ignoring short chunk (^LEN<^MIN)",
status);
keepchunk[i] = 0;
} else {
keepchunk[i] = 1;
if (maxconcat < chunklen[thischunk]) maxconcat = chunklen[thischunk];
nkeep++;
}
}
/* If no useful chunks generate an error */
if( (*status==SAI__OK) && (!nkeep) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No useful chunks.", status );
goto CLEANUP;
}
/* Allocate a subgroup array of the right size and fill it. They keymap
is sorted by date (and wavelength) so we can always index into it by using
indices from the subgroup. */
subgroups = astCalloc( nkeep, sizeof(*subgroups) );
new_chunk = astCalloc( nkeep, sizeof(*new_chunk) );
new_tlen = astCalloc( nkeep, sizeof(*new_tlen) );
currentindex = 0;
for (i=0;i<ngroups;i++) {
AstKeyMap * thismap = NULL;
AstKeyMap * grpindices = NULL;
size_t nsubarrays = 0;
size_t *indices = astCalloc( maxrelated, sizeof(*indices) );
/* skip if we are dropping this chunk */
if (!keepchunk[i]) continue;
/* Get the keymap entry for this slot */
astMapGet0A( grouped, astMapKey(grouped, i), &thismap );
/* Get the indices keymap */
astMapGet0A( thismap, "GRPINDICES", &grpindices );
nsubarrays = astMapSize( grpindices );
for (j=0; j<nsubarrays; j++) {
int myindex;
astMapGet0I( grpindices, astMapKey(grpindices, j), &myindex );
indices[j] = myindex;
}
grpindices = astAnnul( grpindices );
thismap = astAnnul( thismap );
subgroups[currentindex] = indices;
new_chunk[currentindex] = chunk[i];
new_tlen[currentindex] = piecelen[i];
currentindex++;
}
/* Create the smfGroup */
*group = smf_construct_smfGroup( igrp, subgroups, new_chunk, new_tlen,
nkeep, maxrelated, 0, status );
/* Return maxfilelen if requested */
if( maxfilelen ) {
*maxfilelen = maxflen;
}
/* Return maxconcatlen if requested */
if( maxconcatlen ) {
*maxconcatlen = maxconcat;
}
/* Create a base group for output files if required */
/* Create a base group of filenames */
if (*status == SAI__OK && basegrp ) {
*basegrp = smf_grp_new( (*group)->grp, "Base Group", status );
/* Loop over time chunks */
for( i=0; (*status==SAI__OK)&&(i<(*group)->ngroups); i++ ) {
size_t idx;
/* Check for new continuous chunk */
if( i==0 || ( (*group)->chunk[i] != (*group)->chunk[i-1]) ) {
/* Loop over subarray */
for( idx=0; idx<(*group)->nrelated; idx++ ) {
size_t grpindex = (*group)->subgroups[i][idx];
if ( grpindex > 0 ) {
ndgCpsup( (*group)->grp, grpindex, *basegrp, status );
}
}
}
}
}
CLEANUP:
keepchunk = astFree( keepchunk );
chunk = astFree( chunk );
chunklen = astFree( chunklen );
piecelen = astFree( piecelen );
grouped = astAnnul( grouped );
if( *status != SAI__OK ) {
/* free the group */
if (basegrp && *basegrp) grpDelet( basegrp, status );
if (group && *group) {
smf_close_smfGroup( group, status );
} else {
/* have to clean up manually */
new_chunk = astFree( new_chunk );
new_tlen = astFree( new_tlen );
if( subgroups ) {
size_t isub;
for( isub=0; isub<nkeep; isub++ ) {
subgroups[isub] = astFree( subgroups[isub] );
}
subgroups = astFree( subgroups );
}
}
}
/* Return the maximum padding if required. */
if( pad ) *pad = maxpad;
}
void smurf_sc2clean( int *status ) {
smfArray *array = NULL; /* Data to be cleaned */
Grp *basegrp=NULL; /* Grp containing first file each chunk */
size_t basesize; /* Number of files in base group */
smfArray *bbms = NULL; /* Bad bolometer masks */
smfArray *concat=NULL; /* Pointer to a smfArray */
size_t contchunk; /* Continuous chunk counter */
smfArray *darks = NULL; /* Dark data */
int ensureflat; /* Flag for flatfielding data */
smfArray *flatramps = NULL;/* Flatfield ramps */
AstKeyMap *heateffmap = NULL; /* Heater efficiency data */
smfData *odata = NULL; /* Pointer to output data struct */
Grp *fgrp = NULL; /* Filtered group, no darks */
size_t gcount=0; /* Grp index counter */
size_t idx; /* Subarray counter */
Grp *igrp = NULL; /* Input group of files */
smfGroup *igroup=NULL; /* smfGroup corresponding to igrp */
dim_t maxconcat=0; /* Longest continuous chunk length in samples */
double maxlen=0; /* Constrain maxconcat to this many seconds */
size_t ncontchunks=0; /* Number continuous chunks outside iter loop */
Grp *ogrp = NULL; /* Output group of files */
size_t osize; /* Total number of NDF names in the output group */
dim_t padStart=0; /* How many samples padding at start */
dim_t padEnd=0; /* How many samples padding at end */
size_t size; /* Number of files in input group */
int temp; /* Temporary signed integer */
int usedarks; /* flag for using darks */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
int writecom; /* Write COMmon mode to NDF if calculated? */
int writegai; /* Write GAIns to NDF if calculated? */
/* Main routine */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Read the input file */
kpg1Rgndf( "IN", 0, 1, "", &igrp, &size, status );
/* Filter out darks */
smf_find_science( wf, igrp, &fgrp, 1, NULL, NULL, 1, 1, SMF__NULL, &darks,
&flatramps, &heateffmap, NULL, status );
/* input group is now the filtered group so we can use that and
free the old input group */
size = grpGrpsz( fgrp, status );
grpDelet( &igrp, status);
igrp = fgrp;
fgrp = NULL;
if (size == 0) {
msgOutif(MSG__NORM, " ","All supplied input frames were filtered,"
" nothing to do", status );
goto CLEANUP;
}
/* --- Parse ADAM parameters ---------------------------------------------- */
/* Maximum length of a continuous chunk */
parGdr0d( "MAXLEN", 0, 0, VAL__MAXD, 1, &maxlen, status );
/* Padding */
parGdr0i( "PADSTART", 0, 0, VAL__MAXI, 1, &temp, status );
padStart = (dim_t) temp;
parGdr0i( "PADEND", 0, 0, VAL__MAXI, 1, &temp, status );
padEnd = (dim_t) temp;
/* Are we using darks? */
parGet0l( "USEDARKS", &usedarks, status );
/* Are we flatfielding? */
parGet0l( "FLAT", &ensureflat, status );
/* Write COM/GAI to NDFs if calculated? */
parGet0l( "COM", &writecom, status );
parGet0l( "GAI", &writegai, status );
/* Get group of bolometer masks and read them into a smfArray */
smf_request_mask( wf, "BBM", &bbms, status );
/* Group the input files by subarray and continuity ----------------------- */
smf_grp_related( igrp, size, 1, 0, maxlen-padStart-padEnd, NULL, NULL,
&maxconcat, NULL, &igroup, &basegrp, NULL, status );
/* Obtain the number of continuous chunks and subarrays */
if( *status == SAI__OK ) {
ncontchunks = igroup->chunk[igroup->ngroups-1]+1;
}
basesize = grpGrpsz( basegrp, status );
/* Get output file(s) */
kpg1Wgndf( "OUT", basegrp, basesize, basesize,
"More output files required...",
&ogrp, &osize, status );
/* Loop over continuous chunks and clean -----------------------------------*/
gcount = 1;
for( contchunk=0;(*status==SAI__OK)&&contchunk<ncontchunks; contchunk++ ) {
AstKeyMap *keymap=NULL;
int dkclean;
AstKeyMap *sub_instruments=NULL;
/* Place cleaning parameters into a keymap and set defaults. Do
this inside the loop in case we are cleaning files with
differing sub-instruments. Note that we use the map-maker
defaults file here (which loads the sc2clean defaults) so that
we populate the locked keymap with all the parameters that
people may come across to allow them to load their map-maker
config directly into sc2clean.
*/
sub_instruments = smf_subinst_keymap( SMF__SUBINST_NONE,
NULL, igrp,
igroup->subgroups[contchunk][0],
status );
keymap = kpg1Config( "CONFIG", "$SMURF_DIR/smurf_makemap.def",
sub_instruments, 1, status );
if( sub_instruments ) sub_instruments = astAnnul( sub_instruments );
/* Now rerun smf_grp_related to figure out how long each downsampled
chunk of data will be. */
if( basegrp ) grpDelet( &basegrp, status );
if( igroup ) smf_close_smfGroup( &igroup, status );
smf_grp_related( igrp, size, 1, 0, maxlen-padStart-padEnd, NULL, keymap,
&maxconcat, NULL, &igroup, &basegrp, NULL, status );
/* Concatenate this continuous chunk */
smf_concat_smfGroup( wf, NULL, igroup, usedarks ? darks:NULL, bbms, flatramps,
heateffmap, contchunk, ensureflat, 1, NULL, 0, NULL,
NULL, NO_FTS, padStart, padEnd, 0, &concat, NULL, status );
if( *status == SAI__OK) {
/* clean the dark squids now since we might need to use them
to clean the bolometer data */
smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
&dkclean, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, status );
for( idx=0; dkclean&&(*status==SAI__OK)&&idx<concat->ndat; idx++ ) {
odata = concat->sdata[idx];
if( odata && odata->da && odata->da->dksquid ) {
smfData *dksquid = odata->da->dksquid;
AstKeyMap *kmap=NULL;
msgOut("", TASK_NAME ": cleaning dark squids", status);
/* fudge the header so that we can get at JCMTState */
dksquid->hdr = odata->hdr;
/* clean darks using cleandk.* parameters */
astMapGet0A( keymap, "CLEANDK", &kmap );
array = smf_create_smfArray( status );
smf_addto_smfArray( array, dksquid, status );
smf_clean_smfArray( wf, array, NULL, NULL, NULL, kmap, status );
if( array ) {
array->owndata = 0;
smf_close_related( wf, &array, status );
}
if( kmap ) kmap = astAnnul( kmap );
/* Unset hdr pointer so that we don't accidentally close it */
dksquid->hdr = NULL;
}
}
/* Then the main data arrays */
if( *status == SAI__OK ) {
smfArray *com = NULL;
smfArray *gai = NULL;
char filename[GRP__SZNAM+1];
msgOut("", TASK_NAME ": cleaning bolometer data", status );
smf_clean_smfArray( wf, concat, NULL, &com, &gai, keymap, status );
/* If ADAM parameters for COM or GAI were specified, and the
common-mode was calculated, export to files here */
if( writecom && com ) {
for( idx=0; (*status==SAI__OK)&&(idx<com->ndat); idx++ ) {
smf_model_createHdr( com->sdata[idx], SMF__COM, concat->sdata[idx],
status );
smf_stripsuffix( com->sdata[idx]->file->name,
SMF__DIMM_SUFFIX, filename, status );
smf_dataOrder( wf, com->sdata[idx], 1, status );
smf_write_smfData( wf, com->sdata[idx], NULL, filename, NULL, 0,
NDF__NOID, MSG__NORM, 0, NULL, NULL, status );
}
}
if( writegai && gai ) {
for( idx=0; (*status==SAI__OK)&&(idx<gai->ndat); idx++ ) {
smf_model_createHdr( gai->sdata[idx], SMF__GAI, concat->sdata[idx],
status );
smf_stripsuffix( gai->sdata[idx]->file->name,
SMF__DIMM_SUFFIX, filename, status );
smf_dataOrder( wf, gai->sdata[idx], 1, status );
smf_write_smfData( wf, gai->sdata[idx], NULL, filename, NULL, 0,
NDF__NOID, MSG__NORM, 0, NULL, NULL, status );
}
}
/* Close com and gai */
if( com ) smf_close_related( wf, &com, status );
if( gai ) smf_close_related( wf, &gai, status );
}
/* Report statistics (currently need a smfArray for that) */
if (*status == SAI__OK) {
size_t last_qcount[SMF__NQBITS];
size_t last_nmap = 0;
smf_qualstats_report( wf, MSG__VERB, SMF__QFAM_TSERIES, 1, concat,
last_qcount, &last_nmap, 1, NULL, NULL, status );
}
/* Clean up for contchunk loop */
if( keymap ) keymap = astAnnul( keymap );
}
/* Export concatenated/cleaned data for each subarray to NDF file */
for( idx=0; (*status==SAI__OK)&&idx<concat->ndat; idx++ ) {
odata = concat->sdata[idx];
/* Complete the history information in the output NDF so that it
includes group parameters accessed since the default history
information was written to the NDF (in smf_open_and_flatfield). */
smf_puthistory( odata, "SMURF:SC2CLEAN", status );
/* Ensure ICD data order */
smf_dataOrder( wf, odata, 1, status );
if( odata->file && odata->file->name ) {
smf_write_smfData( wf, odata, NULL, NULL, ogrp, gcount, NDF__NOID,
MSG__VERB, 0, NULL, NULL, status );
} else {
*status = SAI__ERROR;
errRep( FUNC_NAME,
"Unable to determine file name for concatenated data.",
status );
}
/* Increment the group index counter */
gcount++;
}
/* Close the smfArray */
smf_close_related( wf, &concat, status );
}
/* Write out the list of output NDF names, annulling the error if a null
parameter value is supplied. */
if( *status == SAI__OK && ogrp ) {
grpList( "OUTFILES", 0, 0, NULL, ogrp, status );
if( *status == PAR__NULL ) errAnnul( status );
}
CLEANUP:
/* Tidy up after ourselves: release the resources used by the grp routines */
if( darks ) smf_close_related( wf, &darks, status );
if( flatramps ) smf_close_related( wf, &flatramps, status );
if (heateffmap) heateffmap = smf_free_effmap( heateffmap, status );
if( bbms ) smf_close_related( wf, &bbms, status );
if( igrp ) grpDelet( &igrp, status);
if( ogrp ) grpDelet( &ogrp, status);
if( basegrp ) grpDelet( &basegrp, status );
if( igroup ) smf_close_smfGroup( &igroup, status );
fftw_cleanup();
ndfEnd( status );
}
HDSLoc *cupidGaussClumps( int type, int ndim, int *slbnd, int *subnd, void *ipd,
double *ipv, double rms, AstKeyMap *config, int velax,
double beamcorr[ 3 ], int *status ){
/*
*+
* Name:
* cupidGaussClumps
* Purpose:
* Identify clumps of emission within a 1, 2 or 3 dimensional NDF using
* the GAUSSCLUMPS algorithm.
* Language:
* Starlink C
* Synopsis:
* HDSLoc *cupidGaussClumps( int type, int ndim, int *slbnd, int *subnd,
* void *ipd, double *ipv, double rms,
* AstKeyMap *config, int velax,
* double beamcorr[ 3 ], int *status )
* Description:
* This function identifies clumps within a 1, 2 or 3 dimensional data
* array using the GAUSSCLUMPS algorithm, described by Stutski & Gusten
* (1990, ApJ 356, 513). This algorithm proceeds by fitting a Gaussian
* profile to the brightest peak in the data. It then subtracts the fit
* from the data and iterates, fitting a new ellipse to the brightest peak
* in the residuals. This continues until a termination criterion is
* reached. The main termination criterion in this implementation is
* not quite the same as in the Stutski & Gusten paper. They had two main
* termination criteria; 1) the total data sum of the fitted gaussians
* is close to the total data sum of the original data, and 2) the peak
* residual is less than a given multiple of the RMS noise in the data.
* However, 1) is very sensitive to errors in the estimation of the
* background level in the data, and 2) may never be achieved because
* the expected residuals depend not only on the RMS noise in the data
* but also on how accurately gaussian the clumps are, which is not
* known. Therefore, this implementation instead terminates when the
* peak amplitude of the fitted clumps falls below a given fraction of
* the first (i.e. largest) fitted peak.
*
* Two additional termination criteria are used; 1) If there are many
* failed attempts to fit a clump to the peak residual or if 2) a
* specified maximum number of clumps are found, then the process
* terminates early.
* Parameters:
* type
* An integer identifying the data type of the array values pointed to
* by "ipd". Must be either CUPID__DOUBLE or CUPID__FLOAT (defined in
* cupid.h).
* ndim
* The number of dimensions in the data array. Must be 1, 2 or 3.
* slbnd
* Pointer to an array holding the lower pixel index bound of the
* data array on each axis.
* subnd
* Pointer to an array holding the upper pixel index bound of the
* data array on each axis.
* ipd
* Pointer to the data array. The elements should be stored in
* Fortran order. The data type of this array is given by "itype".
* ipv
* Pointer to the input Variance array, or NULL if there is no Variance
* array. The elements should be stored in Fortran order. The data
* type of this array is "double".
* rms
* The default value for the global RMS error in the data array.
* config
* An AST KeyMap holding tuning parameters for the algorithm.
* velax
* The index of the velocity axis in the data array (if any). Only
* used if "ndim" is 3.
* beamcorr
* An array in which is returned the FWHM (in pixels) describing the
* instrumental smoothing along each pixel axis. The clump widths
* stored in the output catalogue are reduced to correct for this
* smoothing.
* status
* Pointer to the inherited status value.
* Notes:
* - The specific form of algorithm used here is informed by a Fortran
* implementation of GaussClumps obtained on 27/9/05 from
* ftp.astro.uni-bonn.de/pub/heith/gaussclumps.
* - Most of the "cupid..." functions used in this file which start
* with a "type" parameter (e.g. cupidFindMax, cupidUpdateArrays, etc) are
* actually not functions at all, but macros defined in cupid.h. These
* macros are wrappers which invoke a type-specific function (e.g.
* cupidFindMaxD, cupidFindMaxF) appropriate to the specific data type
* being used (as indicated by the "type" parameter). Macros are used in
* order to simplify the code here, and thus make the flow of the
* algorithm clearer. The source code for the type-specific functions
* are generated automatically at build time from equivalent files which
* have file type ".cupid". For instance, the files cupidfindmaxD.c and
* cupidfindmaxF.c are generated automatically from cupidfindmax.cupid.
* Also, the rlevant macros definitions and prototypes within cupid.h
* are generated automatically at build time from these ".cupid" files.
* Returned Value:
* A locator for a new HDS object which is an array of NDF structures.
* Each NDF will hold the data values associated with a single clump and
* will be the smallest possible NDF that completely contains the
* corresponding clump. Pixels not in the clump will be set bad. The
* pixel origin is set to the same value as the supplied NDF.
* Copyright:
* Copyright (C) 2009 Science & Technology Facilities Council.
* Copyright (C) 2005 Particle Physics & Astronomy Research Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* TIMJ: Tim Jenness (JAC, Hawaii)
* {enter_new_authors_here}
* History:
* 29-SEP-2005 (DSB):
* Original version.
* 7-MAR-2007 (DSB):
* Use VELORES instead of FWHMBEAM if the data is 1D.
* 7-MAR-2007 (DSB):
* Guard against segvio caused by use of null pointers that are
* returned by astMalloc if an error has occurred.
* 14-JAN-2009 (TIMJ):
* Use MERS for message filtering.
* {enter_further_changes_here}
* Bugs:
* {note_any_bugs_here}
*-
*/
/* Local Variables: */
AstKeyMap *gcconfig; /* Configuration parameters for this algorithm */
char buf[30]; /* File name buffer */
HDSLoc *ret; /* Locator for the returned array of NDFs */
double *peaks; /* Holds the "npeak" most recently fitted peak values */
double chisq; /* Chi-squared value of most recently fitted Gaussian */
double mean_peak; /* The mean of the values within "peaks" */
double mlim; /* Truncation level for Gaussians */
double new_peak; /* The value most recently added to "peaks" */
double nsig; /* No.of standard deviations at which to reject peaks */
double old_peak; /* The oldest value within "peaks" */
double peak_thresh; /* The lower threshold for clump peak values */
double sigma_peak; /* The standard deviation of the values within "peaks" */
double sumdata; /* Sum of the supplied data values */
double sum_peak2; /* Sum of the squares of the values in "peaks" */
double sum_peak; /* Sum of the values in "peaks" */
double sumclumps; /* Sum of the values in all the used clumps so far */
double x[ CUPID__GCNP3 ]; /* Parameters describing new Gaussian clump */
int *dims; /* Pointer to array of array dimensions */
int allbad; /* Are all the residuals bad? */
int area; /* Number of pixels contributing to the clump */
int area_thresh; /* The lower threshold for clump areas */
int excols; /* Are extra output columns required? */
int el; /* Number of elements in array */
size_t i; /* Loop count */
size_t iclump; /* Number of clumps found so far */
int imax; /* Index of element with largest residual */
size_t ipeak; /* Index within "peaks" at which to store the new peak */
int iter; /* Continue finding more clumps? */
double maxbad; /* Max fraction of bad pixels allowed in a clump */
size_t maxclump; /* Max no. of clumps */
int maxskip; /* Max no. of failed fits between good fits */
size_t nclump; /* Number of usable clumps */
int niter; /* Iterations performed so far */
int npad; /* No. of peaks below threshold for temination */
size_t npeak; /* The number of elements in the "peaks" array. */
int nskip; /* No. of failed fits since last good fit */
int area_below; /* Count of consecutive clump areas below the threshold */
int peaks_below; /* Count of consecutive peaks below the threshold */
void *res; /* Pointer to residuals array */
/* Initialise */
ret = NULL;
/* Abort if an error has already occurred. */
if( *status != SAI__OK ) return ret;
/* Initialise things to avoid compiler warnings. */
mean_peak = 0.0;
sigma_peak = 0.0;
new_peak = 0.0;
/* Say which method is being used. */
msgBlankif( MSG__NORM, status );
msgOutif( MSG__NORM, "", "GaussClumps:", status );
msgBlankif( MSG__VERB, status );
/* Get the AST KeyMap holding the configuration parameters for this
algorithm. */
if( !astMapGet0A( config, "GAUSSCLUMPS", (AstObject *) &gcconfig ) ) {
gcconfig = astKeyMap( " " );
astMapPut0A( config, "GAUSSCLUMPS", gcconfig, " " );
}
/* The configuration file can optionally omit the algorithm name. In this
case the "config" KeyMap may contain values which should really be in
the "gcconfig" KeyMap. Add a copy of the "config" KeyMap into "gcconfig"
so that it can be searched for any value which cannot be found in the
"gcconfig" KeyMap. */
astMapPut0A( gcconfig, CUPID__CONFIG, astCopy( config ), NULL );
/* Return the instrumental smoothing FWHMs. For 1D data, we assume the
axis is spectral and so use VELORES instead of FWHMBEAM. */
if( ndim == 1 ) {
beamcorr[ 0 ] = cupidConfigD( gcconfig, "VELORES", 2.0, status );
} else {
beamcorr[ 0 ]= cupidConfigD( gcconfig, "FWHMBEAM", 2.0, status );
beamcorr[ 1 ] = beamcorr[ 0 ];
if( ndim == 3 ) {
beamcorr[ 2 ] = beamcorr[ 0 ];
beamcorr[ velax ]= cupidConfigD( gcconfig, "VELORES", 2.0, status );
}
}
/* See if extra diagnostic info is required. */
excols = cupidConfigI( gcconfig, "EXTRACOLS", 0, status );
/* Get the maximum allowed number of failed fits between succesful fits. */
maxskip = cupidConfigI( gcconfig, "MAXSKIP", 10, status );
/* Get the maximum allowed number of failed fits between succesful fits. */
maxclump = cupidConfigI( gcconfig, "MAXCLUMPS", VAL__MAXI, status );
/* The iterative process ends when "npad" consecutive clumps all had peak
values below "peak_thresh" or all had areas below "area_thresh". */
npad = cupidConfigI( gcconfig, "NPAD", 10, status );
/* Get the RMS noise level to use. */
rms = cupidConfigD( gcconfig, "RMS", rms, status );
/* Find the size of each dimension of the data array, and the total number
of elements in the array. We use the memory management functions of the
AST library since they provide greater security and functionality than
direct use of malloc, etc. */
dims = astMalloc( sizeof( *dims )*(size_t) ndim );
el = 1;
if( dims ) {
for( i = 0; i < (size_t)ndim; i++ ) {
dims[ i ] = subnd[ i ] - slbnd[ i ] + 1;
el *= dims[ i ];
}
}
/* Copy the supplied data array into a work array which will hold the
residuals remaining after subtraction of the fitted Gaussians. The
cupidStore macro is a wrapper around the astStore function. */
res = cupidStore( NULL, ipd, el, type, "cupidGaussClumps" );
if( res ) {
/* Set the lower threshold for clump peaks to a user-specified multiple
of the RMS noise. */
peak_thresh = cupidConfigD( gcconfig, "THRESH", 2.0, status );
/* Set the lower threshold for clump area to a user-specified number of
pixels. */
area_thresh = cupidConfigI( gcconfig, "MINPIX", 3, status );
/* Get the lowest value (normalised to the RMS noise level) at which
model Gaussians should be evaluated. */
mlim = cupidConfigD( gcconfig, "MODELLIM", 0.5, status );
/* Get the max allowed fraction of bad pixels in a clump. */
maxbad = cupidConfigD( gcconfig, "MAXBAD", 0.05, status );
/* Initialise the number of clumps found so far. */
iclump = 0;
/* Indicate that no peaks have been found below the lower threshold for clump
peak values, or below the lower area threshold. */
peaks_below = 0;
area_below = 0;
/* Initialise the variables used to keep track of the mean and standard
deviation of the most recent "npeak" fitted peak values. */
nsig = cupidConfigD( gcconfig, "NSIGMA", 3.0, status );
npeak = cupidConfigI( gcconfig, "NPEAK", 9, status );
ipeak = 0;
sum_peak = 0.0;
sum_peak2 = 0.0;
iter = 1;
niter = 0;
nskip = 0;
sumclumps = 0.0;
sumdata = VAL__BADD;
peaks = astMalloc( sizeof( *peaks )*npeak );
if( peaks ) {
for( i = 0; i < npeak; i++ ) peaks[ i ] = 0.0;
/* Use the setjmp function to define here to be the place to which the
signal handling function will jump when a signal is detected. Zero is
returned on the first invocation of setjmp. If a signal is detected,
a jump is made into setjmp which then returns a positive signal
identifier. */
if( setjmp( CupidGCHere ) ) {
iter = 0;
msgBlankif( MSG__QUIET, status );
msgOutif( MSG__QUIET, "",
"Interupt detected. Clumps found so far will be saved",
status );
msgBlankif( MSG__QUIET, status );
}
}
/* Set up a signal handler for the SIGINT (interupt) signal. If this
signal occurs, the function "cupidGCHandler" will be called. */
signal( SIGINT, cupidGCHandler );
/* Loop round fitting a gaussian to the largest remaining peak in the
residuals array. */
while( iter && *status == SAI__OK ) {
/* Report the iteration number to the user if required. */
++niter;
msgBlankif( MSG__DEBUG1, status );
msgOutiff( MSG__DEBUG1, "", "Iteration %d:", status, niter );
/* Find the 1D vector index of the elements with the largest value in the
residuals array. */
allbad = cupidGCFindMax( type, res, el, &imax, &sumdata, status );
/* Finish iterating if all the residuals are bad, or if too many iterations
have been performed since the last succesfully fitted clump. */
if( allbad ) {
iter = 0;
niter--;
msgBlankif( MSG__DEBUG, status );
msgOutif( MSG__DEBUG1, "",
"There are no good pixels left to be fitted.",
status );
msgBlankif( MSG__DEBUG1, status );
} else if( nskip > maxskip ){
iter = 0;
niter--;
msgBlankif( MSG__DEBUG, status );
msgOutiff( MSG__DEBUG1, "",
"The previous %d fits were unusable.", status, maxskip );
msgBlankif( MSG__DEBUG1, status );
}
/* If not, make an initial guess at the Gaussian clump parameters centred
on the current peak. */
if( iter ) {
cupidGCSetInit( type, res, ipv, ndim, dims, imax, rms, gcconfig,
( niter == 1 ), velax, x, slbnd, status );
/* Find the best fitting parameters, starting from the above initial guess.
This returns a function value of zero if no fit could be performed. */
if( cupidGCFit( type, res, imax, x, &chisq, status ) ) {
/* Skip this fit if we have an estimate of the standard deviation of the
"npeak" most recent clump peak values, and the peak value of the clump
just fitted is a long way (more than NSIGMA standard deviations) from the
peak value of the previously fitted clump. Also skip it if the peak
value is less than the "mlim" value. */
if( ( npeak == 0 || iclump < npeak ||
fabs( x[ 0 ] - new_peak ) < nsig*sigma_peak ) &&
x[ 0 ] > mlim ) {
/* Record the new peak value for use with the next peak, and update the
standard deviation of the "npeak" most recent peaks. These values are
stored cyclically in the "peaks" array. */
if( npeak > 0 ) {
new_peak = x[ 0 ];
old_peak = peaks[ ipeak ];
peaks[ ipeak ] = new_peak;
if( ++ipeak == npeak ) ipeak = 0;
sum_peak += new_peak - old_peak;
sum_peak2 += new_peak*new_peak - old_peak*old_peak;
if( sum_peak2 < 0.0 ) sum_peak2 = 0.0;
mean_peak = sum_peak/npeak;
sigma_peak = sqrt( sum_peak2/npeak - mean_peak*mean_peak );
}
/* Increment the number of peaks found. */
iclump++;
/* Reset the number of failed fits since the last good fit. */
nskip = 0;
/* Remove the model fit (excluding the background) from the residuals
array. This also creates an NDF containing the data values associated
with the clump. This NDF is stored in the HDS array of NDFs in the
returned HDS object. The standard deviation of the new residuals is
returned. */
cupidGCUpdateArrays( type, res, ipd, el, ndim, dims,
x, rms, mlim, imax, peak_thresh, slbnd,
&ret, iclump, excols, mean_peak,
maxbad, &area, &sumclumps, status );
/* Dump the modified residuals if required. */
sprintf( buf, "residuals%lu", iclump );
cupidGCDump( type, MSG__DEBUG3, res, ndim, dims, buf,
status );
/* Display the clump parameters on the screen if required. */
cupidGCListClump( iclump, ndim, x, chisq, slbnd,
rms, status );
/* If this clump has a peak value which is below the threshold, increment
the count of consecutive clumps with peak value below the threshold.
Otherwise, reset this count to zero. */
if( x[ 0 ] < peak_thresh ) {
peaks_below++;
} else {
peaks_below = 0;
}
/* If this clump has an area which is below the threshold, increment
the count of consecutive clumps with area below the threshold.
Otherwise, reset this count to zero. */
if( area < area_thresh ) {
area_below++;
} else {
area_below = 0;
}
/* If the maximum number of clumps have now been found, exit.*/
if( iclump == maxclump ) {
iter = 0;
msgBlankif( MSG__DEBUG, status );
msgOutiff( MSG__DEBUG1, "",
"The specified maximum number of "
"clumps (%lu) have been found.", status,
maxclump );
msgBlankif( MSG__DEBUG1, status );
/* If the integrated data sum in the fitted gaussians exceeds or equals
the integrated data sum in th einput, exit. */
} else if( sumclumps >= sumdata ) {
iter = 0;
msgBlankif( MSG__DEBUG, status );
msgOutiff( MSG__DEBUG1,"",
"The total data sum of the fitted "
"Gaussians (%g) has reached the total "
"data sum in the supplied data (%g).",
status, (float)sumclumps, (float)sumdata );
msgBlankif( MSG__DEBUG1, status );
/* If the count of consecutive peaks below the threshold has reached
"Npad", terminate. */
} else if( peaks_below == npad ) {
iter = 0;
msgBlankif( MSG__DEBUG, status );
msgOutiff( MSG__DEBUG1, "",
"The previous %d clumps all had peak "
"values below the threshold.", status,
npad );
msgBlankif( MSG__DEBUG1, status );
/* If the count of consecutive clumps with area below the threshold has reached
"Npad", terminate. */
} else if( area_below == npad ) {
iter = 0;
msgBlankif( MSG__DEBUG, status );
msgOutiff( MSG__DEBUG1, "",
"The previous %d clumps all had areas "
"below the threshold.", status, npad );
msgBlankif( MSG__DEBUG1, status );
}
/* If the peak value fitted is very different from the previous fitted peak
value, set the residuals array element bad in order to prevent the
algorithm from trying to fit a peak to the same pixel again. */
} else {
if( type == CUPID__DOUBLE ) {
((double *)res)[ imax ] = VAL__BADD;
} else {
((float *)res)[ imax ] = VAL__BADR;
}
new_peak = 0.5*( new_peak + x[ 0 ] );
nskip++;
msgOutif( MSG__DEBUG1, "", " Clump rejected due to "
"aberrant peak value.", status );
}
/* Tell the user if no clump could be fitted around the current peak
pixel value */
} else {
nskip++;
msgOutif( MSG__DEBUG1, "", " No clump fitted.", status );
/* Set the specified element of the residuals array bad if no fit was
performed. This prevents the any subsequent attempt to fit a Gaussian
to the same peak value.*/
if( type == CUPID__DOUBLE ) {
((double *)res)[ imax ] = VAL__BADD;
} else {
((float *)res)[ imax ] = VAL__BADR;
}
}
/* Tell the user if one of the trmination criteria has ben met. */
} else {
msgOutif( MSG__DEBUG1, "",
" At least one termination criterion has been reached.",
status );
msgBlankif( MSG__DEBUG1, status );
}
}
/* Tell the user how clumps are being returned. */
if( ret ) {
datSize( ret, &nclump, status );
} else {
nclump = 0;
}
if( nclump == 0 ) msgOutif( MSG__NORM, "", "No usable clumps found.", status );
if( iclump - nclump == 1 ) {
msgOutif( MSG__NORM, "",
"1 clump rejected because it touches an edge of "
"the data array.", status );
} else if( iclump - nclump > 1 ) {
msgOutiff( MSG__NORM, "",
"%d clumps rejected because they touch an edge of "
"the data array.", status, (int)( iclump - nclump ) );
}
/* Tell the user how many iterations have been performed (i.e. how many
attempts there have been to fit a Gaussian peak). */
if( niter == 1 ){
msgOutif( MSG__DEBUG1, "", "No fit attempted.", status );
} else {
msgOutiff( MSG__DEBUG1, "",
"Fits attempted for %d candidate clumps (%d failed).",
status, (int)( niter - iclump ), niter );
}
/* Free resources */
peaks = astFree( peaks );
}
/* Remove the secondary KeyMap added to the KeyMap containing configuration
parameters for this algorithm. This prevents the values in the secondary
KeyMap being written out to the CUPID extension when cupidStoreConfig is
called. */
astMapRemove( gcconfig, CUPID__CONFIG );
/* Free resources */
res = astFree( res );
dims = astFree( dims );
cupidGC.data = astFree( cupidGC.data );
cupidGC.weight = astFree( cupidGC.weight );
cupidGC.res = astFree( cupidGC.res );
cupidGC.resu = astFree( cupidGC.resu );
cupidGC.initmodel = astFree( cupidGC.initmodel );
cupidGC.model = astFree( cupidGC.model );
cupidGC.resids = astFree( cupidGC.resids );
gcconfig = astAnnul( gcconfig );
/* Return the list of clump NDFs. */
return ret;
}
void smf_calcmodel_gai( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel, int flags,
int *status) {
/* Local Variables */
size_t bstride; /* bolometer stride */
dim_t gain_box=0; /* No. of time slices in a block */
size_t gbstride; /* GAIn bolo stride */
size_t gcstride; /* GAIn coeff stride */
int gflat=0; /* correct flatfield using GAI */
dim_t i; /* Loop counter */
dim_t idx=0; /* Index within subgroup */
dim_t j; /* Loop counter */
AstKeyMap *kmap=NULL; /* Local GAIn keymap */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
dim_t nblock; /* No. of time slice blocks */
dim_t nbolo; /* Number of bolometers */
dim_t ndata; /* Number of data points */
smfArray *noi=NULL; /* Pointer to NOI at chunk */
double *noi_data=NULL; /* Pointer to DATA component of model */
size_t noibstride; /* bolo stride for noise */
dim_t nointslice; /* number of time slices for noise */
size_t noitstride; /* Time stride for noise */
dim_t npar; /* No. of parameters per bolometer */
dim_t ntslice; /* Number of time slices */
int oldalg = 1; /* Is the old COM algorithm being used? */
smfArray *qua=NULL; /* Pointer to QUA at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to quality data */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DAT */
double *scale; /* Pointer to scale factor */
size_t tstride; /* time slice stride */
double *wg; /* Workspace holding time slice gains */
double *woff; /* Workspace holding time slice offsets */
/* Main routine */
if( *status != SAI__OK ) return;
if( !(flags&SMF__DIMM_INVERT) ) return;
/* See if the new sigma-clipping COM algorithm is being used. */
astMapGet0A( keymap, "COM", &kmap );
astMapGet0I( kmap, "OLDALG", &oldalg );
kmap = astAnnul( kmap );
/* Obtain pointer to sub-keymap containing GAI parameters */
if( !astMapHasKey( keymap, "GAI" ) ) return;
astMapGet0A( keymap, "GAI", &kmap );
astMapGet0I( kmap, "FLATFIELD", &gflat );
if( kmap ) kmap = astAnnul( kmap );
/* Report an error if gai.flatfield is used with the new COM algorithm. */
if( !oldalg && gflat && *status == SAI__OK ) {
errRep( "", "Cannot use GAI.FLATFIELD with new COM algorithm.", status );
}
/* Only have to do something if gai.flatfield set */
if( !gflat || *status != SAI__OK ) return;
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
model = allmodel[chunk];
if(dat->noi) noi = dat->noi[chunk];
/* Get the number of blocks into which to split each time series. Each box
(except possibly the last one contains "gain_box" time slices. */
astMapGet0A( keymap, "COM", &kmap );
smf_get_nsamp( kmap, "GAIN_BOX", res->sdata[0], &gain_box, status );
if (kmap) kmap = astAnnul( kmap );
if (*status != SAI__OK) return;
/* Ensure everything is in bolo-order */
smf_model_dataOrder( wf, dat, allmodel, chunk, SMF__RES|SMF__QUA|SMF__NOI, 0, status );
/* Loop over index in subgrp (subarray) */
for( idx=0; idx<res->ndat; idx++ ) {
/* Get pointers to DATA components */
res_data = (res->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
if( noi ) {
smf_get_dims( noi->sdata[idx], NULL, NULL, NULL, &nointslice,
NULL, &noibstride, &noitstride, status);
noi_data = (double *)(noi->sdata[idx]->pntr)[0];
}
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": Null data in inputs", status);
} else {
/* Get the raw data dimensions */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice, &ndata,
&bstride, &tstride, status);
smf_get_dims( model->sdata[idx], NULL, NULL, NULL, &npar, NULL,
&gbstride, &gcstride, status);
/* If com.gain_box is zero, use a value of ntslice, so that a single
box will be used covering the whoel time stream. */
if( gain_box == 0 ) gain_box = ntslice;
/* Allocate work space for the gain and offset for each time slice. */
woff = astMalloc( ntslice*sizeof( *woff ) );
wg = astMalloc( ntslice*sizeof( *wg ) );
/* Get the number of blocks into which the time stream is divided.
Each block has a separate gain, offset and correlation factor
for each bolometer. */
nblock = npar/3;
/* Undo the gain correction stored in GAI (the gain is applied to
the signal and noise in smf_calcmodel_com) */
for( i=0; i<nbolo; i++ ) {
if( !(qua_data[i*bstride]&SMF__Q_BADB) ) {
/* Get the gain and offset for each time slice of this bolometer. */
smf_gandoff( i, 0, ntslice - 1, ntslice, gbstride, gcstride,
model_data, nblock, gain_box, wg, woff, NULL, status );
/* First undo the flatfield correction to the signal */
scale = wg;
for( j=0; j<ntslice; j++,scale++ ) {
if( !(qua_data[i*bstride + j*tstride]&SMF__Q_MOD) &&
*scale != VAL__BADD && *scale > 0.0 ) {
res_data[i*bstride + j*tstride] *= *scale;
}
}
/* Then scale the noise. */
if( noi ) {
scale = wg;
for( j=0; j<nointslice; j++,scale++ ) {
if( noi_data[i*noibstride + j*noitstride] != VAL__BADD &&
*scale != VAL__BADD && *scale > 0.0 ) {
noi_data[i*noibstride + j*noitstride] *= (*scale) * (*scale);
}
}
}
}
}
/* Free work space. */
woff = astFree( woff );
wg = astFree( wg );
}
}
}
HDSLoc *cupidReinhold( int type, int ndim, int *slbnd, int *subnd, void *ipd,
double *ipv, double rms, AstKeyMap *config, int velax,
double beamcorr[ 3 ], int *status ){
/*
*+
* Name:
* cupidReinhold
* Purpose:
* Identify clumps of emission within a 1, 2 or 3 dimensional NDF using
* the REINHOLD algorithm.
* Language:
* Starlink C
* Synopsis:
* HDSLoc *cupidReinhold( int type, int ndim, int *slbnd, int *subnd,
* void *ipd, double *ipv, double rms,
* AstKeyMap *config, int velax,
* double beamcorr[ 3 ], int *status )
* Description:
* This function identifies clumps within a 1, 2 or 3 dimensional data
* array using the REINHOLD algorithm, developed by Kim Reinhold at
* JAC. This algorithm identifies the boundaries between clumps by
* looking for minima in 1D sections through the data. No a priori clump
* profile is assumed. In this algorithm, clumps never overlap.
* Parameters:
* type
* An integer identifying the data type of the array values pointed to
* by "ipd". Must be either CUPID__DOUBLE or CUPID__FLOAT (defined in
* cupid.h).
* ndim
* The number of dimensions in the data array. Must be 2 or 3.
* slbnd
* Pointer to an array holding the lower pixel index bound of the
* data array on each axis.
* subnd
* Pointer to an array holding the upper pixel index bound of the
* data array on each axis.
* ipd
* Pointer to the data array. The elements should be stored in
* Fortran order. The data type of this array is given by "itype".
* ipv
* Pointer to the input Variance array, or NULL if there is no Variance
* array. The elements should be stored in Fortran order. The data
* type of this array is "double".
* rms
* The default value for the global RMS error in the data array.
* config
* An AST KeyMap holding tuning parameters for the algorithm.
* velax
* The index of the velocity axis in the data array (if any). Only
* used if "ndim" is 3.
* beamcorr
* An array in which is returned the FWHM (in pixels) describing the
* instrumental smoothing along each pixel axis. The clump widths
* stored in the output catalogue are reduced to correct for this
* smoothing.
* status
* Pointer to the inherited status value.
* Returned Value:
* A locator for a new HDS object which is an array of NDF structures.
* Each NDF will hold the data values associated with a single clump
* and will be the smallest possible NDF that completely contains the
* corresponding clump. Pixels not in the clump will be set bad. The
* pixel origin is set to the same value as the supplied NDF.
* Copyright:
* Copyright (C) 2009 Science & Technology Facilities Council.
* Copyright (C) 2006 Particle Physics & Astronomy Research Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* TIMJ: Tim Jenness (JAC, Hawaii)
* {enter_new_authors_here}
* History:
* 16-JAN-2006 (DSB):
* Original version.
* 14-JAN-2009 (TIMJ):
* Use MERS for message filtering.
* {enter_further_changes_here}
* Bugs:
* {note_any_bugs_here}
*-
*/
/* Local Variables: */
AstKeyMap *rconfig; /* Configuration parameters for this algorithm */
HDSLoc *ret; /* Locator for the returned array of NDFs */
double *pd; /* Pointer to next element of data array */
double *peakvals; /* Pointer to array holding clump peak values */
double cathresh; /* Threshold for second cellular automata */
double flatslope; /* Minimum significant slope at edge of a peak */
double noise; /* Noise level */
double pv; /* Pixel value */
double thresh; /* Minimum peak value to be considered */
float *pf; /* Pointer to next element of data array */
int *clbnd; /* Array holding lower axis bounds of all clumps */
int *cubnd; /* Array holding upper axis bounds of all clumps */
int *igood; /* Pointer to array holding usable clump indices */
int *m1; /* Pointer to mask array */
int *m2; /* Pointer to mask array */
int *m3; /* Pointer to mask array */
int *mask2; /* Pointer to array marking out edge pixels */
int *mask; /* Pointer to array marking out edge pixels */
int *nrem; /* Pointer to array marking out edge pixels */
int *pa; /* Pointer to next element in the pixel assignment array */
int caiter; /* The number of CA iterations to perform */
int dims[3]; /* Pointer to array of array dimensions */
int el; /* Number of elements in array */
int fixiter; /* The number of CA iterations to perform */
int i; /* Loop count */
int ii; /* Loop count */
int ix; /* Grid index on 1st axis */
int iy; /* Grid index on 2nd axis */
int iz; /* Grid index on 3rd axis */
int j; /* Loop count */
int maxid; /* Largest id for any peak (smallest is zero) */
int minlen; /* Minimum size of a clump in pixels along one dimension*/
int minpix; /* Minimum total size of a clump in pixels */
int more; /* Continue looping?*/
int ngood; /* Number of good clumps */
int nsmall; /* Number of clumps that are too small */
int nthin; /* Number of clumps that span only a single pixel */
int peakval; /* Minimum value used to flag peaks */
int skip[3]; /* Pointer to array of axis skips */
/* Initialise */
ret = NULL;
/* Abort if an error has already occurred. */
if( *status != SAI__OK ) return ret;
/* Say which method is being used. */
msgBlankif( MSG__NORM, status );
msgOutif( MSG__NORM, "", "Reinhold:", status );
/* Get the AST KeyMap holding the configuration parameters for this
algorithm. */
if( !astMapGet0A( config, "REINHOLD", (AstObject *) &rconfig ) ) {
rconfig = astKeyMap( " " );
astMapPut0A( config, "REINHOLD", rconfig, " " );
}
/* The configuration file can optionally omit the algorithm name. In this
case the "config" KeyMap may contain values which should really be in
the "rconfig" KeyMap. Add a copy of the "config" KeyMap into "rconfig"
so that it can be searched for any value which cannot be found in the
"rconfig" KeyMap. */
astMapPut0A( rconfig, CUPID__CONFIG, astCopy( config ), NULL );
/* Return the instrumental smoothing FWHMs */
beamcorr[ 0 ] = cupidConfigD( rconfig, "FWHMBEAM", 2.0, status );
beamcorr[ 1 ] = beamcorr[ 0 ];
if( ndim == 3 ) {
beamcorr[ 2 ] = beamcorr[ 0 ];
beamcorr[ velax ]= cupidConfigD( rconfig, "VELORES", 2.0, status );
}
/* Find the size of each dimension of the data array, and the total number
of elements in the array, and the skip in 1D vector index needed to
move by pixel along an axis. We use the memory management functions of the
AST library since they provide greater security and functionality than
direct use of malloc, etc. */
el = 1;
for( i = 0; i < ndim; i++ ) {
dims[ i ] = subnd[ i ] - slbnd[ i ] + 1;
el *= dims[ i ];
skip[ i ] = ( i == 0 ) ? 1 : skip[ i - 1 ]*dims[ i - 1 ];
}
for( ; i < 3; i++ ) {
dims[ i ] = 1;
skip[ i ] = 0;
}
/* Get various configuration parameters. */
rms = cupidConfigD( rconfig, "RMS", rms, status );
minlen = cupidConfigI( rconfig, "MINLEN", 4, status );
noise = cupidConfigRMS( rconfig, "NOISE", rms, 2*rms, status );
thresh = cupidConfigRMS( rconfig, "THRESH", rms, noise + 2*rms, status );
flatslope = cupidConfigRMS( rconfig, "FLATSLOPE", rms, rms, status );
cathresh = pow( 3, ndim ) - 1.0;
cathresh = cupidConfigI( rconfig, "CATHRESH", (int) cathresh, status );
caiter = cupidConfigI( rconfig, "CAITERATIONS", 1, status );
fixiter = cupidConfigI( rconfig, "FIXCLUMPSITERATIONS", 1, status );
/* Get the minimum allowed number of pixels in a clump. */
minpix = cupidDefMinPix( ndim, beamcorr, noise, thresh, status );
minpix = cupidConfigI( rconfig, "MINPIX", minpix, status );
/* Convert CATHRESH from a number of pixels to a fraction. */
cathresh = cathresh/pow( 3, ndim );
if( cathresh > 0.98 ) cathresh = 0.98;
if( cathresh < 0 ) cathresh = 0.0;
cathresh += 0.01;
/* Get a mask which is the same size and shape as the data array and which
holds CUPID__KEDGE at every pixel thought to be on the edge of a clump.
This is done by scanning the data cube using sets of parallel lines in
different directions. Peaks are searched for in each line, and then the
edges found by following the curve down from each peak until the
gradient becomes zero or positive, or until the data value drops below a
threshold value. Pixels which correspond to peaks in the data cube
are flagged with the value greater than or equal to the returned
"*peakval" value. All other pixels are set to some other value (which
will usually be CUPID__KBACK but will be something else at positions of
peaks which were not peaks in all scan directions). */
msgOutif( MSG__DEBUG, "", "Finding clump edges...", status );
mask = cupidRInitEdges( type, ipd, el, ndim, dims, skip, minlen, thresh,
noise, rms, flatslope, &peakval, status );
/* Dilate the edge regions using a cellular automata. This creates a new
mask array in which a pixel is marked as an edge pixel if any of its
neighbours are marked as edge pixels in the mask array created above. */
msgOutif( MSG__DEBUG, "", "Dilating clump edges...", status );
mask2 = cupidRCA( mask, NULL, el, dims, skip, 0.0, peakval, CUPID__KEDGE,
CUPID__KBACK, 0, status );
/* Erode the edge regions using a second cellular automata. This over-writes
the original mask array so that a pixel is marked as an edge pixel if a
fraction greater than "cathresh" of neighbouring pixels are marked as edge
pixels in "mask2". We loop doing this "CAiteration" times. */
if( caiter > 0 ) msgOutif( MSG__DEBUG,"", "Eroding clump edges...", status );
m1 = mask;
m2 = mask2;
for( i = 0; i < caiter; i++ ) {
m1 = cupidRCA( m2, m1, el, dims, skip, cathresh, peakval, CUPID__KEDGE,
CUPID__KBACK, 0, status );
m3 = m1;
m1 = m2;
m2 = m3;
}
/* Fill the volume around each peak with integer values which indicate
which peak they are close to. All the pixels around one peak form one
clump. */
msgOutif( MSG__DEBUG, "", "Filling clumps...", status );
maxid = cupidRFillClumps( m2, m1, el, ndim, skip, dims, peakval, status );
/* Abort if no clumps found. */
if( maxid < 0 ) {
msgOutif( MSG__NORM, "", "No usable clumps found.", status );
msgBlankif( MSG__NORM, status );
goto L10;
}
/* Smooth the boundaries between the clumps. This cellular automata replaces
each output pixel by the most commonly occuring value within a 3x3x3
cube of input pixels centred on the output pixel. Put the smoothed
results back into the supplied "m1" array. */
if( fixiter >0 ) msgOutif( MSG__DEBUG, "", "Smoothing clump boundaries...", status );
for( i = 0; i < fixiter; i++ ) {
m2 = cupidRCA2( m1, m2, el, dims, skip, status );
m3 = m2;
m2 = m1;
m1 = m3;
}
/* Allocate an array used to store the number of pixels remaining in each
clump. */
nrem = astMalloc( sizeof( int )*( maxid + 1 ) );
/* Allocate an array used to store the peak value in every clump. */
peakvals = astMalloc( sizeof( double )*( maxid + 1 ) );
/* Determine the bounding box of every clump. First allocate memory to
hold the bounding boxes. */
clbnd = astMalloc( sizeof( int )*( maxid + 1 )*3 );
cubnd = astMalloc( sizeof( int )*( maxid + 1 )*3 );
igood = astMalloc( sizeof( int )*( maxid + 1 ) );
if( igood ) {
/* Initialise a list to hold zero for every clump id. These values are
used to count the number of pixels remaining in each clump. Also
initialise the peak values to a very negative value. */
for( i = 0; i <= maxid; i++ ) {
nrem[ i ] = 0;
peakvals[ i ] = VAL__MIND;
}
/* Initialise the bounding boxes. */
for( i = 0; i < 3*( maxid + 1 ); i++ ) {
clbnd[ i ] = VAL__MAXI;
cubnd[ i ] = VAL__MINI;
}
/* Loop round every pixel in the final pixel assignment array. */
if( type == CUPID__DOUBLE ) {
pd = (double *) ipd;
pf = NULL;
} else {
pf = (float *) ipd;
pd = NULL;
}
pa = m1;
for( iz = 1; iz <= dims[ 2 ]; iz++ ){
for( iy = 1; iy <= dims[ 1 ]; iy++ ){
for( ix = 1; ix <= dims[ 0 ]; ix++, pa++ ){
/* Get the data value at this pixel */
if( type == CUPID__DOUBLE ) {
pv = *(pd++);
} else {
pv = (double) *(pf++);
}
/* Skip pixels which are not in any clump. */
if( *pa >= 0 ) {
/* Increment the number of pixels in this clump. */
++( nrem[ *pa ] );
/* If this pixel value is larger than the current peak value for this
clump, record it. */
if( pv > (double) peakvals[ *pa ] ) peakvals[ *pa ] = pv;
/* Get the index within the clbnd and cubnd arrays of the current bounds
on the x axis for this clump. */
i = 3*( *pa );
/* Update the bounds for the x axis, then increment to get the index of the y
axis bounds. */
if( ix < clbnd[ i ] ) clbnd[ i ] = ix;
if( ix > cubnd[ i ] ) cubnd[ i ] = ix;
i++;
/* Update the bounds for the y axis, then increment to get the index of the z
axis bounds. */
if( iy < clbnd[ i ] ) clbnd[ i ] = iy;
if( iy > cubnd[ i ] ) cubnd[ i ] = iy;
i++;
/* Update the bounds for the z axis. */
if( iz < clbnd[ i ] ) clbnd[ i ] = iz;
if( iz > cubnd[ i ] ) cubnd[ i ] = iz;
}
}
}
}
/* Loop round counting the clumps which are too small or too low. Put the
indices of usable clumps into another array. */
nsmall = 0;
ngood = 0;
nthin = 0;
for( i = 0; i <= maxid; i++ ) {
j = 3*i;
if( nrem[ i ] <= minpix ) {
nsmall++;
} else if( clbnd[ j ] == cubnd[ j ] ||
( clbnd[ j + 1 ] == cubnd[ j + 1 ] && ndim > 1 ) ||
( clbnd[ j + 2 ] == cubnd[ j + 2 ] && ndim > 2 ) ) {
nthin++;
} else {
igood[ ngood++ ] = i;
}
}
if( ngood == 0 ) msgOutif( MSG__NORM,"", "No usable clumps found.", status );
if( nsmall == 1 ){
msgOutif( MSG__NORM, "", "One clump rejected because it contains too few pixels.", status );
} else if( nsmall > 1 ) {
msgSeti( "N", nsmall );
msgOutif( MSG__NORM, "", "^N clumps rejected because they contain too few pixels.", status );
}
if( nthin == 1 ) {
msgOutif( MSG__NORM, "", "1 clump rejected because it spans only a single "
"pixel along one or more axes.", status );
} else if( nthin > 1 ) {
msgSeti( "N", nthin );
msgOutif( MSG__NORM, "", "^N clumps rejected because they spans only a single "
"pixel along one or more axes.", status );
}
/* Sort the clump indices into descending order of peak value. */
j = ngood;
more = 1;
while( more ) {
j--;
more = 0;
for( i = 0; i < j; i++ ) {
if( peakvals[ igood[ i ] ] < peakvals[ igood[ i + 1 ] ] ) {
ii = igood[ i + 1 ];
igood[ i + 1 ] = igood[ i ];
igood[ i ] = ii;
more = 1;
}
}
}
/* Loop round creating an NDF describing each usable clump. */
for( j = 0; j < ngood; j++ ) {
i = igood[ j ];
ret = cupidNdfClump( type, ipd, m1, el, ndim, dims, skip, slbnd,
i, clbnd + 3*i, cubnd + 3*i, NULL, ret,
cupidConfigD( rconfig, "MAXBAD", 0.05, status ),
status );
}
/* Free resources */
clbnd = astFree( clbnd );
cubnd = astFree( cubnd );
igood = astFree( igood );
}
peakvals = astFree( peakvals );
nrem = astFree( nrem );
L10:;
/* Remove the secondary KeyMap added to the KeyMap containing configuration
parameters for this algorithm. This prevents the values in the secondary
KeyMap being written out to the CUPID extension when cupidStoreConfig is
called. */
astMapRemove( rconfig, CUPID__CONFIG );
/* Free resources */
rconfig = astAnnul( rconfig );
mask = astFree( mask );
mask2 = astFree( mask2 );
/* Return the list of clump NDFs. */
return ret;
}
/* Main entry point. */
void smf_calcmodel_smo( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel,
int flags __attribute__((unused)),
int *status) {
/* Local Variables */
size_t bstride; /* bolo stride */
dim_t boxcar = 0; /* size of boxcar smooth window */
smf_filt_t filter_type; /* The type of smoothing to perform */
size_t i; /* Loop counter */
dim_t idx=0; /* Index within subgroup */
int iworker; /* Owkrer index */
smfCalcmodelSmoJobData *job_data=NULL; /* Pointer to all job data structures */
AstKeyMap *kmap=NULL; /* Pointer to PLN-specific keys */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
double *model_data_copy=NULL; /* Copy of model_data for one bolo */
dim_t nbolo=0; /* Number of bolometers */
dim_t ndata=0; /* Total number of data points */
int notfirst=0; /* flag for delaying until after 1st iter */
dim_t ntslice=0; /* Number of time slices */
int nworker; /* No. of worker threads in supplied Workforce */
smfCalcmodelSmoJobData *pdata=NULL; /* Pointer to current data structure */
smfArray *qua=NULL; /* Pointer to QUA at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to quality data */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
int step; /* Number of bolometers per thread */
size_t tstride; /* Time slice stride in data array */
const char * typestr = NULL; /* smo.type value */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
/* Obtain pointer to sub-keymap containing PLN parameters. Something will
always be available.*/
astMapGet0A( keymap, "SMO", &kmap );
/* Are we skipping the first iteration? */
astMapGet0I(kmap, "NOTFIRST", ¬first);
if( notfirst && (flags & SMF__DIMM_FIRSTITER) ) {
msgOutif( MSG__VERB, "", FUNC_NAME
": skipping SMO this iteration", status );
return;
}
/* Get the boxcar size */
if( kmap ) smf_get_nsamp( kmap, "BOXCAR", res->sdata[0], &boxcar, status );
/* Get the type of smoothing filter to use. Anthing that is not "MEDIAN" is mean */
filter_type = SMF__FILT_MEAN;
if (astMapGet0C( kmap, "TYPE", &typestr ) ) {
if (strncasecmp( typestr, "MED", 3 ) == 0 ) {
filter_type = SMF__FILT_MEDIAN;
}
}
/* Assert bolo-ordered data */
smf_model_dataOrder( wf, dat, allmodel, chunk, SMF__RES|SMF__QUA,
0, status );
smf_get_dims( res->sdata[0], NULL, NULL, NULL, &ntslice,
&ndata, NULL, NULL, status);
model = allmodel[chunk];
msgOutiff(MSG__VERB, "",
" Calculating smoothed model using boxcar of width %" DIM_T_FMT " time slices",
status, boxcar);
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
/* Loop over index in subgrp (subarray) and put the previous iteration
of the filtered component back into the residual before calculating
and removing the new filtered component */
for( idx=0; (*status==SAI__OK)&&(idx<res->ndat); idx++ ) {
/* Obtain dimensions of the data */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice,
&ndata, &bstride, &tstride, status);
/* Get pointers to data/quality/model */
res_data = (res->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Uncomment to aid debugging */
/*
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_in",
NULL, 0, 0, MSG__VERB, 0, status );
*/
if( *status == SAI__OK ) {
/* Place last iteration back into residual if this is a smoothable section of the time series */
for (i=0; i< ndata; i++) {
if ( !(qua_data[i]&SMF__Q_FIT) && res_data[i] != VAL__BADD && model_data[i] != VAL__BADD ) {
res_data[i] += model_data[i];
}
}
}
/* Uncomment to aid debugging */
/*
smf_write_smfData( model->sdata[idx], NULL, qua_data, "model_b4",
NULL, 0, 0, MSG__VERB, 0, status );
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_b4",
NULL, 0, 0, MSG__VERB, 0, status );
*/
/* Determine which bolometers are to be processed by which threads. */
step = nbolo/nworker;
if( step < 1 ) step = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->b1 = iworker*step;
pdata->b2 = pdata->b1 + step - 1;
}
/* Ensure that the last thread picks up any left-over bolometers */
pdata->b2 = nbolo - 1;
/* Store all the other info needed by the worker threads, and submit the
jobs to apply the smoothing. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->boxcar = boxcar;
pdata->bstride = bstride;
pdata->bstride = bstride;
pdata->filter_type = filter_type;
pdata->model_data = model_data;
pdata->nbolo = nbolo;
pdata->nbolo = nbolo;
pdata->ntslice = ntslice;
pdata->ntslice = ntslice;
pdata->qua_data = qua_data;
pdata->qua_data = qua_data;
pdata->res_data = res_data;
pdata->res_data = res_data;
pdata->tstride = tstride;
pdata->tstride = tstride;
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_calcmodel_smo_job,
0, NULL, status );
}
thrWait( wf, status );
/* Uncomment to aid debugging */
/*
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_af",
NULL, 0, 0, MSG__VERB, 0, status );
smf_write_smfData( model->sdata[idx], NULL, qua_data, "model_af",
NULL, 0, 0, MSG__VERB, 0, status );
*/
}
}
/* Free work space (astFree returns without action if a NULL pointer is
supplied). */
model_data_copy = astFree( model_data_copy );
job_data = astFree( job_data );
/* Annul AST Object pointers (astAnnul reports an error if a NULL pointer
is supplied). */
if( kmap ) kmap = astAnnul( kmap );
}
F77_SUBROUTINE(configecho)( INTEGER(STATUS) ){
/*
*+
* Name:
* CONFIGECHO
* Purpose:
* Displays one or more configuration parameters.
* Language:
* C (designed to be called from Fortran)
* Type of Module:
* ADAM A-task
* Invocation:
* CALL CONFIGECHO( STATUS )
* Arguments:
* STATUS = INTEGER (Given and Returned)
* The global status.
* Description:
* This application displays the name and value of one or all
* configuration parameters, specified using Parameters CONFIG or
* NDF. If a single parameter is displayed, its value is also
* written to an output parameter. If the parameter value is not
* specified by the CONFIG, NDF or DEFAULTS parameter, then the
* value supplied for DEFVAL is displayed.
*
* If an input NDF is supplied then configuration parameters
* are read from its history (see Parameters NDF and APPLICATION).
*
* If values are supplied for both CONFIG and NDF, then the
* differences between the two sets of configuration parameters
* are displayed (see Parameter NDF).
* Usage:
* configecho name config [defaults] [select] [defval]
* ADAM Parameters:
* APPLICATION = LITERAL (Read)
* When reading configuration parameters from the history
* of an NDF, this parameter specifies the name of the application
* to find in the history. There must be a history component
* corresponding to the value of this parameter, and it must
* include a CONFIG group. [current value]
* CONFIG = GROUP (Read)
* Specifies values for the configuration parameters. If the string
* "def" (case-insensitive) or a null (!) value is supplied, the
* configuration parameters are obtained using Parameter NDF. If
* a null value is also supplied for NDF, a set of default
* configuration parameter values will be used, as specified by
* Parameter DEFAULTS.
*
* The supplied value should be either a comma-separated list of
* strings or the name of a text file preceded by an up-arrow
* character "^", containing one or more comma-separated lists of
* strings. Each string is either a "keyword=value" setting, or the
* name of a text file preceded by an up-arrow character "^". Such
* text files should contain further comma-separated lists which
* will be read and interpreted in the same manner (any blank lines
* or lines beginning with "#" are ignored). Within a text file,
* newlines can be used as delimiters, as well as commas. Settings
* are applied in the order in which they occur within the list,
* with later settings overriding any earlier settings given for
* the same keyword.
*
* Each individual setting should be of the form "<keyword>=<value>".
* If a non-null value is supplied for Parameter DEFAULTS, an error
* will be reported if CONFIG includes values for any parameters
* that are not included in DEFAULTS.
* DEFAULTS = LITERAL (Read)
* The path to a file containing the default value for every
* allowed configuration parameter. If null (!) is supplied, no
* defaults will be supplied for parameters that are not specified
* by CONFIG, and no tests will be performed on the validity of
* paramter names supplied by CONFIG. [!]
* DEFVAL = LITERAL (Read)
* The value to return if no value can be obtained for the named
* parameter, or if the value is "<undef>". [<***>]
* NAME = LITERAL (Read)
* The name of the configuration parameter to display. If set to
* null (!), then all parameters defined in the configuration are
* displayed.
* NDF = NDF (Read)
* An NDF file containing history entries which include
* configuration parameters. If not null (!) the history
* of the NDF will be searched for a component corresponding
* to the Parameter APPLICATION. The Parameter CONFIG
* is then optional, but if it too is not null (!) then
* the output will show the differences between the configuration
* stored in the NDF history and the given configuration:
* new parameters and those different from the reference
* configuration (given by Parameter CONFIG) are prefixed
* with "+" and those which are the same as the reference
* configuration are prefixed with "-". [!]
* SELECT = GROUP (Read)
* A group that specifies any alternative prefixes that can be
* included at the start of any parameter name. For instance, if
* this group contains the two entries "450=1" and "850=0", then
* either CONFIG or DEFAULTS can specify two values for any single
* parameter -- one for the parameter prefixed by "450." and another
* for the parameter prefixed by "850.". Thus, for instance, if
* DEFAULTS defines a parameter called "filter", it could include
* "450.filter=300" and "850.filter=600". The CONFIG parameter could
* then either set the filter parameter for a specific prefix (as
* in "450.filter=234"); or it could leave the prefix unspecified,
* in which case the prefix used is the first one with a
* non-zero value in SELECT (450 in the case of this example - 850
* has a value zero in SELECT). Thus the names of the items in
* SELECT define the set of allowed alternative prefixes, and the
* values indicate which one of these alternatives is to be used
* (the first one with non-zero value). [!]
* SORT = _LOGICAL (Read)
* If TRUE then sort the listed parameters in to alphabetical order.
* Otherwise, retain the order they have in the supplied
* configuration. Only used if a null (!) value is supplied for
* Parameter NAME. [FALSE]
* VALUE = LITERAL (Write)
* The value of the configuration parameter, or "<***>" if the
* parameter has no value in CONFIG and DEFAULTS.
* Examples:
* configecho m81 ^myconf
* Report the value of configuration parameter "m81" defined within
* the file "myconf". If the file does not contain a value for
* "m81", then "<***>" is displayed.
* configecho type ^myconf select="m57=0,m31=1,m103=0"
* Report the value of configuration parameter "type" defined within
* the file "myconf". If the file does not contain a value for
* "type", then the value of "m31.type" will be reported instead. If
* neither is present, then "<***>" is displayed.
* configecho flt.filt_edge_largescale \
* config=^/star/share/smurf/dimmconfig.lis \
* defaults=/star/bin/smurf/smurf_makemap.def \
* select="450=1,850=0"
* Report the value of configuration parameter "flt.filt_edge_largescale"
* defined within the file "/star/share/smurf/dimmconfig.lis", using
* defaults from the file "/star/bin/smurf/smurf_makemap.def". If
* dimmconfig.lis does not contain a value for "flt.filt_edge_largescale"
* then it is searched for "450.flt.filt_edge_largescale" instead. An
* error is reported if dimmconfig.lis contains values for any
* items that are not defined in smurf_makemap.def.
* configecho ndf=omc1 config=^/star/share/smurf/dimmconfig.lis \
* defaults=/star/bin/smurf/smurf_makemap.def \
* application=makemap name=! sort select="450=0,850=1"
* Show how the configuration used to generate the 850um map
* of OMC1 differs from the basic dimmconfig.lis file.
* Copyright:
* Copyright (C) 2012-3 Science & Technology Facilities Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either Version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
* Authors:
* DSB: David S. Berry
* GSB: Graham S. Bell
* {enter_new_authors_here}
* History:
* 10-DEC-2012 (DSB):
* Original version.
* 6-FEB-2013 (DSB):
* Added parameter DEFVAL.
* 11-FEB-2013 (DSB):
* Added parameter SORT and allow all parameters to be listed by
* providing a null value for NAME.
* 11-FEB-2013 (GSB):
* Added ability to read configuration from history entries.
* 13-FEB-2013 (DSB):
* Nullify AST object pointers when the objects are annulled,
* to avoid re-use of dead pointers.
* 14-FEB-2013 (DSB):
* Allow the SELECT feature to be used even if no DEFAULTS file is
* supplied (see the new entry in the "Examples:" section).
* 15-FEB-2013 (DSB):
* Expand the prologue docs, and use NULL in place of zero for pointers.
* 22-FEB-2013 (DSB):
* Guard against seg fault in HistoryKeymap when the NDF does
* not contain the required CONFIG entry in the History
* component.
* {enter_further_changes_here}
*-
*/
GENPTR_INTEGER(STATUS)
/* Local Variables: */
AstKeyMap *keymap2;
AstKeyMap *keymap;
Grp *grp = NULL;
char *dot;
char *pname;
char defs[250];
char defval[250];
char name[250];
const char *value;
const char *historyValue = NULL;
int showall;
int sort;
size_t size;
int indf = 0;
int nrec;
int i;
char application[NDF__SZAPP];
char applicationi[NDF__SZAPP];
/* Abort if an error has already occurred. */
if( *STATUS != SAI__OK ) return;
/* Begin an AST context */
astBegin;
/* Get the value to return if no value can be obtained for the named
parameter, of it it has a value of <undef>. */
parGet0c( "DEFVAL", defval, sizeof(defval), STATUS );
/* Get any defaults file, annuling the error if null (!) is supplied. */
if( *STATUS == SAI__OK ) {
parGet0c( "DEFAULTS", defs, sizeof(defs), STATUS );
if( *STATUS == PAR__NULL ) {
errAnnul( STATUS );
defs[0] = 0;
}
}
/* Get the NDF identifier if requested. */
ndfBegin();
if (*STATUS == SAI__OK) {
ndfAssoc("NDF", "READ", &indf, STATUS);
if (*STATUS == PAR__NULL) {
errAnnul(STATUS);
indf = 0;
}
else {
parGet0c("APPLICATION", application, sizeof(application), STATUS);
/* Check now for error because the block below allowing an undefined
* CONFIG clears this status otherwise. */
if (*STATUS != SAI__OK) goto L999;
}
}
/* See if any alternate keyword prefixes are allowed, and if so determine
which of the alternatices is to be displayed. */
kpg1Gtgrp( "SELECT", &grp, &size, STATUS );
if( *STATUS == PAR__NULL ) {
grpDelet( &grp, STATUS );
errAnnul( STATUS );
keymap2 = NULL;
} else {
kpg1Kymap( grp, &keymap2, STATUS );
grpDelet( &grp, STATUS );
}
/* Create a KeyMap holding the selected alternative for each keyword, and
also supply defaults for any missing values (if a defaults file was
supplied by the user). */
keymap = kpg1Config( "CONFIG", defs[0]?defs:NULL, keymap2, 0, STATUS );
/* Allow it to be NULL if we're reading an NDF because we'll replace
keymap with historyConfig later if necessary. */
if( indf && *STATUS == PAR__NULL ) {
errAnnul(STATUS);
keymap = NULL;
}
/* Abort if an error has occurred. */
if( *STATUS != SAI__OK ) goto L999;
/* Get the name of the required parameter, and convert to upper case (if
supplied). If not supplied, set a flag indicating that all parameters
should be displayed. */
parGet0c( "NAME", name, sizeof(name), STATUS );
if( *STATUS == PAR__NULL ) {
errAnnul( STATUS );
showall = 1;
} else {
showall = 0;
astChrCase( NULL, name, 1, 0 );
}
/* Attempt to find the NDF's corresponding history record. */
if (indf && *STATUS == SAI__OK) {
ndfHnrec(indf, &nrec, STATUS);
for (i = 0; i < nrec; i ++) {
ndfHinfo(indf, "APPLICATION", i + 1, applicationi,
sizeof(applicationi), STATUS);
if (! strncasecmp(application, applicationi, strlen(application))) {
ndfHout(indf, i + 1, HistoryKeyMap, STATUS);
break;
}
}
if (*STATUS == SAI__OK && ! historyConfig) {
*STATUS = SAI__ERROR;
errRepf("CONFIGECHO_ERR", "CONFIGECHO: Failed to find %s "
"configuration in NDF history.", STATUS, application);
}
else if (! keymap) {
keymap = historyConfig;
historyConfig = NULL;
}
}
if( *STATUS == SAI__OK ) {
/* First deal with cases where we are displaying a single parameter
value. */
if( !showall ) {
/* Loop round each section of the name that ends with a dot. */
value = defval;
pname = name;
dot = strchr( pname, '.' );
while( dot && keymap ) {
/* Get a nested keymap with the name that occurs prior to the dot. If
found, use it in place of the parent keymap. */
pname[ dot - pname ] = 0;
if( astMapGet0A( keymap, pname, &keymap2 ) ) {
astAnnul( keymap );
keymap = keymap2;
} else {
keymap = astAnnul( keymap );
}
/* If historyConfig exists, do the same there. */
if (historyConfig) {
if (astMapGet0A(historyConfig, pname, &keymap2)) {
astAnnul(historyConfig);
historyConfig = keymap2;
}
else {
historyConfig = astAnnul(historyConfig);
}
}
/* Re-instate the original dot, and move on to find the next dot. */
pname[ dot - pname ] = '.';
pname = dot + 1;
dot = strchr( pname, '.' );
}
/* Ensure no error is reported if the parameter is not found in the
KeyMap. */
if( keymap ) {
astClear( keymap, "KeyError" );
/* Get the parameter value as a string. */
astMapGet0C( keymap, pname, &value );
}
if (historyConfig) {
astClear(historyConfig, "KeyError");
astMapGet0C(historyConfig, pname, &historyValue);
/* In NDF history mode we only want to return a value if it
was found in the configuration from the history. */
if (historyValue) {
if (strcmp(value, historyValue)) {
msgOutf("", "+ %s", STATUS, historyValue);
}
else {
msgOutf("", "- %s", STATUS, historyValue);
}
parPut0c("VALUE", historyValue, STATUS);
}
}
else {
/* Display it. */
msgOut( "", value, STATUS );
/* Write it to the output parameter. */
parPut0c( "VALUE", value, STATUS );
}
/* Now deal with cases were we are displaying all parameter values. */
} else {
/* See if the values should be sorted. */
parGet0l( "SORT", &sort, STATUS );
/* Display them. */
if (historyConfig) {
DisplayKeyMap( historyConfig , sort, "", keymap, STATUS );
}
else {
DisplayKeyMap( keymap, sort, "", NULL, STATUS );
}
}
}
/* Tidy up. */
L999:;
/* End the AST context */
astEnd;
/* Close the NDF if open. */
ndfEnd(STATUS);
/* If an error has occurred, issue another error report identifying the
program which has failed (i.e. this one). */
if( *STATUS != SAI__OK ) {
errRep( "CONFIGECHO_ERR", "CONFIGECHO: Failed to echo configuration "
"parameters.", STATUS );
}
}
