void smf_get_extpar( AstKeyMap *keymap, smf_tausrc *tausrc,
smf_extmeth *extmeth, int *import, int *status ) {
const char *tempstr = NULL;
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain parameters from keymap when non-NULL pointers given */
if( import ) {
if( ! astMapGet0I( keymap, "IMPORT", import ) ) {
*import = 0;
}
}
if( tausrc ) {
if( astMapGet0C( keymap, "TAUSRC", &tempstr ) ) {
switch( toupper(tempstr[0]) ) {
case 'A': /* AUTO */
*tausrc = SMF__TAUSRC_AUTO;
break;
case 'C': /* CSO* */
/* Need to compare CSOTAU to CSOFIT */
switch( toupper(tempstr[3]) ) {
case 'T':
*tausrc = SMF__TAUSRC_CSOTAU;
break;
case 'F':
*tausrc = SMF__TAUSRC_CSOFIT;
break;
default:
*tausrc = SMF__TAUSRC_NULL;
}
break;
case 'F': /* FILTERTAU */
*tausrc = SMF__TAUSRC_TAU;
break;
case 'W': /* WVM */
/* Need to compare WVMRAW to WVMFIT */
switch (toupper(tempstr[3])) {
case 'R':
*tausrc = SMF__TAUSRC_WVMRAW;
break;
case 'F':
*tausrc = SMF__TAUSRC_WVMFIT;
break;
default:
*tausrc = SMF__TAUSRC_NULL;
}
break;
default:
*tausrc = SMF__TAUSRC_NULL;
}
} else {
*tausrc = SMF__TAUSRC_NULL;
}
}
if( extmeth ) {
if( astMapGet0C( keymap, "TAUMETHOD", &tempstr ) ) {
switch( toupper(tempstr[0]) ) {
case 'A':
*extmeth = SMF__EXTMETH_ADAPT;
break;
case 'Q':
*extmeth = SMF__EXTMETH_SINGLE;
break;
case 'F':
*extmeth = SMF__EXTMETH_FULL;
break;
default:
*extmeth = SMF__EXTMETH_NONE;
}
} else {
*extmeth = SMF__EXTMETH_NONE;
}
}
}
Hero * Hero::constructFromFitsFile(const QString &fname)
{
FitsParser parser;
bool parsedOk = parser.loadFile( FitsFileLocation::fromLocal( fname));
if( ! parsedOk) {
dbg(1) << "Parser failed to load " << fname;
Hero * heroPtr = new Hero;
heroPtr-> addError( "FitsParser failed to load the file");
return heroPtr;
}
// alias hdr
auto & hdr = parser.getHeaderInfo().headerLines;
Hero * heroPtr = new Hero;
Hero & hero = * heroPtr;
AstErrorGuard guard( heroPtr);
AstGCGuard gcGuard;
// set naxes in case AST fails to read this file
hero.m_ast.naxes = parser.getHeaderInfo().naxis;
// set up bunit
{
hero.m_bunit = parser.getHeaderInfo().bunit;
}
// and the nicer version of bunit
{
QString u = hero.m_bunit.simplified();
if( u.toLower() == "kelvin") {
hero.m_bunitNiceHtml = "K";
}
else {
hero.m_bunitNiceHtml = u;
}
}
// Create a FitsChan and feed it the fits header
AstFitsChan *fitschan;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat-zero-length"
fitschan = astFitsChan( NULL, NULL, "" );
#pragma GCC diagnostic pop
std::cout << "astOK = " << astOK << "\n";
// feed the header lines one by one and check for errors
for( const QString & s : hdr) {
std::string stdstr = s.toStdString();
astPutFits( fitschan, stdstr.c_str(), 1);
if( ! astOK) {
astClearStatus;
QString ss = s.trimmed();
std::cout << "Skipping bad card: " << ss << "\n";
hero.addError( "Skipping card: " + ss);
}
}
// reposition to the beginning of the channel (ast thing, it's required, hmmmkey)
astClear( fitschan, "Card" );
std::cout << "astOK = " << astOK << "\n";
std::cout << "Here\n";
auto encoding = AstWrappers::getC( fitschan, "Encoding" );
std::cout << "Encoding = " << encoding << "\n";
// do we have warnings?
AstKeyMap * warnings = static_cast<AstKeyMap *>( astWarnings( fitschan));
if( warnings && astOK ) {
std::cout << "Warnings:\n";
int iwarn = 1;
while( astOK ) {
std::string key = QString("Warning_%1").arg( iwarn).toStdString();
const char * message = nullptr;
if( astMapGet0C( warnings, key.c_str(), & message ) ) {
printf( "\n- %s\n", message );
hero.addError( QString( "Warning: %1").arg( message));
} else {
break;
}
}
}
else {
std::cout << "No warnings\n";
}
// create a frameset for this file
AstFrameSet * wcsinfo = static_cast<AstFrameSet *> ( astRead( fitschan ));
std::cout << "astOK = " << astOK << "\n";
if ( ! astOK ) {
std::cout << "astOK is not ok\n";
hero.addError( "astRead failed");
astClearStatus;
return heroPtr;
}
else if ( wcsinfo == AST__NULL ) {
hero.addError( "No WCS found in the fits file");
std::cout << "No WCS found\n";
return heroPtr;
}
else if ( AstWrappers::getC( wcsinfo, "Class" ) != "FrameSet") {
std::cout << "Some other weird error occured\n";
hero.addError( "AstLib returned non-frame-set");
return heroPtr;
}
// frame was read in OK, save it
hero.m_ast.origWcsInfo = wcsinfo;
astExempt( hero.m_ast.origWcsInfo);
hero.m_ast.currWcsInfo = astClone( hero.m_ast.origWcsInfo);
astExempt( hero.m_ast.currWcsInfo);
astShow( wcsinfo);
// extract the current sky system
// TODO: this assumes axis1 is a skycs
QString skysys = AstWrappers::getC( wcsinfo, "System(1)");
hero.m_currentSkyCs = string2skycs( skysys);
hero.m_originalSkyCs = hero.m_currentSkyCs;
// extract the labels/etc for axes
hero.m_ast.naxes = AstWrappers::getI( wcsinfo, "Naxes" );
hero.parseAxesInfo();
return heroPtr;
}
int main( int argc, char **argv ){
/* Local variables: */
AstBox *pixbox;
AstFitsChan *fchan;
AstFrame *pixfrm;
AstFrame *wcsfrm;
AstFrameSet *frameset;
AstKeyMap *warnings;
AstMapping *pix2wcs;
AstObject *object;
AstRegion *wcsbox;
AstStcsChan *schan;
FILE *fd;
char key[ 15 ];
char keyword[ 9 ];
const char *message;
double p1[ MAX_AXES ];
double p2[ MAX_AXES ];
int axis;
int iwarn;
int naxis;
int status;
/* Initialised the returned system status to indicate success. */
status = 0;
/* Check a file was specified on the command line, and attempt to open it
for read access. */
if( argc < 2 ) {
printf( "Usage: stcschan-demo2 <header-file>\n" );
status = 1;
} else {
fd = fopen( argv[ 1 ], "r" );
if( !fd ) {
printf("Failed to open input file '%s'.\n", argv[ 1 ] );
status = 1;
}
}
/* If a disk file was opened successfully... */
if( !status ) {
/* Start an AST object context. This means we do not need to annull
each AST Object individually. Instead, all Objects created within
this context will be annulled automatically by the corresponding
invocation of astEnd. */
astBegin;
/* Create a FitsChan. This is the object that converts external FITS
headers into corresponding AST Objects. Tell it to use the "source"
function for obtaining lines of text from the disk file. */
fchan = astFitsChan( source, NULL, " " );
/* Associate the descriptor for the input disk file with the StcsChan.
This makes it available to the "source" function. Since this
application is single threaded, we could instead have made "fd" a
global variable, but the ChannelData facility is used here to illustrate
how to pass data to a source or sink function safely in a multi-threaded
application. */
astPutChannelData( fchan, fd );
/* Attempt to read the FITS heades and convert them into an AST FrameSet. */
object = astRead( fchan );
/* The astRead function is a generic function and so returns a generic
AstObject pointer. Check an Object was created successfully. */
if( !object ) {
printf( "Failed to read an AST Object from file '%s'.\n",
argv[ 1 ] );
status = 1;
/* Now check that the object read is actually an AST FrameSet, rather than
some other class of AST Object. */
} else if( !astIsAFrameSet( object ) ) {
printf( "Expected a FrameSet but read a %s from file '%s'.\n",
astGetC( object, "Class" ), argv[ 1 ] );
status = 1;
/* We now know we have a FrameSet so it is safe to use the pointer
returned by astRead as a FrameSet pointer. Do the cast now to avoid
repeated casting in future. */
} else {
frameset = (AstFrameSet *) object;
/* Get a pointer to the Frame that describes the attributes of the FITS
world coordinate system. This is the current Frame in the FrameSet
read from the FITS headers. */
wcsfrm = astGetFrame( frameset, AST__CURRENT );
/* Get a pointer to the Frame that describes the attributes of the FITS
pixel coordinate system. This is the base Frame in the FrameSet
read from the FITS headers. */
pixfrm = astGetFrame( frameset, AST__BASE );
/* Get the Mapping that transforms pixel positions into WCS positions.
The is the Mapping from base to current Frame in the FrameSet read
from the FITS headers. */
pix2wcs = astGetMapping( frameset, AST__BASE, AST__CURRENT );
/* Get the number of axes in ther pixel Frame. */
naxis = astGetI( pixfrm, "Naxes" );
/* For each pixel axis, form the name of the corresponding NAXISi
keyword. */
for( axis = 0; axis < naxis; axis++ ) {
sprintf( keyword, "NAXIS%d", axis + 1 );
/* Store the pixel coordinate on the current axis at the lower left corner
of the first pixel. */
p1[ axis ] = 0.5;
/* Get the NAXISi value for the current axis from the FITS header, and
store it in array "p2". Report an error if NAXISi is not found. */
if( !astGetFitsF( fchan, keyword, p2 + axis ) ){
printf("Keyword '%s' not found in header\n", keyword );
status = 1;
break;
/* If it is found, modify "p2" so that it holds the pixel coordinate on
the current axis at the upper right corner of the last pixel. */
} else {
p2[ axis ] += 0.5;
}
}
}
/* If all has gone well, create an AST Region (a Box) describing the
rectangular region of pixel coordinates covered by the pixel array. */
if( !status ) {
pixbox = astBox( pixfrm, 1, p1, p2, NULL, " " );
/* Map this box into the FITS world coordinate system. The Mapping is
specified by "pix2wcs", and the attributes of the resulting axes is
described by "wcsfrm". */
wcsbox = astMapRegion( pixbox, pix2wcs, wcsfrm );
/* Create an StcsChan. This is the object that converts (either way)
between external STC-S descriptions and their corresponding AST Objects.
Tell it to use the "source" function for obtaining lines of text from
the disk file. Also tell it to store all warnings generated by the
conversion for later use. Other attributes of the StcsChan class retain
their default values. */
schan = astStcsChan( NULL, NULL, "ReportLevel=3" );
/* Attempt to write out the Region describing the pixel array (in WCS)
as an STC-S description. Report an error if this fails. */
if( ! astWrite( schan, wcsbox ) && astOK ) {
printf( "Failed to convert the Region into an STC-S "
"description.\n" );
}
}
/* We asked the StcsChan to record any warnings that were generated
whilst converting the AST Region into a corresponding STC-S description.
We now see if any such warnings were generated by the earlier call to
astWrite. */
warnings = astWarnings( schan );
/* If any warnings were generated, and if no other error has occurred so
far, display the warnings. */
if( warnings && !status && astOK ) {
printf( "\nThe following warnings were issued:\n" );
/* The warnings are stored in an AST KeyMap (a sort of hashmap). Each
warning message is associated with a key of the form "Warning_1",
"Warning_2", etc. Loop round successive keys, obtaining a value for
each key from the warnings KeyMap, and displaying it. */
iwarn = 1;
while( astOK ) {
sprintf( key, "Warning_%d", iwarn++ );
if( astMapGet0C( warnings, key, &message ) ) {
printf( "\n- %s\n", message );
} else {
break;
}
}
}
/* End the AST Object context. All Objects created since the
corresponding invocation of astbegin will be annulled automatically. */
astEnd;
/* Close the disk file. */
(void) fclose( fd );
}
/* If an error occurred in the AST library, set the retiurns system
status non-zero. */
if( !astOK ) status = 1;
return status;
}
void
smf_flat_params( const smfData * refdata, const char resistpar[],
const char methpar[], const char orderpar[], const char snrminpar[],
double * refohms, double **resistance, int * outrows,
int * outcols, smf_flatmeth *flatmeth,
int * order, double * snrmin, smfData ** heateff,
int * status ) {
dim_t datarows = 0; /* Number of rows in refdata */
dim_t datacols = 0; /* Number of columns in refdata */
size_t j = 0; /* Counter, index */
char method[SC2STORE_FLATLEN]; /* flatfield method string */
size_t nbols; /* Number of bolometers */
double refohmsval = 0.0; /* Internal version of refohms */
AstKeyMap * resmap = NULL; /* Resistor map */
AstKeyMap * subarrays = NULL; /* Subarray lookup table */
char thissub[32]; /* This sub-instrument string */
if (resistance) *resistance = NULL;
if (*status != SAI__OK) return;
if (!refdata) {
*status = SAI__ERROR;
errRep( "", "Must provide reference data file to calculate flatfield parameters"
" (possible programming error)", status );
return;
}
/* Based on refdata we now need to calculate the default reference
resistance and retrieve the correct heater efficiency file for each array.
We need the unique subarray string so that we can set up a look up keymap.
There is no code in SMURF to return all the known subarrays but
we need to know all the options in order to use kpg1Config. */
subarrays = astKeyMap( " " );
astMapPut0I( subarrays, "CG450MK2_M0907D0501", 0, NULL );
astMapPut0I( subarrays, "CG850MK2_M0904D0503", 0, NULL );
astMapPut0I( subarrays, "SG850_M0906D1005", 0, NULL );
astMapPut0I( subarrays, "SG850_M1002D1006", 0, NULL );
astMapPut0I( subarrays, "SG850_M1005D1007", 0, NULL );
astMapPut0I( subarrays, "SG850_M1003D1004", 0, NULL );
astMapPut0I( subarrays, "SG450_M1004D1000", 0, NULL );
astMapPut0I( subarrays, "SG450_M1007D1002", 0, NULL );
astMapPut0I( subarrays, "SG450_M1006D1003", 0, NULL );
astMapPut0I( subarrays, "SG450_M1009D1008", 0, NULL );
/* and indicate which subarray we are interested in (uppercased) */
smf_fits_getS( refdata->hdr, "ARRAYID", thissub, sizeof(thissub), status );
{ /* need to uppercase */
size_t l = strlen(thissub);
for (j=0;j<l;j++) {
thissub[j] = toupper(thissub[j]);
}
}
astMapPut0I( subarrays, thissub, 1, NULL );
/* Read the config file */
resmap = kpg1Config( resistpar, "$SMURF_DIR/smurf_calcflat.def",
subarrays, 1, status );
subarrays = astAnnul( subarrays );
if (*status != SAI__OK) goto CLEANUP;
/* Read the reference resistance */
astMapGet0D( resmap, "REFRES", &refohmsval );
if (refohms && *status == SAI__OK) {
*refohms = refohmsval;
msgOutiff(MSG__VERB, "",
"Read reference resistance for subarray %s of %g ohms\n",
status, thissub, *refohms );
}
/* We no longer want to read per-bolometer resistor values from the
config file. To retain backwards compatibility with the current
implementation of smf_flat_standardpow we simply fill the
per-bol resistance array with the reference resistance which
effectively disables smf_flat_standardpow */
smf_get_dims( refdata, &datarows, &datacols, NULL, NULL, NULL, NULL, NULL, status );
nbols = datacols * datarows;
if (*status == SAI__OK && resistance ) {
*resistance = astMalloc( nbols*sizeof(**resistance) );
for (j = 0; j < (size_t)nbols; j++) {
(*resistance)[j] = refohmsval;
}
}
/* Get the heater efficiency file */
if (heateff && astMapHasKey( resmap, "HEATEFF" ) ) {
const char * heateffstr = NULL;
if (astMapGet0C( resmap, "HEATEFF", &heateffstr )) {
Grp * heateffgrp = NULL;
smfData * heatefftmp = NULL;
heateffgrp = grpNew( "heateff", status );
grpPut1( heateffgrp, heateffstr, 0, status );
smf_open_file( NULL, heateffgrp, 1, "READ", SMF__NOTTSERIES|SMF__NOFIX_METADATA, &heatefftmp, status );
/* Divorce the smfData from the underlying file. This file stays open for the entire
duration of the data processing and can some times lead to issues when we attempt
to close it an hour after we opened it (it's usually on an NFS disk) */
if (*status == SAI__OK) {
*heateff = smf_deepcopy_smfData( NULL, heatefftmp, 0,
SMF__NOCREATE_FILE | SMF__NOCREATE_FTS |
SMF__NOCREATE_DA,
0, 0, status );
smf_close_file(NULL, &heatefftmp, status);
}
/* Check the dimensions */
if (*status == SAI__OK) {
dim_t heatrows = 0;
dim_t heatcols = 0;
smf_get_dims( *heateff, &heatrows, &heatcols, NULL, NULL, NULL, NULL, NULL, status );
if (*status == SAI__OK) {
if ( datarows != heatrows || datacols != heatcols ) {
*status = SAI__ERROR;
errRepf( "", "Dimensions of heater efficiency file %s are (%zu, %zu)"
" but flatfield has dimensions (%zu, %zu)",
status, heateffstr, (size_t)heatrows, (size_t)heatcols,
(size_t)datarows, (size_t)datacols);
}
}
if (*status == SAI__OK) {
smf_dtype_check_fatal( *heateff, NULL, SMF__DOUBLE, status );
if (*status == SMF__BDTYP) {
errRepf("", "Heater efficiency data in %s should be double precision",
status, heateffstr);
}
}
if (*status == SAI__OK) {
char heateffarrid[32];
smf_fits_getS( refdata->hdr, "ARRAYID", heateffarrid, sizeof(heateffarrid), status );
if (*status != SAI__OK) errAnnul( status );
if (strcasecmp( thissub, heateffarrid ) != 0 ) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRepf("", "Subarray associated with heater efficiency image (%s)"
" does not match that of the data to be flatfielded (%s)",
status, heateffarrid, thissub );
}
}
}
}
if (heateffgrp) grpDelet( &heateffgrp, status );
}
}
if (methpar && flatmeth) {
/* See if we want to use TABLE or POLYNOMIAL mode */
parChoic( methpar, "POLYNOMIAL", "POLYNOMIAL, TABLE", 1,
method, sizeof(method), status );
*flatmeth = smf_flat_methcode( method, status );
if (*flatmeth == SMF__FLATMETH_POLY) {
/* need an order for the polynomial */
if (order && orderpar) {
parGdr0i( orderpar, 1, 1, 3, 1, order, status );
/* and if the order is 1 then we can ask for the snr min */
if (snrminpar && *order == 1) {
parGet0d( snrminpar, snrmin, status );
}
}
} else {
/* need an snr min for table mode responsivities */
if (snrminpar) parGet0d( snrminpar, snrmin, status );
}
}
if (outrows) *outrows = datarows;
if (outcols) *outcols = datacols;
CLEANUP:
resmap = astAnnul( resmap );
if (*status != SAI__OK) {
if (resistance && *resistance) *resistance = astFree( *resistance );
if (heateff && *heateff) smf_close_file( NULL, heateff, status );
}
return;
}
void smf_find_science(const Grp * ingrp, Grp **outgrp, int reverttodark,
Grp **darkgrp, Grp **flatgrp, int reducedark,
int calcflat, smf_dtype darktype, smfArray ** darks,
smfArray **fflats, AstKeyMap ** heateffmap,
double * meanstep, int * status ) {
smfSortInfo *alldarks; /* array of sort structs for darks */
smfSortInfo *allfflats; /* array of fast flat info */
Grp * dgrp = NULL; /* Internal dark group */
double duration_darks = 0.0; /* total duration of all darks */
double duration_sci = 0.0; /* Duration of all science observations */
size_t dkcount = 0; /* Dark counter */
size_t ffcount = 0; /* Fast flat counter */
Grp * fgrp = NULL; /* Fast flat group */
size_t i; /* loop counter */
smfData *infile = NULL; /* input file */
size_t insize; /* number of input files */
size_t nsteps_dark = 0; /* Total number of steps for darks */
size_t nsteps_sci = 0; /* Total number of steps for science */
AstKeyMap * heatermap = NULL; /* Heater efficiency map */
AstKeyMap * obsmap = NULL; /* Info from all observations */
AstKeyMap * objmap = NULL; /* All the object names used */
AstKeyMap * scimap = NULL; /* All non-flat obs indexed by unique key */
Grp *ogrp = NULL; /* local copy of output group */
size_t sccount = 0; /* Number of accepted science files */
struct timeval tv1; /* Timer */
struct timeval tv2; /* Timer */
if (meanstep) *meanstep = VAL__BADD;
if (outgrp) *outgrp = NULL;
if (darkgrp) *darkgrp = NULL;
if (darks) *darks = NULL;
if (fflats) *fflats = NULL;
if (heateffmap) *heateffmap = NULL;
if (*status != SAI__OK) return;
/* Sanity check to make sure we return some information */
if ( outgrp == NULL && darkgrp == NULL && darks == NULL && fflats == NULL) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": Must have some non-NULL arguments"
" (possible programming error)", status);
return;
}
/* Start a timer to see how long this takes */
smf_timerinit( &tv1, &tv2, status );
/* Create new group for output files */
ogrp = smf_grp_new( ingrp, "Science", status );
/* and a new group for darks */
dgrp = smf_grp_new( ingrp, "DarkFiles", status );
/* and for fast flats */
fgrp = smf_grp_new( ingrp, "FastFlats", status );
/* and also create a keymap for the observation description */
obsmap = astKeyMap( "KeyError=1" );
/* and an object map */
objmap = astKeyMap( "KeyError=1" );
/* This keymap contains the sequence counters for each related
subarray/obsidss/heater/shutter combination and is used to decide
if a bad flat is relevant */
scimap = astKeyMap( "KeyError=1,KeyCase=0" );
/* This keymap is used to contain relevant heater efficiency data */
heatermap = astKeyMap( "KeyError=1,KeyCase=0" );
/* Work out how many input files we have and allocate sufficient sorting
space */
insize = grpGrpsz( ingrp, status );
alldarks = astCalloc( insize, sizeof(*alldarks) );
allfflats = astCalloc( insize, sizeof(*allfflats) );
/* check each file in turn */
for (i = 1; i <= insize; i++) {
int seqcount = 0;
char keystr[100]; /* Key for scimap entry */
/* open the file but just to get the header */
smf_open_file( ingrp, i, "READ", SMF__NOCREATE_DATA, &infile, status );
if (*status != SAI__OK) break;
/* Fill in the keymap with observation details */
smf_obsmap_fill( infile, obsmap, objmap, status );
/* Find the heater efficiency map if required */
if (*status == SAI__OK && heateffmap) {
char arrayidstr[32];
smf_fits_getS( infile->hdr, "ARRAYID", arrayidstr, sizeof(arrayidstr),
status );
if (!astMapHasKey( heatermap, arrayidstr ) ) {
smfData * heateff = NULL;
dim_t nbolos = 0;
smf_flat_params( infile, "RESIST", NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, &heateff, status );
smf_get_dims( heateff, NULL, NULL, &nbolos, NULL, NULL, NULL, NULL,
status );
if (heateff) astMapPut0P( heatermap, arrayidstr, heateff, NULL );
}
}
/* Get the sequence counter for the file. We do not worry about
duplicate sequence counters (at the moment) */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* The key identifying this subarray/obsidss/heater/shutter combo */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
if (smf_isdark( infile, status )) {
/* Store the sorting information */
dkcount = smf__addto_sortinfo( infile, alldarks, i, dkcount, "Dark", status );
smf__addto_durations( infile, &duration_darks, &nsteps_dark, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
} else {
/* compare sequence type with observation type and drop it (for now)
if they differ */
if ( infile->hdr->obstype == infile->hdr->seqtype ) {
/* Sanity check the header for corruption. Compare RTS_NUM with SEQSTART
and SEQEND. The first RTS_NUM must either be SEQSTART or else between
SEQSTART and SEQEND (if someone has giving us a section) */
int seqstart = 0;
int seqend = 0;
int firstnum = 0;
JCMTState *tmpState = NULL;
smf_getfitsi( infile->hdr, "SEQSTART", &seqstart, status );
smf_getfitsi( infile->hdr, "SEQEND", &seqend, status );
tmpState = infile->hdr->allState;
if( tmpState ) {
firstnum = (tmpState[0]).rts_num;
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
if ( firstnum >= seqstart && firstnum <= seqend ) {
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif(MSG__DEBUG, " ", "Non-dark file: ^F",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
} else {
msgOutif( MSG__QUIET, "",
"File ^F has a corrupt FITS header. Ignoring it.",
status );
}
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif( MSG__DEBUG, " ",
"File ^F lacks JCMTState: assuming it is non-dark",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
}
} else if (infile->hdr->seqtype == SMF__TYP_FASTFLAT ) {
ffcount = smf__addto_sortinfo( infile, allfflats, i, ffcount, "Fast flat", status );
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
msgOutif(MSG__DEBUG, " ", "Sequence type mismatch with observation type: ^F",status);
}
}
/* close the file */
smf_close_file( &infile, status );
}
/* Store output group in return variable or else free it */
if (outgrp) {
*outgrp = ogrp;
} else {
grpDelet( &ogrp, status );
}
/* process flatfields if necessary */
if (ffcount > 0 && fflats ) {
smfArray * array = NULL;
/* sort flats into order */
qsort( allfflats, ffcount, sizeof(*allfflats), smf_sort_bydouble);
if (fflats) array = smf_create_smfArray( status );
/* now open the flats and store them if requested */
if (*status == SAI__OK && array && ffcount) {
size_t start_ffcount = ffcount;
AstKeyMap * flatmap = NULL;
if (calcflat) {
/* Use AgeUp so that we get the keys out in the sorted order
that allfflats used */
flatmap = astKeyMap( "KeyCase=0,KeyError=1,SortBy=AgeDown" );
}
/* Read each flatfield. Calculate a responsivity image and a flatfield
solution. Store these in a keymap along with related information
which is itself stored in a keymap indexed by a string made of
OBSIDSS, reference heater value, shutter and subarray.
*/
for (i = 0; i < start_ffcount; i++ ) {
size_t ori_index = (allfflats[i]).index;
smfData * outfile = NULL;
char keystr[100];
AstKeyMap * infomap = astKeyMap( "KeyError=1" );
int oplen = 0;
char thisfile[MSG__SZMSG];
int seqcount = 0;
/* read filename from group */
infile = NULL;
smf_open_file( ingrp, ori_index, "READ", 0, &infile, status );
if ( *status != SAI__OK ) {
/* This should not happen because we have already opened
the file. If it does happen we abort with error. */
if (infile) smf_close_file( &infile, status );
break;
}
/* Calculate the key for this observation */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
/* Get the file name for error messages */
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>" );
msgLoad( "", "^F", thisfile, sizeof(thisfile), &oplen, status );
/* And the sequence counter to link against science observations */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* Prefill infomap */
astMapPut0C( infomap, "FILENAME", thisfile, "");
astMapPut0I( infomap, "SEQCOUNT", seqcount, "");
/* Collapse it */
if (*status == SAI__OK) {
smf_flat_fastflat( infile, &outfile, status );
if (*status == SMF__BADFLAT) {
errFlush( status );
if (calcflat) {
/* Need to generate an outfile like smf_flat_fastflat
and one heater setting will force smf_flat_calcflat to fail */
smf_flat_malloc( 1, infile, NULL, &outfile, status );
} else {
if (outfile) smf_close_file( &outfile, status );
if (infile) smf_close_file( &infile, status );
infomap = astAnnul( infomap );
ffcount--;
continue;
}
}
}
if (outfile && *status == SAI__OK) {
smf_close_file( &infile, status );
infile = outfile;
if (calcflat) {
size_t ngood = 0;
smfData * curresp = NULL;
int utdate;
if (*status == SAI__OK) {
ngood = smf_flat_calcflat( MSG__VERB, NULL, "RESIST",
"FLATMETH", "FLATORDER", NULL, "RESPMASK",
"FLATSNR", NULL, infile, &curresp, status );
if (*status != SAI__OK) {
/* if we failed to calculate a flatfield we continue but force the
flatfield to be completely bad. This will force the science data associated
with the flatfield to be correctly blanked. We do not annul though
if we have a SUBPAR error telling us that we have failed to define
our parameters properly. */
if (*status != SUBPAR__NOPAR) errAnnul(status);
/* parameters of flatfield */
ngood = 0;
/* Generate a blank flatfield and blank responsivity image */
smf_flat_badflat( infile, &curresp, status );
}
/* Retrieve the UT date so we can decide whether to compare
flatfields */
smf_getfitsi( infile->hdr, "UTDATE", &utdate, status );
/* Store the responsivity data for later on and the processed
flatfield until we have vetted it */
astMapPut0P( infomap, "CALCFLAT", infile, "" );
astMapPut0P( infomap, "RESP", curresp, "" );
astMapPut0I( infomap, "UTDATE", utdate, "" );
astMapPut0I( infomap, "ISGOOD", 1, "" );
astMapPut0I( infomap, "NGOOD", ngood, "" );
astMapPut0I( infomap, "GRPINDEX", ori_index, "" );
astMapPut0I( infomap, "SMFTYP", infile->hdr->obstype, "" );
astMapPutElemA( flatmap, keystr, -1, infomap );
}
} else { /* if (calcflat) */
/* Store the collapsed flatfield - the processed flat is not stored here yet */
smf_addto_smfArray( array, infile, status );
/* Copy the group info */
ndgCpsup( ingrp, ori_index, fgrp, status );
}
} /* if (outfile) */
/* Annul the keymap (will be fine if it is has been stored in another keymap) */
infomap = astAnnul( infomap );
} /* End loop over flatfields */
/* Now we have to loop over the related flatfields to disable
bolometers that are not good and also decide whether we
need to set status to bad. */
if (*status == SAI__OK && calcflat ) {
size_t nkeys = astMapSize( flatmap );
for (i = 0; i < nkeys; i++ ) {
const char *key = astMapKey( flatmap, i );
int nf = 0;
AstKeyMap ** kmaps = NULL;
int nelem = astMapLength( flatmap, key );
kmaps = astMalloc( sizeof(*kmaps) * nelem );
astMapGet1A( flatmap, key, nelem, &nelem, kmaps );
for ( nf = 0; nf < nelem && *status == SAI__OK; nf++ ) {
AstKeyMap * infomap = kmaps[nf];
int isgood = 0;
astMapGet0I( infomap, "ISGOOD", &isgood );
if (isgood) {
/* The flatfield worked */
size_t ngood = 0;
int itemp;
int utdate = 0;
int ratioFlats = 0;
/* Get the UT date - we do not compare flatfields after
the time we enabled heater tracking at each sequence. */
astMapGet0I( infomap, "UTDATE", &utdate );
/* Get the number of good bolometers at this point */
astMapGet0I( infomap, "NGOOD", &itemp );
ngood = itemp;
/* Decide if we want to do the ratio test. We default to
not doing it between 20110901 and 20120827 which is
the period when we did mini-heater tracks before each
flat. ! indicates that we choose based on date. */
if (*status == SAI__OK) {
parGet0l( "FLATUSENEXT", &ratioFlats, status );
if ( *status == PAR__NULL ) {
errAnnul( status );
if (utdate >= 20110901 || utdate <= 20120827 ) {
ratioFlats = 0;
} else {
ratioFlats = 1;
}
}
}
/* Can we compare with the next flatfield? */
if (ngood < SMF__MINSTATSAMP || !ratioFlats ) {
/* no point doing all the ratio checking for this */
} else if ( nelem - nf >= 2 ) {
AstKeyMap * nextmap = kmaps[nf+1];
const char *nextfname = NULL;
const char *fname = NULL;
smfData * curresp = NULL;
smfData * nextresp = NULL;
smfData * curflat = NULL;
void *tmpvar = NULL;
size_t bol = 0;
smfData * ratio = NULL;
double *in1 = NULL;
double *in2 = NULL;
double mean = VAL__BADD;
size_t nbolo;
double *out = NULL;
double sigma = VAL__BADD;
float clips[] = { 5.0, 5.0 }; /* 5.0 sigma iterative clip */
size_t ngoodz = 0;
astMapGet0C( nextmap, "FILENAME", &nextfname );
astMapGet0C( infomap, "FILENAME", &fname );
/* Retrieve the responsivity images from the keymap */
astMapGet0P( infomap, "RESP", &tmpvar );
curresp = tmpvar;
astMapGet0P( nextmap, "RESP", &tmpvar );
nextresp = tmpvar;
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
nbolo = (curresp->dims)[0] * (curresp->dims)[1];
/* get some memory for the ratio if we have not already.
We could get some memory once assuming each flat has the
same number of bolometers... */
ratio = smf_deepcopy_smfData( curresp, 0, 0, 0, 0, status );
if( *status == SAI__OK ) {
/* divide: smf_divide_smfData ? */
in1 = (curresp->pntr)[0];
in2 = (nextresp->pntr)[0];
out = (ratio->pntr)[0];
for (bol=0; bol<nbolo;bol++) {
if ( in1[bol] != VAL__BADD && in1[bol] != 0.0 &&
in2[bol] != VAL__BADD && in2[bol] != 0.0 ) {
out[bol] = in1[bol] / in2[bol];
} else {
out[bol] = VAL__BADD;
}
}
}
/* find some statistics */
smf_clipped_stats1D( out, 2, clips, 1, nbolo, NULL, 0, 0, &mean,
&sigma, NULL, 0, &ngoodz, status );
if (*status == SMF__INSMP) {
errAnnul(status);
msgOutiff( MSG__QUIET, "",
"Flatfield ramp ratio of %s with %s had too few bolometers (%zu < %d).",
status, fname, nextfname, ngoodz, SMF__MINSTATSAMP );
ngood = ngoodz; /* Must be lower or equal to original ngood */
} else if (*status == SAI__OK && mean != VAL__BADD && sigma != VAL__BADD && curflat->da) {
/* Now flag the flatfield as bad for bolometers that have changed
more than n%. We expect the variation to be 1+/-a small bit */
const double pmrange = 0.10;
double thrlo = 1.0 - pmrange;
double thrhi = 1.0 + pmrange;
size_t nmasked = 0;
double *flatcal = curflat->da->flatcal;
msgOutiff( MSG__DEBUG, "", "Flatfield fast ramp ratio mean = %g +/- %g (%zu bolometers)",
status, mean, sigma, ngood);
/* we can just set the first slice of the flatcal to bad. That should
be enough to disable the entire bolometer. We have just read these
data so they should be in ICD order. */
for (bol=0; bol<nbolo;bol++) {
if ( out[bol] != VAL__BADD &&
(out[bol] < thrlo || out[bol] > thrhi ) ) {
flatcal[bol] = VAL__BADD;
nmasked++;
} else if ( in1[bol] != VAL__BADD && in2[bol] == VAL__BADD ) {
/* A bolometer is bad next time but good now so we must set it bad now */
flatcal[bol] = VAL__BADD;
nmasked++;
}
}
if ( nmasked > 0 ) {
msgOutiff( MSG__NORM, "", "Masked %zu bolometers in %s from unstable flatfield",
status, nmasked, fname );
/* update ngood to take into account the masking */
ngood -= nmasked;
}
}
smf_close_file( &ratio, status );
} /* End of flatfield responsivity comparison */
/* if we only have a few bolometers left we now consider this
a bad flat unless it is actually an engineering measurement
where expect some configurations to give zero bolometers */
if (ngood < SMF__MINSTATSAMP) {
const char *fname = NULL;
void * tmpvar = NULL;
int smftyp = 0;
smfData * curflat = NULL;
astMapGet0I( infomap, "SMFTYP", &smftyp );
astMapGet0C( infomap, "FILENAME", &fname );
if (smftyp != SMF__TYP_NEP) {
msgOutiff( MSG__QUIET, "",
"Flatfield %s has %zu good bolometer%s.%s",
status, fname, ngood, (ngood == 1 ? "" : "s"),
( ngood == 0 ? "" : " Keeping none.") );
isgood = 0;
/* Make sure that everything is blanked. */
if (ngood > 0) {
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
if (curflat && curflat->da) {
size_t bol;
size_t nbolo = (curflat->dims)[0] * (curflat->dims)[1];
double *flatcal = curflat->da->flatcal;
for (bol=0; bol<nbolo; bol++) {
/* Just need to set the first element to bad */
flatcal[bol] = VAL__BADD;
}
}
}
} else {
msgOutiff( MSG__NORM, "",
"Flatfield ramp file %s has %zu good bolometer%s. Eng mode.",
status, fname, ngood, (ngood == 1 ? "" : "s") );
}
}
/* We do not need the responsivity image again */
{
void *tmpvar = NULL;
smfData * resp = NULL;
astMapGet0P( infomap, "RESP", &tmpvar );
resp = tmpvar;
if (resp) smf_close_file( &resp, status );
astMapRemove( infomap, "RESP" );
}
} /* End of isgood comparison */
/* We are storing flats even if they failed. Let the downstream
software worry about it */
{
int ori_index;
smfData * flatfile = NULL;
void *tmpvar = NULL;
/* Store in the output group */
astMapGet0I( infomap, "GRPINDEX", &ori_index );
ndgCpsup( ingrp, ori_index, fgrp, status );
/* And store in the smfArray */
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
astMapRemove( infomap, "CALCFLAT" );
flatfile = tmpvar;
smf_addto_smfArray( array, flatfile, status );
}
/* Free the object as we go */
kmaps[nf] = astAnnul( kmaps[nf] );
} /* End of loop over this obsidss/subarray/heater */
kmaps = astFree( kmaps );
}
}
if (array->ndat) {
if (fflats) *fflats = array;
} else {
smf_close_related(&array, status );
if (fflats) *fflats = NULL;
}
}
}
/* no need to do any more if neither darks nor darkgrp are defined or we might
be wanting to revert to darks. */
if (dkcount > 0 && (darks || darkgrp || reverttodark ) ) {
smfArray * array = NULL;
/* sort darks into order */
qsort( alldarks, dkcount, sizeof(*alldarks), smf_sort_bydouble);
if (darks) array = smf_create_smfArray( status );
/* now open the darks and store them if requested */
if (*status == SAI__OK) {
for (i = 0; i < dkcount; i++ ) {
size_t ori_index = (alldarks[i]).index;
/* Store the entry in the output group */
ndgCpsup( ingrp, ori_index, dgrp, status );
if (darks) {
/* read the value from the new group */
smf_open_file( dgrp, i+1, "READ", 0, &infile, status );
/* do we have to process these darks? */
if (reducedark) {
smfData *outfile = NULL;
smf_reduce_dark( infile, darktype, &outfile, status );
if (outfile) {
smf_close_file( &infile, status );
infile = outfile;
}
}
smf_addto_smfArray( array, infile, status );
}
}
if (darks) *darks = array;
}
}
/* free memory */
alldarks = astFree( alldarks );
allfflats = astFree( allfflats );
if( reverttodark && outgrp && (grpGrpsz(*outgrp,status)==0) &&
(grpGrpsz(dgrp,status)>0) ) {
/* If outgrp requested but no science observations were found, and
dark observations were found, return darks in outgrp and set
flatgrp and darkgrp to NULL. This is to handle cases where we
want to process data taken in the dark like normal science
data. To activate this behaviour set reverttodark */
msgOutiff( MSG__NORM, "", "Treating the dark%s as science data",
status, ( dkcount > 1 ? "s" : "" ) );
*outgrp = dgrp;
if( darkgrp ){
*darkgrp = NULL;
}
if( flatgrp ) {
*flatgrp = NULL;
}
grpDelet( &ogrp, status);
grpDelet( &fgrp, status);
if (meanstep && nsteps_dark > 0) *meanstep = duration_darks / nsteps_dark;
/* Have to clear the darks smfArray as well */
if (darks) smf_close_related( darks, status );
} else {
/* Store the output groups in the return variable or free it */
if (darkgrp) {
*darkgrp = dgrp;
} else {
grpDelet( &dgrp, status);
}
if (flatgrp) {
*flatgrp = fgrp;
} else {
grpDelet( &fgrp, status);
}
if (meanstep && nsteps_sci > 0) *meanstep = duration_sci / nsteps_sci;
}
msgSeti( "ND", sccount );
msgSeti( "DK", dkcount );
msgSeti( "FF", ffcount );
msgSeti( "TOT", insize );
if ( insize == 1 ) {
if (dkcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a dark",
status);
} else if (ffcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a fast flatfield",
status);
} else if (sccount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was accepted (observation type same as sequence type)",
status);
} else {
msgOutif( MSG__VERB, " ", "Single input file was not accepted.",
status);
}
} else {
if (dkcount == 1) {
msgSetc( "DKTXT", "was a dark");
} else {
msgSetc( "DKTXT", "were darks");
}
if (ffcount == 1) {
msgSetc( "FFTXT", "was a fast flat");
} else {
msgSetc( "FFTXT", "were fast flats");
}
if (sccount == 1) {
msgSetc( "NDTXT", "was science");
} else {
msgSetc( "NDTXT", "were science");
}
/* This might be a useful message */
msgOutif( MSG__NORM, " ", "Out of ^TOT input files, ^DK ^DKTXT, ^FF ^FFTXT "
"and ^ND ^NDTXT", status );
}
if (meanstep && *meanstep != VAL__BADD) {
msgOutiff( MSG__VERB, "", "Mean step time for input files = %g sec",
status, *meanstep );
}
/* Store the heater efficiency map */
if (*status != SAI__OK) heatermap = smf_free_effmap( heatermap, status );
if (heateffmap) *heateffmap = heatermap;
/* Now report the details of the observation */
smf_obsmap_report( MSG__NORM, obsmap, objmap, status );
obsmap = astAnnul( obsmap );
objmap = astAnnul( objmap );
scimap = astAnnul( scimap );
msgOutiff( SMF__TIMER_MSG, "",
"Took %.3f s to find science observations",
status, smf_timerupdate( &tv1, &tv2, status ) );
return;
}
void atlKy2hd( AstKeyMap *keymap, HDSLoc *loc, int *status ) {
/*
* Name:
* atlKy2hd
* Purpose:
* Copies values from an AST KeyMap to a primitive HDS object.
* Language:
* C.
* Invocation:
* void atlKy2hd( AstKeyMap *keymap, HDSLoc *loc, int *status )
* Description:
* This routine copies the contents of an AST KeyMap into a supplied
* HDS structure.
* Arguments:
* keymap
* An AST pointer to the KeyMap.
* loc
* A locator for the HDS object into which the KeyMap contents
* are to be copied. A new component is added to the HDS object for
* each entry in the KeyMap.
* status
* The inherited status.
* Copyright:
* Copyright (C) 2008, 2010, 2012 Science & Technology Facilities Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful,but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* TIMJ: Tim Jenness (JAC, Hawaii)
* {enter_new_authors_here}
* History:
* 29-APR-2008 (DSB):
* Original version.
* 2010-09-23 (TIMJ):
* Fix arrays of strings.
* 2010-09-30 (TIMJ):
* Make sure that we are using the correct status pointer in AST.
* 2010-10-01 (TIMJ):
* Sort the keys when writing to HDS structured.
* 2010-10-04 (TIMJ):
* Support Short ints in keymap
* 14-SEP-2012 (DSB):
* Moved from kaplibs to atl.
* 17-SEP-2012 (DSB):
* Add support for undefined values.
* {enter_further_changes_here}
* Bugs:
* {note_any_bugs_here}
*/
/* Local Varianles: */
AstObject **objArray = NULL;
AstObject *obj = NULL;
HDSLoc *cloc = NULL;
HDSLoc *dloc = NULL;
const char *cval = NULL;
const char *key;
double dval;
float fval;
int i;
int ival;
int j;
int lenc;
int nval;
char *oldsortby;
int *oldstat = NULL;
int size;
int type;
int veclen;
size_t el;
void *pntr = NULL;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Make sure that we are checking AST status */
oldstat = astWatch( status );
/* If set, save the old SortBy value and then ensure alphabetical sorting.
We need to take a copy of the original string since the buffer in which
the string is stored may be re-used by subsequent incocations of astGetC. */
if( astTest( keymap, "SortBy" ) ) {
int nc = 0;
oldsortby = astAppendString( NULL, &nc, astGetC( keymap, "SortBy" ) );
} else {
oldsortby = NULL;
}
astSet( keymap, "SortBy=KeyUp" );
/* Loop round each entry in the KeyMap. */
size = astMapSize( keymap );
for( i = 0; i < size; i++ ) {
if (*status != SAI__OK) break;
/* Get the key. the data type and the vector length for the current
KeyMap entry. */
key = astMapKey( keymap, i );
type = astMapType( keymap, key );
veclen = astMapLength( keymap, key );
/* If the current entry holds one or more nested KeyMaps, then we call
this function recursively to add them into a new HDS component. */
if( type == AST__OBJECTTYPE ) {
/* First deal with scalar entries holding a single KeyMap. */
if( veclen == 1 ) {
datNew( loc, key, "KEYMAP_ENTRY", 0, NULL, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0A( keymap, key, &obj );
if( astIsAKeyMap( obj ) ) {
atlKy2hd( (AstKeyMap *) obj, cloc, status );
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "atlKy2hd: Supplied KeyMap contains unusable AST "
"objects (programming error).", status );
}
datAnnul( &cloc, status );
/* Now deal with vector entries holding multiple KeyMaps. */
} else {
datNew( loc, key, "KEYMAP_ENTRY", 1, &veclen, status );
datFind( loc, key, &cloc, status );
objArray = astMalloc( sizeof( AstObject *) * (size_t)veclen );
if( objArray ) {
(void) astMapGet1A( keymap, key, veclen, &nval, objArray );
for( j = 1; j <= veclen; j++ ) {
datCell( cloc, 1, &j, &dloc, status );
if( astIsAKeyMap( objArray[ j - 1 ] ) ) {
atlKy2hd( (AstKeyMap *) objArray[ j - 1 ], dloc, status );
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "atlKy2hd: Supplied KeyMap contains unusable AST "
"objects (programming error).", status );
}
datAnnul( &dloc, status );
}
objArray = astFree( objArray );
}
datAnnul( &cloc, status );
}
/* For primitive types... */
} else if( type == AST__INTTYPE ) {
if( veclen == 1 ) {
datNew0I( loc, key, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0I( keymap, key, &ival );
datPut0I( cloc, ival, status );
datAnnul( &cloc, status );
} else {
datNew1I( loc, key, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_INTEGER", "WRITE", &pntr, &el, status );
(void) astMapGet1I( keymap, key, veclen, &nval, (int *) pntr );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
} else if( type == AST__SINTTYPE ) {
short sval = 0;
if( veclen == 1 ) {
datNew0W( loc, key, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0S( keymap, key, &sval );
datPut0W( cloc, sval, status );
datAnnul( &cloc, status );
} else {
datNew1W( loc, key, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_WORD", "WRITE", &pntr, &el, status );
(void) astMapGet1S( keymap, key, veclen, &nval, (short *) pntr );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
} else if( type == AST__DOUBLETYPE ) {
if( veclen == 1 ) {
datNew0D( loc, key, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0D( keymap, key, &dval );
datPut0D( cloc, dval, status );
datAnnul( &cloc, status );
} else {
datNew1D( loc, key, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_DOUBLE", "WRITE", &pntr, &el, status );
(void) astMapGet1D( keymap, key, veclen, &nval, (double *) pntr );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
} else if( type == AST__FLOATTYPE ) {
if( veclen == 1 ) {
datNew0R( loc, key, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0F( keymap, key, &fval );
datPut0R( cloc, fval, status );
datAnnul( &cloc, status );
} else {
datNew1R( loc, key, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_REAL", "WRITE", &pntr, &el, status );
(void) astMapGet1F( keymap, key, veclen, &nval, (float *) pntr );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
} else if( type == AST__STRINGTYPE ) {
lenc = astMapLenC( keymap, key );
if( veclen == 1 ) {
datNew0C( loc, key, lenc, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0C( keymap, key, &cval );
datPut0C( cloc, cval, status );
datAnnul( &cloc, status );
} else {
datNew1C( loc, key, lenc, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_CHAR", "WRITE", &pntr, &el, status );
(void) atlMapGet1C( keymap, key, veclen*lenc, lenc, &nval,
(char *) pntr, status );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
/* KeyMap "UNDEF" values are always scalar and have no corresponding HDS
data type. So arbitrarily use an "_INTEGER" primitive with no defined
value to represent a KeyMap UNDEF value. */
} else if( type == AST__UNDEFTYPE ) {
datNew0L( loc, key, status );
/* Unknown or unsupported data types. */
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "T", type );
errRep( "", "atlKy2hd: Supplied KeyMap contains entries with "
"unusable data type (^T) (programming error).", status );
}
}
/* If it was originally set, re-instate the old SortBy value in the KeyMap,
then free the memory. Otherwise, clear the SortBy attribute. */
if( oldsortby ) {
astSetC( keymap, "SortBy", oldsortby );
oldsortby = astFree( oldsortby );
} else {
astClear( keymap, "SortBy" );
}
/* Reset AST status */
astWatch( oldstat );
}
static void DisplayKeyMap( AstKeyMap *km, int sort, const char *prefix,
AstKeyMap *refkm, int *status ){
/*
* Name:
* DisplayKeyMap
* Purpose:
* Display the contents of a keymap.
* Synopsis:
* void DisplayKeyMap( AstKeyMap *km, int sort, const char *prefix,
* int *status )
* Arguments:
* km
* Pointer to the KeyMaps containing the values to display.
* sort
* If non-zero, sort the values alphabetically by their keys.
* prefix
* A string to prepend to eack key.
* refkm
* Reference key map (e.g. values from the supplied configuration
* rather than the NDF history), or null if not required.
* status
* Inherited status pointer.
* Description:
* This function displays the contents of a supplied KeyMap as
* a series of "key = value" strings, one per line. It calls itself
* recursively if a nested KeyMap is found, adding a suitable
* prefix to the nested keys. If a reference key map is supplied then
* the output shows how the main key map differs from it.
*/
/* Local Variables: */
AstObject *avalue;
AstObject *refavalue;
char cbuffer[ 255 ];
char newpref[ 255 ];
const char *cvalue;
const char *refcvalue;
const char *key;
int ikey;
int ival;
int nc;
int nkey;
int nval;
/* Check the inherited status */
if( *status != SAI__OK ) return;
if (refkm) astClear(refkm, "KeyError");
/* Sort the supplied KeyMap is required. */
if( sort ) astSetC( km, "SortBy", "KeyUp" );
/* Loop round all keys in the supplied KeyMap. */
nkey = astMapSize( km );
for( ikey = 0; ikey < nkey; ikey++ ) {
key = astMapKey( km, ikey );
/* If the current entry is a nest KeyMap, get a pointer to it and call
this function recurisvely to display it, modifying the prefix to add
to each key so that it includes the key associated with the nest keymap. */
if( astMapType( km, key ) == AST__OBJECTTYPE ) {
astMapGet0A( km, key, &avalue );
if (refkm) {
if (! astMapGet0A(refkm, key, &refavalue)) {
refavalue = (AstObject*) astKeyMap("");
}
}
else {
refavalue = NULL;
}
sprintf( newpref, "%s%s.", prefix, key );
DisplayKeyMap( (AstKeyMap *) avalue, sort, newpref,
(AstKeyMap *) refavalue, status );
avalue = astAnnul( avalue );
if (refavalue) refavalue = astAnnul(refavalue);
/* If the current entry is not a nested keymap, we display it now. */
} else {
/* Get the vector length of the entry. */
nval = astMapLength( km, key );
/* If it is a scalar, just get its value as a character string using
the automatic type conversion provided by the KeyMap class, and
display it, putting the supplied prefix at the start of the key. */
if( nval <= 1 ) {
cvalue = "<undef>";
astMapGet0C( km, key, &cvalue );
if (refkm) {
refcvalue = "<undef>";
if (astMapGet0C(refkm, key, &refcvalue)
&& ! strcmp(cvalue, refcvalue)) {
msgOutf("", "- %s%s = %s", status, prefix, key, cvalue);
}
else {
msgOutf("", "+ %s%s = %s", status, prefix, key, cvalue);
}
}
else {
msgOutf( "", "%s%s = %s", status, prefix, key, cvalue );
}
/* If it is a vector, we construct a string containing a comma-separated
list of elements, enclosed in parentheses. */
} else {
nc = 0;
cvalue = astAppendString( NULL, &nc, "(" );
for( ival = 0; ival < nval; ival++ ) {
if( astMapGetElemC( km, key, sizeof( cbuffer) - 1, ival,
cbuffer ) ) {
cvalue = astAppendString( (char *) cvalue, &nc, cbuffer );
}
if( ival < nval - 1 ) cvalue = astAppendString( (char *) cvalue,
&nc, "," );
}
cvalue = astAppendString( (char *) cvalue, &nc, ")" );
/* Do the same for the reference KeyMap. */
if (refkm && (nval = astMapLength(refkm, key))) {
nc = 0;
refcvalue = astAppendString(NULL, &nc, "(");
for (ival = 0; ival < nval; ival++) {
if (ival) {
refcvalue = astAppendString((char*) refcvalue, &nc, "," );
}
if (astMapGetElemC(refkm, key, sizeof(cbuffer) - 1, ival,
cbuffer)) {
refcvalue = astAppendString((char*) refcvalue, &nc,
cbuffer);
}
}
refcvalue = astAppendString((char*) refcvalue, &nc, ")");
}
else {
refcvalue = NULL;
}
/* Display the total string, with the current key prefix, and free the memory
used to store it. */
if (refkm) {
if (refcvalue && ! strcmp(cvalue, refcvalue)) {
msgOutf("", "- %s%s = %s", status, prefix, key, cvalue);
}
else {
msgOutf("", "+ %s%s = %s", status, prefix, key, cvalue);
}
}
else {
msgOutf( "", "%s%s = %s", status, prefix, key, cvalue );
}
cvalue = astFree( (void *) cvalue );
if (refcvalue) refcvalue = astFree((void*) refcvalue);
}
}
}
}
double cupidConfigD( AstKeyMap *config, const char *name, double def,
int *status ){
/*
*+
* Name:
* cupidConfigD
* Purpose:
* Get the value of a configuration parameter.
* Language:
* Starlink C
* Synopsis:
* double cupidConfigD( AstKeyMap *config, const char *name, double def,
* int *status )
* Description:
* This function returns a named value from the supplied KeyMap. If
* the KeyMap does not contain the named value, an attempt is made to
* obtain a value from a secondary KeyMap which should be stored
* within the supplied KeyMap, using a key equal to the constant
* string CUPID__CONFIG. If the secondary KeyMap does not contain a
* value, then the supplied default value is returned. In either case,
* the returned value is stored in the KeyMap.
* Parameters:
* config
* An AST KeyMap holding the configuration parameters. If NULL is
* supplied, the default value is returned without error.
* name
* The name of the value to extract from the KeyMap.
* def
* The default value to use.
* status
* Pointer to the inherited status value.
* Returned Value:
* The required value. The supplied default value is returned if an
* error occurs.
* Copyright:
* Copyright (C) 2005 Particle Physics & Astronomy Research Council.
* Copyright (C) 2008 Science & Technology Facilities Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* {enter_new_authors_here}
* History:
* 5-OCT-2005 (DSB):
* Original version.
* 26-MAR-2008 (DSB):
* Re-report a more friendly message iof the supplied text could
* not be interpreted as a numerical value.
* {enter_further_changes_here}
* Bugs:
* {note_any_bugs_here}
*-
*/
/* Local Variables: */
AstObject *sconfig; /* Object pointer obtained from KeyMap */
const char *text; /* Pointer to uninterprted text value */
double ret; /* The returned value */
/* Initialise */
ret = def;
/* Abort if an error has already occurred, or if no KeyMap was supplied. */
if( *status != SAI__OK || !config ) return ret;
/* Attempt to extract the named value from the supplied KeyMap. */
if( !astMapGet0D( config, name, &ret ) ) {
/* If the value was not found in the KeyMap, see if the KeyMap contains a
secondary KeyMap. */
if( astMapGet0A( config, CUPID__CONFIG, &sconfig ) ) {
/* If it does, see if the secondary KayMap contains a value for the named
entry. If it does, remove the value from the KeyMap so it does not
appear in the CUPID NDF extension. */
if( astMapGet0D( (AstKeyMap *) sconfig, name, &ret ) ) {
astMapRemove( (AstKeyMap *) sconfig, name );
/* If the text supplied by the user could not be interpreted as a
floating point value, re-report the error. */
} else if( *status == AST__MPGER ) {
ret = def;
errAnnul( status );
if( astMapGet0C( config, name, &text ) ) {
*status = SAI__ERROR;
msgSetc( "T", text );
msgSetc( "N", name );
errRep( "", "Illegal value \"^T\" supplied for configuration "
"parameter ^N.", status );
}
/* If the value was not found in either KeyMap, return the default value. */
} else {
ret = def;
}
/* Free the pointer to the secondary KeyMap. */
sconfig = astAnnul( sconfig );
/* If no secondary KeyMap was found, return the default value. */
} else {
ret = def;
}
/* Store the returned value in the supplied KeyMap if it is good. */
if( ret != VAL__BADD ) astMapPut0D( config, name, ret, NULL );
/* If the text supplied by the user could not be interpreted as a
floating point value, re-report the error. */
} else if( *status == AST__MPGER ) {
ret = def;
errAnnul( status );
if( astMapGet0C( config, name, &text ) ) {
*status = SAI__ERROR;
msgSetc( "T", text );
msgSetc( "N", name );
errRep( "", "Illegal value \"^T\" supplied for configuration "
"parameter ^N.", status );
}
}
/* Return the result. */
return ret;
}
void kpg1Kygp1( AstKeyMap *keymap, Grp **igrp, const char *prefix,
int *status ){
/*
*+
* Name:
* kpg1Kygp1
* Purpose:
* Creates a GRP group holding keyword/value pairs read from an AST KeyMap.
* Language:
* C.
* Invocation:
* void kpg1Kygp1( AstKeyMap *keymap, Grp **igrp, const char *prefix,
* int *status )
* Description:
* This function is the inverse of kpg1Kymp1. It extracts the values
* from the supplied AST KeyMap and creates a set of "name=value" strings
* which it appends to a supplied group (or creates a new group). If
* the KeyMap contains nested KeyMaps, then the "name" associated with
* each primitive value stored in the returned group is a hierarchical
* list of component names separated by dots.
* Arguments:
* keymap
* A pointer to the KeyMap. Numerical entries which have bad values
* (VAL__BADI for integer entries or VAL__BADD for floating point
* entries) are not copied into the group.
* igrp
* A location at which is stored a pointer to the Grp structure
* to which the name=value strings are to be appended. A new group is
* created and a pointer to it is returned if the supplied Grp
* structure is not valid.
* prefix
* A string to append to the start of each key extracted from the
* supplied KeyMap. If NULL, no prefix is used.
* status
* Pointer to the inherited status value.
* Notes:
* - This function provides a private implementation for the public
* KPG1_KYGRP Fortran routine and kpg1Kygrp C function.
* - Entries will be stored in the group in alphabetical order
* Copyright:
* Copyright (C) 2008,2010 Science & Technology Facilities Council.
* Copyright (C) 2005 Particle Physics & Astronomy Research Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful,but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* TIMJ: Tim Jenness (JAC, Hawaii)
* {enter_new_authors_here}
* History:
* 7-NOV-2005 (DSB):
* Original version.
* 15-JUL-2008 (TIMJ):
* Tweak to GRP C API.
* 2010-07-19 (TIMJ):
* Handle vector keymap entries.
* 2010-08-12 (TIMJ):
* Store entries in group in alphabetical order.
* 13-AUG-2010 (DSB):
* Re-instate the original SortBy value before exiting.
* {enter_further_changes_here}
* Bugs:
* {note_any_bugs_here}
*-
*/
/* Local Variables: */
AstObject *obj; /* Pointer to nested AST Object */
char *oldsortby; /* The old value of the KeyMap's SortBy attribute */
char *text; /* Sum of concatenated strings */
const char *key; /* Key string for current entry in KeyMap */
const char *value; /* Value of current entry in KeyMap */
double dval; /* Double value */
int *old_status; /* Pointer to original status variable */
int bad; /* Is the numerical entry value bad? */
int i; /* Index into supplied KeyMap */
int ival; /* Integer value */
int n; /* Number of entries in the KeyMap */
int nc; /* Length of "text" excluding trailing null */
int type; /* Data type of current entry in KeyMap */
int valid; /* Is the supplied GRP structure valid? */
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Make AST use the Fortran status variable. */
old_status = astWatch( status );
/* Create a new GRP group if required. */
valid = grpValid( *igrp, status );
if( !valid ) *igrp = grpNew( "Created by kpg1_Kygp1", status );
/* If set, save the old SortBy value and then ensure alphabetical sorting.
We need to take a copy of the original string since the buffer in which
the string is stored may be re-used by subsequent incocations of astGetC. */
if( astTest( keymap, "SortBy" ) ) {
nc = 0;
oldsortby = astAppendString( NULL, &nc, astGetC( keymap, "SortBy" ) );
} else {
oldsortby = NULL;
}
astSet( keymap, "SortBy=KeyUp" );
/* Get the number of entries in the KeyMap. */
n = astMapSize( keymap );
/* Loop round all the entries in the KeyMap.*/
for( i = 0; i < n; i++ ) {
/* Get the name and type of the current KeyMap entry. */
key = astMapKey( keymap, i );
type = astMapType( keymap, key );
/* If the entry is an AST Object, get a pointer to it.*/
if( type == AST__OBJECTTYPE && astMapGet0A( keymap, key, &obj ) ) {
/* If it is a nested KeyMap, update the prefix and call this function
recursively. We ignore other forms of AST Objects. */
if( astIsAKeyMap( obj ) ) {
nc = 0;
text = astAppendString( NULL, &nc, prefix );
text = astAppendString( text, &nc, key );
text = astAppendString( text, &nc, "." );
kpg1Kygp1( (AstKeyMap *) obj, igrp, text, status );
text = astFree( text );
}
/* If it is a primitive, format it and add it to the group. */
} else {
/* If it is a numerical type, see if it has a bad value. */
bad = 0;
if( type == AST__INTTYPE && astMapGet0I( keymap, key, &ival ) ){
if( ival == VAL__BADI ) bad = 1;
} else if( type == AST__DOUBLETYPE && astMapGet0D( keymap, key, &dval ) ){
if( dval == VAL__BADD ) bad = 1;
}
/* If it not bad, get its formatted value. We also make sure that astMapGet0C returns
true because we intend to skip undefined values. */
if( !bad && astMapGet0C( keymap, key, &value ) ) {
size_t length;
/* Write the key and equals sign to a buffer */
nc = 0;
text = astAppendString( NULL, &nc, prefix );
text = astAppendString( text, &nc, key );
text = astAppendString( text, &nc, "=" );
length = astMapLength( keymap, key );
if( length > 1 ) {
/* Vector so we need to use (a,b,c) syntax */
char thiselem[GRP__SZNAM+1];
size_t l;
text = astAppendString( text, &nc, "(");
for ( l = 0; l < length; l++) {
if( astMapGetElemC( keymap, key, sizeof(thiselem), l, thiselem ) ) {
text = astAppendString( text, &nc, thiselem );
}
/* always deal with the comma. Even if value was undef we need to put in the comma */
if( l < (length - 1) ) {
text = astAppendString( text, &nc, "," );
}
}
text = astAppendString( text, &nc, ")");
} else {
/* Scalar */
text = astAppendString( text, &nc, value );
}
/* Put it in the group. */
grpPut1( *igrp, text, 0, status );
text = astFree( text );
}
}
}
/* If it was originally set, re-instate the old SortBy value in the KeyMap,
then free the memory. Otherwise, clear the SortBy attribute. */
if( oldsortby ) {
astSetC( keymap, "SortBy", oldsortby );
oldsortby = astFree( oldsortby );
} else {
astClear( keymap, "SortBy" );
}
/* Make AST use its original status variable. */
astWatch( old_status );
}
void smf_clean_smfArray( ThrWorkForce *wf, smfArray *array,
smfArray **noisemaps, smfArray **com, smfArray **gai,
AstKeyMap *keymap, int *status ) {
/* Local Variables */
double badfrac; /* Fraction of bad samples to flag bad bolo */
smfData *data=NULL; /* Pointer to individual smfData */
int compreprocess; /* COMmon-mode cleaning as pre-processing step */
dim_t dcfitbox; /* width of box for measuring DC steps */
int dclimcorr; /* Min number of correlated steps */
int dcmaxsteps; /* number of DC steps/min. to flag bolo bad */
dim_t dcsmooth; /* median filter width before finding DC steps */
double dcthresh; /* n-sigma threshold for primary DC steps */
int dofft; /* are we doing a freq.-domain filter? */
int dkclean; /* Flag for dark squid cleaning */
smfFilter *filt=NULL; /* Frequency domain filter */
double flagfast; /* Threshold for flagging slow slews */
double flagslow; /* Threshold for flagging slow slews */
dim_t idx; /* Index within subgroup */
size_t nflag; /* Number of elements flagged */
double noisecliphigh = 0; /* Sigma clip high-noise outlier bolos */
double noisecliplow = 0; /* Sigma clip low-noise outlier bolos */
int noiseclipprecom = 0; /* Noise clipping before common-mode cleaning? */
const char *opteff=NULL; /* Pointer to optical efficiency NDF file name*/
int opteffdiv; /* Divide data by the optical efficiencies? */
int order; /* Order of polynomial for baseline fitting */
char param[ 20 ]; /* Buffer for config parameter name */
dim_t pcalen; /* Chunk length for PCA cleaning */
double pcathresh; /* n-sigma threshold for PCA cleaning */
double spikethresh; /* Threshold for finding spikes */
dim_t spikebox=0; /* Box size for spike finder */
struct timeval tv1, tv2; /* Timers */
int whiten; /* Apply whitening filter? */
int zeropad; /* Pad with zeros? */
/* Main routine */
if (*status != SAI__OK) return;
/*** TIMER ***/
smf_timerinit( &tv1, &tv2, status );
/* Check for valid inputs */
if( !array || (array->ndat < 1) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No data supplied", status );
}
if( array->sdata[0]->ndims != 3 ) {
*status = SMF__WDIM;
errRepf( "", FUNC_NAME ": Supplied smfData has %zu dims, needs 3", status,
data->ndims );
return;
}
if( !keymap ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL AstKeyMap supplied", status );
return;
}
/* Get cleaning parameters */
smf_get_cleanpar( keymap, array->sdata[0], &badfrac, &dcfitbox, &dcmaxsteps,
&dcthresh, &dcsmooth, &dclimcorr, &dkclean,
NULL, &zeropad, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, &flagslow, &flagfast, &order,
&spikethresh, &spikebox, &noisecliphigh, &noisecliplow,
NULL, &compreprocess, &pcalen, &pcathresh, NULL, NULL, NULL,
&noiseclipprecom, status );
/* Loop over subarray */
for( idx=0; (idx<array->ndat)&&(*status==SAI__OK); idx++ ) {
data = array->sdata[idx];
/* Update quality by synchronizing to the data array VAL__BADD values */
msgOutif(MSG__VERB,"", FUNC_NAME ": update quality", status);
smf_update_quality( data, 1, NULL, 0, badfrac, status );
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s updating quality",
status, smf_timerupdate(&tv1,&tv2,status) );
/* Fix DC steps */
if( dcthresh && dcfitbox ) {
msgOutiff(MSG__VERB, "", FUNC_NAME
": Flagging bolos with %lf-sigma DC steps in %" DIM_T_FMT " "
"samples as bad, using %" DIM_T_FMT
"-sample median filter and max %d "
"DC steps per min before flagging entire bolo bad...", status,
dcthresh, dcfitbox, dcsmooth, dcmaxsteps);
smf_fix_steps( wf, data, dcthresh, dcsmooth, dcfitbox, dcmaxsteps,
dclimcorr, 0, &nflag, NULL, NULL, status );
msgOutiff(MSG__VERB, "", FUNC_NAME": ...%zd flagged\n", status, nflag);
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s fixing DC steps",
status, smf_timerupdate(&tv1,&tv2,status) );
}
/* Flag Spikes */
if( spikethresh ) {
msgOutif(MSG__VERB," ", FUNC_NAME ": flag spikes...", status);
smf_flag_spikes( wf, data, SMF__Q_FIT, spikethresh, spikebox,
&nflag, status );
msgOutiff(MSG__VERB,"", FUNC_NAME ": ...found %zd", status, nflag );
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s flagging spikes",
status, smf_timerupdate(&tv1,&tv2,status) );
}
/* Flag periods of stationary pointing, and update scanspeed to more
accurate value */
if( flagslow || flagfast ) {
if( data->hdr && data->hdr->allState ) {
double scanvel=0;
if( flagslow ) {
msgOutiff( MSG__VERB, "", FUNC_NAME
": Flagging regions with slew speeds < %.2lf arcsec/sec",
status, flagslow );
}
if( flagfast ) {
msgOutiff( MSG__VERB, "", FUNC_NAME
": Flagging regions with slew speeds > %.2lf arcsec/sec",
status, flagfast );
/* Check to see if this was a sequence type that involved
motion. If not, skip this section */
if( data && data->hdr && (
(data->hdr->seqtype==SMF__TYP_SCIENCE) ||
(data->hdr->seqtype==SMF__TYP_POINTING) ||
(data->hdr->seqtype==SMF__TYP_FOCUS) ||
(data->hdr->seqtype==SMF__TYP_SKYDIP))
&& (data->hdr->obsmode!=SMF__OBS_STARE) ) {
smf_flag_slewspeed( data, flagslow, flagfast, &nflag, &scanvel,
status );
msgOutiff( MSG__VERB,"", "%zu new time slices flagged", status,
nflag);
if( msgIflev( NULL, status ) >= MSG__VERB ) {
msgOutf( "", FUNC_NAME ": mean SCANVEL=%.2lf arcsec/sec"
" (was %.2lf)", status, scanvel, data->hdr->scanvel );
}
data->hdr->scanvel = scanvel;
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME
": ** %f s flagging outlier slew speeds",
status, smf_timerupdate(&tv1,&tv2,status) );
} else {
msgOutif( MSG__VERB, "", FUNC_NAME
": not a moving sequence or missing header, "
"skipping slew speed flagging", status );
}
}
} else {
msgOutif( MSG__DEBUG, "", FUNC_NAME
": Skipping flagslow/flagfast because no header present",
status );
}
}
/* Clean out the dark squid signal */
if( dkclean ) {
msgOutif(MSG__VERB, "", FUNC_NAME
": Cleaning dark squid signals from data.", status);
smf_clean_dksquid( data, 0, 100, NULL, 0, 0, 0, status );
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s DKSquid cleaning",
status, smf_timerupdate(&tv1,&tv2,status) );
}
/* Apply optical efficiency corrections. */
one_strlcpy( param, "OPTEFF", sizeof(param), status );
smf_find_subarray( data->hdr, param + strlen(param),
sizeof(param) - strlen(param), NULL, status );
astChrCase( NULL, param, 1, 0 );
if( astMapHasKey( keymap, param ) ) {
astMapGet0I( keymap, "OPTEFFDIV", &opteffdiv );
if ( astMapGet0C( keymap, param, &opteff ) ) {
msgOutiff( MSG__VERB,"", FUNC_NAME ": %s bolometer values "
"by factors read from NDF %s", status,
opteffdiv ? "Dividing" : "Multiplying", opteff );
smf_scale_bols( wf, data, NULL, opteff, param, opteffdiv, status );
}
}
/* Remove baselines */
if( order >= 0 ) {
msgOutiff( MSG__VERB,"", FUNC_NAME
": Fitting and removing %i-order polynomial baselines",
status, order );
smf_fit_poly( wf, data, order, 1, NULL, status );
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME
": ** %f s removing poly baseline",
status, smf_timerupdate(&tv1,&tv2,status) );
}
}
/* Mask noisy bolos here if happening before common-mode cleaning */
if( (*status == SAI__OK) && ((noisecliphigh>0.0) || (noisecliplow>0.0)) &&
noiseclipprecom ) {
smf__noisymask( wf, data, noisemaps, noisecliphigh, noisecliplow,
zeropad, &tv1, &tv2, status );
}
/* Optionally call smf_calcmodel_com to perform a subset of the following
tasks as a pre-processing step:
- remove the common-mode
- flag outlier data using common-mode rejection
- determine relative flatfields using amplitude of common-mode
In order to do this we need to set up some temporary model container
files so that the routine can be called properly. All of the same
COMmon-mode and GAIn model parameters (e.g. com.* and gai.*) will be
used here. However, in addition the "compreprocess" flag must be set
for this operation to be performed. */
if( compreprocess ) {
smfArray *comdata = NULL;
smfGroup *comgroup = NULL;
smfDIMMData dat;
smfArray *gaidata = NULL;
smfGroup *gaigroup = NULL;
smfArray *quadata = NULL;
smfData *thisqua=NULL;
msgOutif(MSG__VERB," ", FUNC_NAME ": Remove common-mode", status);
/* Create model containers for COM, GAI */
smf_model_create( wf, NULL, &array, NULL, NULL, NULL, NULL, NULL, 1, SMF__COM,
0, NULL, 0, NULL, NULL, &comgroup, &comdata, keymap,
status );
smf_model_create( wf, NULL, &array, NULL, NULL, NULL, NULL, NULL, 1, SMF__GAI,
0, NULL, 0, NULL, NULL, &gaigroup, &gaidata, keymap,
status );
/* Manually create quadata to share memory with the quality already
stored in array */
quadata = smf_create_smfArray( status );
for( idx=0; (*status==SAI__OK) && (idx<array->ndat); idx++ ) {
/* Create several new smfDatas, but they will all be freed
properly when we close quadata */
thisqua = smf_create_smfData( SMF__NOCREATE_DA | SMF__NOCREATE_HEAD |
SMF__NOCREATE_FILE, status );
/* Probably only need pntr->[0], but fill in the dimensionality
information to be on the safe side */
thisqua->dtype = SMF__QUALTYPE;
thisqua->ndims = array->sdata[idx]->ndims;
thisqua->isTordered = array->sdata[idx]->isTordered;
memcpy( thisqua->dims, array->sdata[idx]->dims, sizeof(thisqua->dims) );
memcpy( thisqua->lbnd, array->sdata[idx]->lbnd, sizeof(thisqua->lbnd) );
thisqua->pntr[0] = smf_select_qualpntr( array->sdata[idx], NULL, status );
smf_addto_smfArray( quadata, thisqua, status );
}
/* Set up the smfDIMMData and call smf_calcmodel_com */
memset( &dat, 0, sizeof(dat) );
dat.res = &array;
dat.gai = &gaidata;
dat.qua = &quadata;
dat.noi = NULL;
smf_calcmodel_com( wf, &dat, 0, keymap, &comdata, SMF__DIMM_FIRSTITER,
status );
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME
": ** %f s removing common-mode",
status, smf_timerupdate(&tv1,&tv2,status) );
/* Clean up and/or return values */
if( com ) {
*com = comdata;
} else {
if( comdata ) smf_close_related( &comdata, status );
}
if( gai ) {
*gai = gaidata;
} else {
if( gaidata ) smf_close_related( &gaidata, status );
}
if( comgroup ) smf_close_smfGroup( &comgroup, status );
if( gaigroup ) smf_close_smfGroup( &gaigroup, status );
/* Before closing quadata unset all the pntr[0] since this is shared
memory with the quality associated with array */
if( quadata ) {
for( idx=0; idx<quadata->ndat; idx++ ) {
quadata->sdata[idx]->pntr[0] = NULL;
}
if( quadata ) smf_close_related( &quadata, status );
}
}
/* PCA cleaning */
if( pcathresh ) {
/* Loop over subarray */
for( idx=0; (idx<array->ndat)&&(*status==SAI__OK); idx++ ) {
data = array->sdata[idx];
smf_clean_pca_chunks( wf, data, pcalen, pcathresh, keymap, status );
}
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s PCA cleaning",
status, smf_timerupdate(&tv1,&tv2,status) );
}
/* Allocate space for noisemaps if required */
if( noisemaps ) {
*noisemaps = smf_create_smfArray( status );
}
/* Loop over subarray */
for( idx=0; (idx<array->ndat)&&(*status==SAI__OK); idx++ ) {
data = array->sdata[idx];
/* Filter the data. Note that we call smf_filter_execute to apply
a per-bolometer whitening filter even if there is no
explicitly requested smfFilter (in which case the
smf_filter_fromkeymap call will leave the real/imaginary parts
of the filter as NULL pointers and they will get ignored inside
smf_filter_execute). */
filt = smf_create_smfFilter( data, status );
smf_filter_fromkeymap( filt, keymap, data->hdr, &dofft, &whiten, status );
if( (*status == SAI__OK) && dofft ) {
msgOutif( MSG__VERB, "", FUNC_NAME ": frequency domain filter", status );
smf_filter_execute( wf, data, filt, 0, whiten, status );
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s filtering data",
status, smf_timerupdate(&tv1,&tv2,status) );
}
filt = smf_free_smfFilter( filt, status );
/* Mask noisy bolos here if happening after common-mode cleaning */
if( (*status == SAI__OK) && ((noisecliphigh>0.0) || (noisecliplow>0.0)) &&
!noiseclipprecom ) {
smf__noisymask( wf, data, noisemaps, noisecliphigh, noisecliplow,
zeropad, &tv1, &tv2, status );
}
}
}
void smf_updateprov( int ondf, const smfData *data, int indf,
const char *creator, NdgProvenance **modprov, int *status ){
/* Local Variables */
AstFitsChan *fc = NULL; /* AST FitsChan holding input FITS header */
AstKeyMap *anc = NULL; /* KeyMap holding ancestor info */
AstKeyMap *tkm = NULL; /* KeyMap holding contents of MORE */
NdgProvenance *prov = NULL; /* Pointer to input provenance structure */
char *obsidss = NULL; /* Pointer to OBSIDSS buffer */
char obsidssbuf[SZFITSTR]; /* OBSIDSS value in input file */
const char *vptr = NULL; /* Pointer to OBSIDSS string */
int found; /* Was OBSIDSS value found in an input ancestor? */
int ianc; /* Ancestor index */
int isroot; /* Ignore any ancestors in the input NDF? */
/* Check the inherited status */
if( *status != SAI__OK ) return;
/* Get a FitsChan holding the contents of the input NDF FITS extension. */
if( !data ) {
kpgGtfts( indf, &fc, status );
} else if (data->hdr) {
fc = data->hdr->fitshdr;
}
/* Get the input NDF identifier. */
if( data && data->file &&
data->file->ndfid != NDF__NOID) indf = data->file->ndfid;
/* Get a structure holding provenance information from indf. */
prov = ndgReadProv( indf, " ", status );
/* Initially, assume that we should include details of ancestor NDFs. */
isroot = 0;
/* Get the OBSIDSS keyword value from the input FITS header if we have a FITS
header. There can be cases (eg in STACKFRAMES) where we do not have a
FITS header so handle that gracefully. */
if (fc) obsidss = smf_getobsidss( fc, NULL, 0, obsidssbuf,
sizeof(obsidssbuf), status );
if( obsidss ) {
/* Search through all the ancestors of the input NDF (including the input
NDF itself) to see if any of them refer to the same OBSIDSS value. */
found = 0;
ianc = 0;
anc = ndgGetProv( prov, ianc, status );
while( anc ) {
if( astMapGet0A( anc, "MORE", &tkm ) ) {
if( astMapGet0C( tkm, "OBSIDSS", &vptr ) ) {
found = !strcmp( obsidss, vptr );
}
tkm = astAnnul( tkm );
}
anc = astAnnul( anc );
if( ! found ) anc = ndgGetProv( prov, ++ianc, status );
}
/* If the OBSIDSS value was not found in any ancestor, then we add it
now. So put the OBSIDSS keyword value in an AST KeyMap that will be
stored with the output provenance information. */
if( ! found ) {
tkm = astKeyMap( " " );
astMapPut0C( tkm, "OBSIDSS", obsidss, NULL );
/* Ignore ancestor NDFs if none of them referred to the correct OBSIDSS. */
isroot = 1;
}
}
/* Update the provenance for the output NDF to include the input NDF as
an ancestor. Indicate that each input NDF is a root NDF (i.e. has no
parents). Do nothing if we have no provenance to propagate, unless
we have root information and are propagating that. */
if ( ndgCountProv( prov, status ) > 0 || tkm) {
NdgProvenance *oprov = NULL;
if (modprov && *modprov) {
oprov = *modprov;
} else {
oprov = ndgReadProv( ondf, creator, status );
if (modprov) *modprov = oprov;
}
ndgPutProv( oprov, indf, tkm, isroot, status );
/* do not update the file or free provenance if we
are using an external provenance struct or returning
it to the caller */
if (!modprov) {
if( ondf != NDF__NOID ) {
ndgWriteProv( oprov, ondf, 1, status );
oprov = ndgFreeProv( oprov, status );
} else if( *status == SAI__OK ){
*status = SAI__ERROR;
errRep( " ", "smf_updateprov: Provenance is to be stored in the "
"output NDF but no output NDF identifier was supplied "
"(programming error).", status );
}
}
}
/* Free resources. */
if( prov ) prov = ndgFreeProv( prov, status );
if( tkm ) tkm = astAnnul( tkm );
if( !data && fc ) fc = astAnnul( fc );
}
void smf_checkmem_dimm( dim_t maxlen, inst_t instrument, int nrelated,
smf_modeltype *modeltyps, dim_t nmodels, dim_t msize,
AstKeyMap *keymap, size_t available, dim_t maxfilelen,
size_t *necessary, int *status ) {
/* Local Variables */
int dofft=0; /* flag if we need temp space for FFTs */
dim_t i; /* Loop counter */
dim_t gain_box; /* Length of blocks for GAI/COM model */
AstKeyMap *kmap=NULL; /* Local keymap */
dim_t nblock; /* Number of blocks for GAI/COM model */
size_t ncol; /* Number of columns */
size_t ndet; /* Number of detectors each time step */
size_t ndks; /* dksquid samples in a subarray, ncol*maxlen */
size_t nrow; /* Number of rows */
size_t nsamp; /* bolo samples in a subarray, ndet*maxlen */
const char *tempstr=NULL; /* Temporary pointer to static char buffer */
size_t total = 0; /* Total bytes required */
/* Main routine */
if (*status != SAI__OK) return;
/* Check inputs */
if( maxlen < 1 ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": maxlen cannot be < 1", status);
return;
}
if( (nrelated < 1) || (nrelated > SMF__MXSMF) ) {
msgSeti("NREL",nrelated);
msgSeti("MAXREL",SMF__MXSMF);
*status = SAI__ERROR;
errRep("", FUNC_NAME ": nrelated, ^NREL, must be in the range [1,^MAXREL]",
status);
return;
}
if( modeltyps ) {
if( nmodels < 1 ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": modeltyps specified, mmodels cannot be < 1",
status);
return;
}
}
if( *status == SAI__OK ) {
/* Work out the data dimensions */
switch( instrument ) {
case INST__SCUBA2:
/* Kludgey, but at least we check SC2STORE__COL_INDEX so this will help
us catch possible future problems if order is changed */
if( SC2STORE__COL_INDEX ) {
ncol = 32;
nrow = 40;
} else {
ncol = 40;
nrow = 32;
}
ndet = ncol*nrow;
break;
default:
*status = SAI__ERROR;
errRep("", FUNC_NAME ": Invalid instrument given.", status);
}
}
/* Number of samples in a full data cube for one subarray */
nsamp = ndet*maxlen;
/* Check to see if we need to do filtering as part of
pre-processing. If so, we need an extra nsamp-sized buffer to
store the FFT. */
smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
&dofft, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
status );
/* Calculate memory usage of static model components: -------------------- */
if( *status == SAI__OK ) {
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated; /* RES */
total += nsamp*smf_dtype_sz(SMF__INTEGER,status)*nrelated; /* LUT */
total += nsamp*smf_dtype_sz(SMF__QUALTYPE,status)*nrelated; /* QUA */
}
/* Add on memory usage for the JCMTState (one per time slice per array) */
if( *status == SAI__OK ) {
total += maxlen*sizeof(JCMTState)*nrelated;
}
/* Add on space for dark squids */
ndks = ncol*maxlen;
total += ndks*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
/* Add on space for fakemap */
astMapGet0C( keymap, "FAKEMAP", &tempstr );
if( tempstr ) {
total += msize*sizeof(double);
}
/* Apply fudge factor */
total *= CHECKMEM_FUDGE;
/* Calculate memory usage of dynamic model components: ------------------- */
if( *status == SAI__OK ) {
/* Most of these will have data arrays associated with each
subarray (hence the multiplication by nrelated). An exception
is SMF__COM for which a single-common mode is used across all
subarrays. */
if( modeltyps ) {
for( i=0; i<nmodels; i++ ) {
switch( modeltyps[i] ) {
case SMF__NOI:
/* SMF__NOI also estimates the noise in each detector from the
power spectra, requiring a temporary buffer to store an FFT.
Currently we just store one variance per detector */
dofft = 1;
total += ndet*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__COM:
CHECK_MASK("COM")
total += maxlen*smf_dtype_sz(SMF__DOUBLE,status);
break;
case SMF__EXT:
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__DKS:
total += (maxlen + nrow*3)*ncol*smf_dtype_sz(SMF__DOUBLE,status) *
nrelated;
break;
case SMF__GAI:
/* Every COM.GAIN_BOX samples there are 3 planes of data
corresponding to each bolometer in the subarray. The
conversion of COM.GAIN_BOX from seconds to samples within
smf_get_nsamp assumes a sample rate of 200 Hz. Later
downsampling may result in a lower sample rate, but at least
we are erring on the conservative side by assuming 200 Hz. */
if( astMapGet0A( keymap, "COM", &kmap ) ) {
smf_get_nsamp( kmap, "GAIN_BOX", NULL, &gain_box, status );
nblock = maxlen/gain_box;
if( nblock == 0 ) nblock = 1;
total += nblock*3*nrow*ncol*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
kmap = astAnnul( kmap );
}
break;
case SMF__FLT:
/* Presently the filter temporarily transforms the entire
data cube into a second array. We therefore need to
ensure enough memory to temporarily store the data cube
twice. */
CHECK_MASK("FLT")
dofft = 1;
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__PLN:
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__SMO:
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__TMP:
/* An externally supplied template. The model just stores the gain,
offset and correlation coefficient for each bolometer, similar
to SMF__GAI except with only one chunk considered */
total += 3*nrow*ncol*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__TWO:
/* two common-mode time series and coefficients for all the
detectors */
total += 2*(ndet+maxlen)*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__AST:
/* Mostly accounted for as static memory usage above, but add space
for mask if required */
CHECK_MASK("AST")
break;
default:
*status = SAI__ERROR;
errRep("", FUNC_NAME ": Invalid smf_modeltype given.", status);
}
/* Exit on bad status */
if( *status != SAI__OK ) {
i = nmodels;
}
}
}
/* Calculate temporary space here -------------------------------------- */
if( *status == SAI__OK ) {
size_t temp=0; /* current temp memory required */
size_t maxtemp=0; /* max temp memory required */
/* Some temp space required when we initially read in the
data. Normally smf_concat_smfGroup will
smf_open_and_flatfield one file at a time before copying into
the concatenated data array. If there are N time slices in a
file, smf_open_and_flatfield will usually require space both
for the raw, and double-precision flatfielded data
simultaneously. This is an upper limit, since if previously
flatfielded data are provided no extra space for the raw
data will be required. We use the supplied maxfilelen to figure
out this maximum temporary buffer size. Remember to account
for bolo data, dark squids, and JCMTState. Also, the way
smf_iteratemap is currently written, a concatenated memory-mapped
pointing LUT is initially created, and then copied to a new
malloc'd array, so we need extra temp space for the full LUT. */
temp = (ndet + ncol)*( smf_dtype_sz(SMF__INTEGER,status)+
smf_dtype_sz(SMF__DOUBLE,status) )*maxfilelen +
2*maxfilelen*sizeof(JCMTState);
temp += nsamp*smf_dtype_sz(SMF__INTEGER,status)*nrelated;
if( temp > maxtemp ) maxtemp = temp;
/* If we are doing FFTs */
if( dofft ) {
temp = nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
if( temp > maxtemp ) maxtemp = temp;
}
/* Data cleaning... seems to be a lot. Just putting in a rough
estimate of 3 extra full sized arrays for the standard
options. */
temp = 3.*nsamp*smf_dtype_sz(SMF__DOUBLE,status);
if( temp > maxtemp ) maxtemp = temp;
/* Add on the maximum chunk of temporary memory that is required */
total += maxtemp;
}
/* Set bad status if too big */
if( (*status == SAI__OK) && (total > available) ) {
*status = SMF__NOMEM;
msgSeti("REQ",total/SMF__MIB);
msgSeti("AVAIL",available/SMF__MIB);
errRep("", FUNC_NAME
": Requested memory ^REQ MiB for map exceeds available ^AVAIL MiB",
status);
}
/* Return the required space */
if( necessary ) {
*necessary = total;
}
}
}
void smf_grp_related( const Grp *igrp, const size_t grpsize,
const int grouping, const int checksubinst,
double maxlen_s, double *srate_maxlen,
AstKeyMap *keymap, dim_t *maxconcatlen,
dim_t *maxfilelen, smfGroup **group,
Grp **basegrp, dim_t *pad, int *status ) {
/* Local variables */
size_t *chunk=NULL; /* Array of flags for continuous chunks */
dim_t * chunklen = NULL; /* Length of continuous chunk */
size_t currentindex = 0; /* Counter */
char cwave[10]; /* String containing wavelength */
smfData *data = NULL; /* Current smfData */
double downsampscale=0; /* Angular scale downsampling size */
double downsampfreq=0; /* Target downsampling frequency */
AstKeyMap * grouped = NULL; /* Primary AstKeyMap for grouping */
size_t i; /* Loop counter for index into Grp */
int isFFT=0; /* Set if data are 4d FFT */
size_t j; /* Loop counter */
int *keepchunk=NULL; /* Flag for chunks that will be kept */
dim_t maxconcat=0; /* Longest continuous chunk length */
dim_t maxflen=0; /* Max file length in time steps */
dim_t maxlen=0; /* Maximum concat length in samples */
int maxlen_scaled=0; /* Set once maxlen has been scaled, if needed */
dim_t maxpad=0; /* Maximum padding neeed for any input file */
size_t maxrelated = 0; /* Keep track of max number of related items */
size_t *new_chunk=NULL; /* keeper chunks associated with subgroups */
dim_t *new_tlen=NULL; /* tlens for new_subgroup */
size_t ngroups = 0; /* Counter for subgroups to be stored */
size_t nkeep = 0; /* Number of chunks to keep */
dim_t * piecelen = NULL; /* Length of single file */
smf_subinst_t refsubinst; /* Subinst of first file */
size_t **subgroups = NULL; /* Array containing index arrays to parent Grp */
smf_subinst_t subinst; /* Subinst of current file */
if ( *status != SAI__OK ) return;
if( maxlen_s < 0 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": maxlen_s cannot be < 0!", status );
return;
}
/* Get downsampling parameters */
if( keymap ) {
smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, &downsampscale, &downsampfreq,
NULL, NULL, NULL, NULL, status );
if( downsampscale && downsampfreq ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": both downsampscale and downsampfreq are set",
status );
return;
}
}
/* Initialize refcwave */
refsubinst = SMF__SUBINST_NONE;
/* Loop over files in input Grp: remember Grps are indexed from 1 */
grouped = astKeyMap( "SortBy=KeyUp" );
for (i=1; i<=grpsize; i++) {
char newkey[128];
char dateobs[81];
char subarray[10];
size_t nrelated = 0;
AstKeyMap * filemap = NULL;
AstKeyMap * indexmap = NULL;
/* First step: open file and harvest metadata */
smf_open_file( NULL, igrp, i, "READ", SMF__NOCREATE_DATA, &data, status );
if (*status != SAI__OK) break;
if( i==1 ) {
isFFT = smf_isfft( data, NULL, NULL, NULL, NULL, NULL, status );
} else if( smf_isfft(data, NULL, NULL, NULL, NULL, NULL, status) != isFFT ){
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": mixture of time-series and FFT data encountered!",
status );
break;
}
/* If maxlen has not been set, do it here */
if( !maxlen && maxlen_s && data->hdr->steptime) {
maxlen = (dim_t) (maxlen_s / data->hdr->steptime );
}
/* Return srate_maxlen if requested: may want to know this number
even if maxlen_s is not set. Only calculate once, although it
gets overwritten once later if down-sampling. */
if( (i==1) && srate_maxlen && data->hdr->steptime ) {
*srate_maxlen = 1. / (double) data->hdr->steptime;
}
/* If requested check to see if we are mixing wavelengths */
if( checksubinst ) {
if( refsubinst == SMF__SUBINST_NONE ) {
refsubinst = smf_calc_subinst( data->hdr, status );
}
subinst = smf_calc_subinst( data->hdr, status );
if( subinst != refsubinst ) {
const char *refsubstr = smf_subinst_str( refsubinst, status );
const char *substr = smf_subinst_str( subinst, status );
*status = SAI__ERROR;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
msgSetc( "REFSUB", refsubstr );
msgSetc( "SUB", substr );
errRep( "", FUNC_NAME
": ^FILE uses sub-instrument ^SUB which doesn't match "
"reference ^REFSUB", status );
}
}
/* Want to form a key that will be unique for a particular subscan
We know that DATE-OBS will be set for SCUBA-2 files and be the same
for a single set. Prefix by wavelength if we are grouping by wavelength.
*/
newkey[0] = '\0';
smf_find_subarray( data->hdr, subarray, sizeof(subarray), NULL, status );
if( grouping == 1 ) {
/* Group different wavelengths separately */
smf_fits_getS( data->hdr, "WAVELEN", cwave, sizeof(cwave), status);
one_strlcat( newkey, cwave, sizeof(newkey), status );
one_strlcat( newkey, "_", sizeof(newkey), status );
}
if( grouping == 2 ) {
/* Group different subarrays separately */
one_strlcat( newkey, subarray, sizeof(newkey), status );
}
smf_fits_getS( data->hdr, "DATE-OBS", dateobs, sizeof(dateobs), status );
one_strlcat( newkey, dateobs, sizeof(newkey), status );
/* Include the dimentionality of the time series in the primary key
so that we do not end up doing something confusing like relating
a truncated file with a full length file */
if (*status == SAI__OK) {
dim_t dims[3];
char formatted[32];
smf_get_dims( data, &dims[0], &dims[1], NULL, &dims[2], NULL, NULL, NULL,
status );
sprintf(formatted, "_%" DIM_T_FMT "_%" DIM_T_FMT "_%" DIM_T_FMT, dims[0], dims[1], dims[2]);
one_strlcat( newkey, formatted, sizeof(newkey), status );
}
/* May want to read the dimensionality of the file outside of loop
so that we can compare values when storing in the keymap */
/* Now we want to create a keymap based on this key */
if (!astMapGet0A( grouped, newkey, &filemap ) ) {
int itemp = 0;
double steptime = data->hdr->steptime;
dim_t ntslice = 0;
dim_t thispad; /* Padding neeed for current input file */
filemap = astKeyMap( " " );
astMapPut0A( grouped, newkey, filemap, NULL );
/* Fill up filemap with general information on this file */
smf_find_seqcount( data->hdr, &itemp, status );
astMapPut0I( filemap, "SEQCOUNT", itemp, NULL );
smf_fits_getI( data->hdr, "NSUBSCAN", &itemp, status );
astMapPut0I( filemap, "NSUBSCAN", itemp, NULL );
/* Number of time slices */
smf_get_dims( data, NULL, NULL, NULL, &ntslice, NULL, NULL, NULL,
status );
/* Find length of down-sampled data, new steptime and maxlen */
if( (downsampscale || downsampfreq) && data->hdr && (*status==SAI__OK) ) {
double scalelen;
if( downsampscale ) {
if( data->hdr->scanvel != VAL__BADD ) {
double oldscale = steptime * data->hdr->scanvel;
scalelen = oldscale / downsampscale;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
scalelen = VAL__BADD;
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep( "", FUNC_NAME ": can't resample ^FILE because it has "
"unknown scan velocity", status );
}
} else {
if( steptime ) {
double oldsampfreq = 1./steptime;
scalelen = downsampfreq / oldsampfreq;
} else {
*status = SAI__ERROR;
scalelen = VAL__BADD;
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep( "", FUNC_NAME ": can't resample ^FILE because it has "
"unknown sample rate", status );
}
}
/* only down-sample if it will be a reasonable factor */
if( (*status==SAI__OK) && (scalelen <= SMF__DOWNSAMPLIMIT) ) {
smf_smfFile_msg(data->file, "FILE", 1, "" );
msgOutiff( MSG__VERB, "", FUNC_NAME
": will down-sample file ^FILE from %5.1lf Hz to "
"%5.1lf Hz", status, (1./steptime), (scalelen/steptime) );
ntslice = round(ntslice * scalelen);
/* If maxlen has been requested, and we have not already worked
out a scaled version (just uses the sample rates for the first
file... should be close enough -- the alternative is a 2-pass
system). */
if( !maxlen_scaled ) {
maxlen = round(maxlen*scalelen);
maxlen_scaled = 1;
msgOutiff( MSG__VERB, "", FUNC_NAME
": requested maxlen %g seconds = %" DIM_T_FMT " down-sampled "
"time-slices", status, maxlen_s, maxlen );
/* Return updated srate_maxlen for down-sampling if requested */
if( srate_maxlen ) {
*srate_maxlen = scalelen/steptime;
}
}
}
}
/* Check that an individual file is too long (we assume related
files are all the same) */
if( maxlen && (ntslice > maxlen) && *status == SAI__OK) {
*status = SAI__ERROR;
msgSeti("NTSLICE",ntslice);
msgSeti("MAXLEN",maxlen);
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep(FUNC_NAME,
"Number of time steps in file ^FILE time exceeds maximum "
"(^NTSLICE>^MAXLEN)", status);
}
/* Scaled values of ntslice and maximum length */
astMapPut0I( filemap, "NTSLICE", ntslice, NULL );
/* Work out the padding needed for this file including downsampling. */
if( keymap ) {
thispad = smf_get_padding( keymap, 0, data->hdr, VAL__BADD, status );
if( thispad > maxpad ) maxpad = thispad;
} else {
thispad = 0;
}
astMapPut0I( filemap, "PADDING", thispad, NULL );
/* Update maxflen */
if( ntslice > maxflen ) {
maxflen = ntslice;
}
/* Store OBSID or OBSIDSS depending on whether we are grouping by wavelength */
if (grouping) {
astMapPut0C( filemap, "OBSID", data->hdr->obsidss, NULL );
} else {
char obsid[81];
smf_getobsidss( data->hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status );
astMapPut0C( filemap, "OBSID", obsid, NULL );
}
}
/* Store the file index in another keymap indexed by subarray */
if ( !astMapGet0A( filemap, "GRPINDICES", &indexmap ) ) {
indexmap = astKeyMap( "SortBy=KeyUp" );
astMapPut0A( filemap, "GRPINDICES", indexmap, NULL );
}
astMapPut0I( indexmap, subarray, i, NULL );
/* Need to track the largest number of related subarrays in a single slot */
nrelated = astMapSize( indexmap );
if (nrelated > maxrelated) maxrelated = nrelated;
/* Free resources */
filemap = astAnnul( filemap );
indexmap = astAnnul( indexmap );
smf_close_file( NULL, &data, status );
}
/* We now know how many groups there are */
ngroups = astMapSize( grouped );
/* Sort out chunking. The items are sorted by date and then by wavelength.
We define a continuous chunk if it has the same OBSID, the same SEQCOUNT
and NSUBSCAN increments by one from the previous entry.
Also count number of related items in each slot.
*/
if (*status == SAI__OK) {
typedef struct { /* somewhere to store the values easily */
char obsid[81];
char related[81]; /* for concatenated subarrays */
int nsubscan;
int seqcount;
} smfCompareSeq;
smfCompareSeq current;
smfCompareSeq previous;
dim_t totlen = 0;
size_t thischunk;
/* Get the chunk flags and also store the size of the chunk */
chunk = astCalloc( ngroups, sizeof(*chunk) );
chunklen = astCalloc( ngroups, sizeof(*chunklen) );
piecelen = astCalloc( ngroups, sizeof(*piecelen) );
thischunk = 0; /* The current chunk */
for (i=0; i<ngroups; i++) {
AstKeyMap * thismap = NULL;
AstKeyMap * grpindices = NULL;
const char * tempstr = NULL;
int thistlen = 0;
size_t nsubarrays = 0;
/* Get the keymap entry for this slot */
astMapGet0A( grouped, astMapKey(grouped, i), &thismap );
/* Get info for length limits */
astMapGet0I( thismap, "NTSLICE", &thistlen );
piecelen[i] = thistlen;
if (isFFT) {
/* Never concatenate FFT data */
thismap = astAnnul(thismap);
chunk[i] = i;
chunklen[i] = thistlen;
continue;
}
/* Get indices information and retrieve the sub-instrument names
in sort order to concatenate for comparison. We only store in
a continuous chunk if we have the same subarrays for the whole
chunk. */
astMapGet0A( thismap, "GRPINDICES", &grpindices );
nsubarrays = astMapSize( grpindices );
(current.related)[0] = '\0';
for (j = 0; j < nsubarrays; j++ ) {
one_strlcat( current.related, astMapKey(grpindices, j), sizeof(current.related), status );
}
grpindices = astAnnul( grpindices );
/* Fill in the current struct */
astMapGet0I( thismap, "SEQCOUNT", &(current.seqcount) );
astMapGet0I( thismap, "NSUBSCAN", &(current.nsubscan) );
astMapGet0C( thismap, "OBSID", &tempstr );
one_strlcpy( current.obsid, tempstr, sizeof(current.obsid), status );
/* First chunk is special, else compare */
if (i == 0) {
totlen = thistlen;
} else {
if ( ( current.seqcount == previous.seqcount ) &&
( current.nsubscan - previous.nsubscan == 1 ) &&
( strcmp( current.obsid, previous.obsid ) == 0 ) &&
( strcmp( current.related, previous.related ) == 0 ) ) {
/* continuous - check length */
totlen += thistlen;
if ( maxlen && totlen > maxlen ) {
thischunk++;
totlen = thistlen; /* reset length */
} else {
/* Continuous */
}
} else {
/* discontinuity */
thischunk++;
totlen = thistlen; /* Update length of current chunk */
}
}
chunklen[thischunk] = totlen;
chunk[i] = thischunk;
memcpy( &previous, ¤t, sizeof(current) );
thismap = astAnnul( thismap );
}
}
/* Decide if we are keeping a chunk by looking at the length. */
maxconcat = 0;
nkeep = 0;
keepchunk = astMalloc( ngroups*sizeof(*keepchunk) );
for (i=0; i<ngroups; i++) {
size_t thischunk;
thischunk = chunk[i];
if ( chunklen[thischunk] < SMF__MINCHUNKSAMP ) {
/* Warning message */
msgSeti("LEN",chunklen[thischunk]);
msgSeti("MIN",SMF__MINCHUNKSAMP);
msgOut( " ", "SMF_GRP_RELATED: ignoring short chunk (^LEN<^MIN)",
status);
keepchunk[i] = 0;
} else {
keepchunk[i] = 1;
if (maxconcat < chunklen[thischunk]) maxconcat = chunklen[thischunk];
nkeep++;
}
}
/* If no useful chunks generate an error */
if( (*status==SAI__OK) && (!nkeep) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No useful chunks.", status );
goto CLEANUP;
}
/* Allocate a subgroup array of the right size and fill it. They keymap
is sorted by date (and wavelength) so we can always index into it by using
indices from the subgroup. */
subgroups = astCalloc( nkeep, sizeof(*subgroups) );
new_chunk = astCalloc( nkeep, sizeof(*new_chunk) );
new_tlen = astCalloc( nkeep, sizeof(*new_tlen) );
currentindex = 0;
for (i=0;i<ngroups;i++) {
AstKeyMap * thismap = NULL;
AstKeyMap * grpindices = NULL;
size_t nsubarrays = 0;
size_t *indices = astCalloc( maxrelated, sizeof(*indices) );
/* skip if we are dropping this chunk */
if (!keepchunk[i]) continue;
/* Get the keymap entry for this slot */
astMapGet0A( grouped, astMapKey(grouped, i), &thismap );
/* Get the indices keymap */
astMapGet0A( thismap, "GRPINDICES", &grpindices );
nsubarrays = astMapSize( grpindices );
for (j=0; j<nsubarrays; j++) {
int myindex;
astMapGet0I( grpindices, astMapKey(grpindices, j), &myindex );
indices[j] = myindex;
}
grpindices = astAnnul( grpindices );
thismap = astAnnul( thismap );
subgroups[currentindex] = indices;
new_chunk[currentindex] = chunk[i];
new_tlen[currentindex] = piecelen[i];
currentindex++;
}
/* Create the smfGroup */
*group = smf_construct_smfGroup( igrp, subgroups, new_chunk, new_tlen,
nkeep, maxrelated, 0, status );
/* Return maxfilelen if requested */
if( maxfilelen ) {
*maxfilelen = maxflen;
}
/* Return maxconcatlen if requested */
if( maxconcatlen ) {
*maxconcatlen = maxconcat;
}
/* Create a base group for output files if required */
/* Create a base group of filenames */
if (*status == SAI__OK && basegrp ) {
*basegrp = smf_grp_new( (*group)->grp, "Base Group", status );
/* Loop over time chunks */
for( i=0; (*status==SAI__OK)&&(i<(*group)->ngroups); i++ ) {
size_t idx;
/* Check for new continuous chunk */
if( i==0 || ( (*group)->chunk[i] != (*group)->chunk[i-1]) ) {
/* Loop over subarray */
for( idx=0; idx<(*group)->nrelated; idx++ ) {
size_t grpindex = (*group)->subgroups[i][idx];
if ( grpindex > 0 ) {
ndgCpsup( (*group)->grp, grpindex, *basegrp, status );
}
}
}
}
}
CLEANUP:
keepchunk = astFree( keepchunk );
chunk = astFree( chunk );
chunklen = astFree( chunklen );
piecelen = astFree( piecelen );
grouped = astAnnul( grouped );
if( *status != SAI__OK ) {
/* free the group */
if (basegrp && *basegrp) grpDelet( basegrp, status );
if (group && *group) {
smf_close_smfGroup( group, status );
} else {
/* have to clean up manually */
new_chunk = astFree( new_chunk );
new_tlen = astFree( new_tlen );
if( subgroups ) {
size_t isub;
for( isub=0; isub<nkeep; isub++ ) {
subgroups[isub] = astFree( subgroups[isub] );
}
subgroups = astFree( subgroups );
}
}
}
/* Return the maximum padding if required. */
if( pad ) *pad = maxpad;
}
int smf_initial_sky( ThrWorkForce *wf, AstKeyMap *keymap, smfDIMMData *dat,
int *iters, int *status ) {
/* Local Variables: */
char refparam[ DAT__SZNAM ];/* Name for reference NDF parameter */
const char *cval; /* The IMPORTSKY string value */
double *ptr; /* Pointer to NDF Data array */
double *vptr; /* Pointer to NDF Variance array */
int indf1; /* Id. for supplied reference NDF */
int indf2; /* Id. for used section of reference NDF */
int nel; /* Number of mapped NDF pixels */
int result; /* Returned flag */
int there; /* Is there a smurf extension in the NDF? */
int update; /* Was NDF opened for UPDATE access? */
size_t i; /* Loop count */
size_t junk; /* Unused value */
/* Initialise the returned value to indicate no sky has been subtractred. */
result = 0;
/* Assume the sky map was not created by an interupted previous run of
makemap. */
*iters = -1;
/* Check inherited status. */
if( *status != SAI__OK ) return result;
/* Begin an AST context. */
astBegin;
/* The IMPORTSKY config parameter should have the name of the ADAM
parameter to use for acquiring the NDF that contains the initial sky
estimate. If IMPORTSKY is "1", use REF. */
cval = NULL;
astMapGet0C( keymap, "IMPORTSKY", &cval );
if( cval ) {
if( !astChrMatch( cval, "REF" ) &&
!astChrMatch( cval, "MASK2" ) &&
!astChrMatch( cval, "MASK3" ) ) {
astMapGet0I( keymap, "IMPORTSKY", &result );
cval = ( result > 0 ) ? "REF" : NULL;
}
if( cval ) {
result = 1;
strcpy( refparam, cval );
astChrCase( NULL, refparam, 1, 0 );
}
}
/* Do nothing more if we are not subtracting an initial sky from the data. */
if( result && *status == SAI__OK ) {
/* Begin an NDF context. */
ndfBegin();
/* Get an identifier for the NDF using the associated ADAM parameter.
First try UPDATE access. If this fails try READ access. */
ndfAssoc( refparam, "UPDATE", &indf1, status );
if( *status != SAI__OK ) {
errAnnul( status );
ndfAssoc( refparam, "READ", &indf1, status );
update = 0;
} else {
update = 1;
}
/* Tell the user what we are doing. */
ndfMsg( "N", indf1 );
msgOut( "", "Using ^N as the initial guess at the sky", status );
/* Get a section from this NDF that matches the bounds of the map. */
ndfSect( indf1, 2, dat->lbnd_out, dat->ubnd_out, &indf2, status );
/* Ensure masked values are not set bad in the mapped data array. */
ndfSbb( 0, indf2, status );
/* Map the data array section, and copy it into the map buffer. */
ndfMap( indf2, "DATA", "_DOUBLE", "READ", (void **) &ptr, &nel, status );
if( *status == SAI__OK ) {
memcpy( dat->map, ptr, dat->msize*sizeof(*ptr));
}
/* Map the variance array section, and copy it into the map buffer. */
ndfState( indf2, "VARIANCE", &there, status );
if( there ) {
ndfMap( indf2, "VARIANCE", "_DOUBLE", "READ", (void **) &vptr, &nel, status );
if( *status == SAI__OK ) {
memcpy( dat->mapvar, vptr, dat->msize*sizeof(*vptr));
}
}
/* If the NDF was created by a previous run of makemap that was interupted
using control-C, it will contain a NUMITER item in the smurf extension,
which gives the number of iterations that were completed before the
map was created. Obtain and return this value, if it exists. */
ndfXstat( indf1, SMURF__EXTNAME, &there, status );
if( there ) ndfXgt0i( indf1, SMURF__EXTNAME, "NUMITER", iters,
status );
/* If the NDF has a Quality component, import it and create initial AST,
FLT, PCA, SSN and COM masks from it. These will often be over-ridden by
new masks calculated with smf_calcmodel_ast below, but will not be
over-written if the masks have been frozen by xxx.zero_freeze. */
ndfState( indf2, "Quality", &there, status );
if( there && dat->mapqual ) {
smf_qual_t *qarray = smf_qual_map( wf, indf2, "Read", NULL, &junk,
status );
if( *status == SAI__OK ) {
smf_qual_t *pq = qarray;
for( i = 0; i < dat->msize; i++,pq++ ) {
if( *pq & SMF__MAPQ_AST ) {
if( !dat->ast_mask ) dat->ast_mask = astCalloc( dat->msize,
sizeof( *(dat->ast_mask) ) );
(dat->ast_mask)[ i ] = 1;
}
if( *pq & SMF__MAPQ_FLT ) {
if( !dat->flt_mask ) dat->flt_mask = astCalloc( dat->msize,
sizeof( *(dat->flt_mask) ) );
(dat->flt_mask)[ i ] = 1;
}
if( *pq & SMF__MAPQ_COM ) {
if( !dat->com_mask ) dat->com_mask = astCalloc( dat->msize,
sizeof( *(dat->com_mask) ) );
(dat->com_mask)[ i ] = 1;
}
if( *pq & SMF__MAPQ_SSN ) {
if( !dat->ssn_mask ) dat->ssn_mask = astCalloc( dat->msize,
sizeof( *(dat->ssn_mask) ) );
(dat->ssn_mask)[ i ] = 1;
}
if( *pq & SMF__MAPQ_PCA ) {
if( !dat->pca_mask ) dat->pca_mask = astCalloc( dat->msize,
sizeof( *(dat->pca_mask) ) );
(dat->pca_mask)[ i ] = 1;
}
}
}
qarray = astFree( qarray );
}
/* Indicate the map arrays within the supplied smfDIMMData structure now
contain usable values. We need to do this before calling
smf_calcmodel_ast below so that the right mask gets used in
smf_calcmodel_ast. */
dat->mapok = 1;
/* Apply any existinction correction to the cleaned bolometer data. */
if( dat->ext ) smf_calcmodel_ext( wf, dat, 0, keymap, dat->ext, 0,
status);
/* Sample the above map at the position of each bolometer sample and
subtract the sampled value from the cleaned bolometer value. */
smf_calcmodel_ast( wf, dat, 0, keymap, NULL, SMF__DIMM_PREITER, status);
/* Remove any existinction correction to the modifed bolometer data. */
if( dat->ext ) smf_calcmodel_ext( wf, dat, 0, keymap, dat->ext,
SMF__DIMM_INVERT, status);
/* If the NDF was opened with UPDATE access, update the quality array in
the NDF to reflect the AST mask created by smf_calcmodel_ast above. */
if( update ) {
smf_qual_t *qarray = astStore( NULL, dat->mapqual, dat->msize*sizeof(*qarray) );
qarray = smf_qual_unmap( wf, indf2, SMF__QFAM_MAP, qarray, status );
}
/* End the NDF context. */
ndfEnd( status );
}
/* End the AST context. */
astEnd;
/* Return the pointer to the boolean mask. */
return result;
}
void smf_pread( Grp *igrp, const char *param, int *status ){
/* Local Variables: */
AstMapping *dlatmap;
AstMapping *dlonmap;
AstMapping *taimap;
AstTable *table;
char file[ GRP__SZNAM + 1 ];
char pbuf[ GRP__SZNAM + 1 ];
const char *system;
void *p;
/* Before we check the error status, see if we are annulling previously
created Mappings. If so, get each formatted pointer from the group
metadata, get an Object pointer form it, lock it for use by the
current thread, and then annul it. Remove the metadata item from the
group. */
if( !param ) {
pbuf[ 0 ] = 0;
smf_get_grp_metadata( igrp, "DLONMAP", pbuf, status );
if( pbuf[ 0 ] ) {
sscanf( pbuf, "%p", &p );
dlonmap = (AstMapping *) p;
astLock( dlonmap, 0 );
dlonmap = astAnnul( dlonmap );
smf_remove_grp_metadata( igrp, "DLONMAP", status );
}
pbuf[ 0 ] = 0;
smf_get_grp_metadata( igrp, "DLATMAP", pbuf, status );
if( pbuf[ 0 ] ) {
sscanf( pbuf, "%p", &p );
dlatmap = (AstMapping *) p;
astLock( dlatmap, 0 );
dlatmap = astAnnul( dlatmap );
smf_remove_grp_metadata( igrp, "DLATMAP", status );
}
return;
}
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Use the specified parameter to get the name of the text file containing
the table of pointing corrections. */
parGet0c( param, file, sizeof( file ) - 1, status );
/* If no file was specified, annul the error. */
if( *status == PAR__NULL ) {
errAnnul( status );
/* If a file was obtained sccuesfully, read it. */
} else if( *status == SAI__OK ) {
/* Start an AST context. */
astBegin;
/* Attempt to read an AST Table from the text file. */
table = atlReadTable( file, status );
/* Create a LutMap from each of the three columns. */
taimap = (AstMapping *) atlTablelutMap( table, "TAI", status );
dlonmap = (AstMapping *) atlTablelutMap( table, "DLON", status );
dlatmap = (AstMapping *) atlTablelutMap( table, "DLAT", status );
/* Create Mappings that transforms TAI into a DLON and DLAT. These use
linear interpolation for non-tabulated TAI values. */
astInvert( taimap );
dlonmap = (AstMapping *) astCmpMap( taimap, dlonmap, 1, " " );
dlatmap = (AstMapping *) astCmpMap( taimap, dlatmap, 1, " " );
/* Format the pointers to these two Mappings and store them in the
supplied group using names "DLONMAP" and "DLATMAP". */
sprintf( pbuf, "%p", (void *) dlonmap );
smf_add_grp_metadata( igrp, "DLONMAP", pbuf, status );
sprintf( pbuf, "%p", (void *) dlatmap );
smf_add_grp_metadata( igrp, "DLATMAP", pbuf, status );
/* See what system the DLON/DLAT values refer to (default to AZEL). Store
it in the group. */
if( !astMapGet0C( table, "SYSTEM", &system ) ) system = "AZEL";
smf_add_grp_metadata( igrp, "PSYSTEM", system, status );
/* Unlock the pointers to the Mappings so that they can be used by a
different thread. This also exempts the pointers from AST context
handling (until they are re-locked) so the following call to astEnd
will not annull them. */
astUnlock( dlonmap, 1 );
astUnlock( dlatmap, 1 );
/* End the AST context. This annuls all Objects created during the
context, except for the unlocked Mappings. */
astEnd;
/* Debug message. */
msgSetc( "F", file );
msgSetc( "S", system );
msgOutif( MSG__DEBUG, " ", "^S pointing corrections read from file ^F",
status );
/* Issue a context message if anything went wrong. */
if( *status != SAI__OK ) {
msgSetc( "F", file );
errRep( " ", "Failed to read pointing corrections from text file ^F.",
status );
}
}
}
static AstTable *ReadNextTable( FILE *fd, const char *fname, int *iline,
int *status ) {
/*
* Name:
* ReadOneTable
* Purpose:
* Reads a single Table from a text file.
* Description:
* This function reads text from the supplied file descriptor until it
* reaches the end of file or encounters an end-of-table marker (a line
* consisting just of two or more minus signs with no leading spaces).
* It creates an AstTable from the text and returns a pointer to it.
* Arguments:
* fd
* The file descriptor.
* fname
* The file name - used for error messages.
* iline
* Pointer to an int holding the number of lines read from the
* file so far. Updated on exit to include the lines read by the
* invocation of this function.
* status
* Pointer to the global status variable.
* Returned Value:
* A pointer to the Table read from the file, or NULL if an error occurs.
*/
/* Local Variables: */
AstTable *result;
AstTable *subtable;
char **cols;
char **words;
char *last_com;
char *line;
char *p;
char *tname;
char key[ 200 ];
const char *cval;
const char *oldname;
const char *newname;
double dval;
int *types;
int blank;
int c;
int com;
int eot;
int first;
int icol;
int irow;
int ival;
int iword;
int line_len;
int max_line_len;
int more;
int nc;
int ncol;
int nrow;
int nword;
int skip;
size_t len;
/* Initialise */
result = NULL;
/* Check the inherited status. */
if( *status != SAI__OK ) return result;
/* Create an empty Table. */
result = astTable( " " );
/* Allocate a buffer for one one line of text. This will be increased in
size as required. */
max_line_len = 80;
line = astMalloc( max_line_len*sizeof( *line ) );
/* Read each line of text from the file. */
eot = 0;
skip = 1;
line_len = 0;
more = 1;
last_com = NULL;
cols = NULL;
first = 1;
irow = 0;
types = NULL;
while( more && *status == SAI__OK ) {
(*iline)++;
line_len = 0;
/* Loop reading characters from the file until a newline or the end of file is
reached. */
while( ( c = fgetc( fd ) ) != EOF && c != '\n' ) {
/* Increment the current line length, and double the size of the line buffer
if it is full. */
if( ++line_len >= max_line_len ) {
max_line_len *= 2;
line = astRealloc( line, max_line_len*sizeof( *line ) );
if( *status != SAI__OK ) break;
}
/* Store the character. Ignore leading white space. */
if( skip ) {
if( ! isspace( c ) ) {
line[ line_len - 1 ] = c;
skip = 0;
} else {
line_len--;
}
} else {
line[ line_len - 1 ] = c;
}
}
/* If the end-of-file was reached indicate that we should leave the main
loop after processing the current line. */
if( c == EOF ) more = 0;
/* Terminate the line. */
line[ line_len ] = 0;
/* Terminate it again to exclude trailing white space. */
line[ astChrLen( line ) ] = 0;
/* Assume the line is a blank non-comment, and store a pointer to the first
character to use. */
blank = 1;
com = 0;
p = line;
/* Skip blank lines. */
if( line[ 0 ] ) {
/* If the line starts with a comment character... */
if( line[ 0 ] == '#' || line[ 0 ] == '!' ) {
com = 1;
/* Get a pointer to the first non-space/tab character after the comment
character. */
p = line + 1;
while( *p == ' ' || *p == '\t' ) p++;
/* Note if it is blank. */
if( *p ) blank = 0;
/* If it is not a comment line, then the line is not blank. */
} else {
blank = 0;
/* See if it is the end-of-table marker - a line containing just two or
more minus signs with no leading spaces. */
eot = ( strspn( line, "-" ) > 1 );
}
}
/* Skip blank lines, whether comment or not. */
if( ! blank ) {
/* First handle comment lines. */
if( com ) {
/* Does it look like a parameter assignment... */
words = astChrSplitRE( p, "^\\s*(\\w+)\\s*=\\s*(.*)$", &nword, NULL );
if( words ) {
/* Add a parameter declaration to the Table. */
astAddParameter( result, words[ 0 ] );
/* Store the parameter value, using an appropriate data type. */
len = strlen( words[ 1 ] );
if( nc = 0, ( 1 == astSscanf( words[ 1 ], "%d%n", &ival, &nc ) )
&& ( nc >= len ) ) {
astMapPut0I( result, words[ 0 ], ival, NULL );
} else if( nc = 0, ( 1 == astSscanf( words[ 1 ], "%lg%n", &dval, &nc ) )
&& ( nc >= len ) ) {
astMapPut0D( result, words[ 0 ], dval, NULL );
} else {
astMapPut0C( result, words[ 0 ], words[ 1 ], NULL );
}
/* Free the words returned by astChrSplitRE. */
for( iword = 0; iword < nword; iword++ ) {
words[ iword ] = astFree( words[ iword ] );
}
words = astFree( words );
/* If it does not look like a parameter assignment... */
} else {
/* Save a copy of it in case it turns out to be the last non-blank comment
line before the first row of data values (in which case it should
contain the column names). */
last_com = astStore( last_com, p, strlen( p ) + 1 );
}
/* If the line is not a comment see if it is an end of table marker. If so
indicate that we should leave the loop. */
} else if( eot ) {
more = 0;
/* If the line is not a comment or an end of table marker ... */
} else {
/* Get the words from the row. */
words = astChrSplit( p, &nword );
/* If this is the first non-blank non-comment line, get the column names from
the previous non-blank comment line. */
if( first ) {
if( last_com ) {
first = 0;
cols = astChrSplit( last_com, &ncol );
/* Create an array to hold the data type for each colum, and initialise
them to "integer". */
types = astMalloc( ncol*sizeof( int ) ) ;
for( iword = 0; iword < nword && astOK; iword++ ) {
if( iword < ncol ) {
types[ iword ] = AST__INTTYPE;
/* The columns are stored initially using interim names which have "T_"
prepended to the names given in the file. */
tname = NULL;
nc = 0;
tname = astAppendString( tname, &nc, "T_" );
tname = astAppendString( tname, &nc, cols[ iword ] );
astFree( cols[ iword ] );
cols[ iword ] = tname;
/* Create the column definition within the returned Table. We store them
initially as strings and then convert to the appropriate column data type
later (once all rows have been read and the the data types are known). */
astAddColumn( result, cols[ iword ], AST__STRINGTYPE,
0, NULL, " " );
}
}
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
msgSetc( "F", fname );
errRep( " ", "No column headers found in file ^F.", status );
}
}
/* Report an error if the line has the wrong number of values. */
if( nword != ncol ) {
if( *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "N", nword );
msgSeti( "I", (*iline) );
msgSeti( "M", ncol );
msgSetc( "F", fname );
errRep( " ", "Wrong number of values (^N) at line ^I in "
"file ^F (should be ^M).", status );
}
/* Otherwise increment the number of rows read. */
} else {
irow++;
/* Store each string value in the table, excluding "null" strings. Also check
the data type of each string an dupdate the column data types if necessary. */
for( iword = 0; iword < nword && *status == SAI__OK; iword++ ) {
if( strcmp( words[ iword ], "null" ) ) {
sprintf( key, "%s(%d)", cols[ iword ], irow );
astMapPut0C( result, key, words[ iword ], NULL );
/* If the column is currently thought to hold integers, check that the
current word looks like an integer. If not, down-grade the column type
to double. */
len = strlen( words[ iword ] );
if( types[ iword ] == AST__INTTYPE ) {
if( nc = 0, ( 1 != astSscanf( words[ iword ], "%d%n",
&ival, &nc ) ) ||
( nc < len ) ) {
types[ iword ] = AST__DOUBLETYPE;
}
}
/* If the column is currently thought to hold doubles, check that the
current word looks like an doubler. If not, down-grade the column type
to string. */
if( types[ iword ] == AST__DOUBLETYPE ) {
if( nc = 0, ( 1 != astSscanf( words[ iword ], "%lg%n",
&dval, &nc ) ) ||
( nc < len ) ) {
types[ iword ] = AST__STRINGTYPE;
}
}
}
}
}
/* Free the words returned by astChrSplit. */
for( iword = 0; iword < nword; iword++ ) {
words[ iword ] = astFree( words[ iword ] );
}
words = astFree( words );
}
}
}
/* The entire file has now been read, and a Table created in which every
column holds strings. We also have flags indicating whether the values
in each column are all integers or doubles. Modify the type of the
column within the Table to match these flags. */
nrow = astGetI( result, "Nrow" );
for( icol = 0; icol < ncol && *status == SAI__OK; icol++ ) {
/* The column will be re-named from "T_<name>" to "<name>". */
oldname = cols[ icol ];
newname = oldname + 2;
/* First convert string columns to integer columns if all the values in
the column look like integers. */
if( types[ icol ] == AST__INTTYPE ) {
/* Create the new column */
astAddColumn( result, newname, AST__INTTYPE, 0, NULL, " " );
/* Copy each cell of the current column, converting from string to integer. */
for( irow = 1; irow <= nrow; irow++ ) {
sprintf( key, "%s(%d)", oldname, irow );
if( astMapGet0I( result, key, &ival ) ) {
sprintf( key, "%s(%d)", newname, irow );
astMapPut0I( result, key, ival, NULL );
}
}
/* Now do double columns in the same way. */
} else if( types[ icol ] == AST__DOUBLETYPE ) {
astAddColumn( result, newname, AST__DOUBLETYPE, 0, NULL, " " );
for( irow = 1; irow <= nrow; irow++ ) {
sprintf( key, "%s(%d)", oldname, irow );
if( astMapGet0D( result, key, &dval ) ) {
sprintf( key, "%s(%d)", newname, irow );
astMapPut0D( result, key, dval, NULL );
}
}
/* Copy string values without change. */
} else {
astAddColumn( result, newname, AST__STRINGTYPE, 0, NULL, " " );
for( irow = 1; irow <= nrow; irow++ ) {
sprintf( key, "%s(%d)", oldname, irow );
if( astMapGet0C( result, key, &cval ) ) {
sprintf( key, "%s(%d)", newname, irow );
astMapPut0C( result, key, cval, NULL );
}
}
}
/* Remove the old column. */
astRemoveColumn( result, oldname );
}
/* Free resources. */
line = astFree( line );
last_com = astFree( last_com );
types = astFree( types );
if( cols ) {
for( icol = 0; icol < ncol; icol++ ) {
cols[ icol ] = astFree( cols[ icol ] );
}
cols = astFree( cols );
}
/* If the table ended with an end-of-table marker, there may be another
Table in the file. Call this function recursively to read it. */
if( eot ) {
subtable = ReadNextTable( fd, fname, iline, status );
/* Store the subtable as a table parameter in the returned table. */
if( subtable ) {
astAddParameter( result, "SubTable" );
astMapPut0A( result, "SubTable", subtable, NULL );
/* The Table clones the pointer, so we must annull our local copy of it. */
subtable = astAnnul( subtable );
}
}
/* Return the Table pointer. */
return result;
}
unsigned char *smf_get_mask( ThrWorkForce *wf, smf_modeltype mtype,
AstKeyMap *config, smfDIMMData *dat, int flags,
int *status ) {
/* Local Variables: */
AstCircle *circle; /* AST Region used to mask a circular area */
AstKeyMap *akm; /* KeyMap holding AST config values */
AstKeyMap *subkm; /* KeyMap holding model config values */
char refparam[ DAT__SZNAM ];/* Name for reference NDF parameter */
char words[100]; /* Buffer for variable message words */
const char *cval; /* The ZERO_MASK string value */
const char *modname; /* The name of the model being masked */
const char *skyrefis; /* Pointer to SkyRefIs attribute value */
dim_t i; /* Pixel index */
double *pd; /* Pointer to next element of map data */
double *predef; /* Pointer to mask defined by previous run */
double *ptr; /* Pointer to NDF Data array */
double *pv; /* Pointer to next element of map variance */
double centre[ 2 ]; /* Coords of circle centre in radians */
double meanhits; /* Mean hits in the map */
double radius[ 1 ]; /* Radius of circle in radians */
double zero_circle[ 3 ]; /* LON/LAT/Radius of circular mask */
double zero_lowhits; /* Fraction of mean hits at which to threshold */
double zero_snr; /* Higher SNR at which to threshold */
double zero_snrlo; /* Lower SNR at which to threshold */
int *ph; /* Pointer to next hits value */
int have_mask; /* Did a mask already exist on entry? */
int imask; /* Index of next mask type */
int indf1; /* Id. for supplied reference NDF */
int indf2; /* Id. for used section of reference NDF */
int isstatic; /* Are all used masks static? */
int lbnd_grid[ 2 ]; /* Lower bounds of map in GRID coords */
int mask_types[ NTYPE ]; /* Identifier for the types of mask to use */
int munion; /* Use union of supplied masks */
int nel; /* Number of mapped NDF pixels */
int nmask; /* The number of masks to be combined */
int nsource; /* No. of source pixels in final mask */
int skip; /* No. of iters for which AST is not subtracted */
int thresh; /* Absolute threshold on hits */
int ubnd_grid[ 2 ]; /* Upper bounds of map in GRID coords */
int zero_c_n; /* Number of zero circle parameters read */
int zero_mask; /* Use the reference NDF as a mask? */
int zero_niter; /* Only mask for the first "niter" iterations. */
int zero_notlast; /* Don't zero on last iteration? */
size_t ngood; /* Number good samples for stats */
smf_qual_t *pq; /* Pinter to map quality */
unsigned char **mask; /* Address of model's mask pointer */
unsigned char *newmask; /* Individual mask work space */
unsigned char *pm; /* Pointer to next returned mask pixel */
unsigned char *pn; /* Pointer to next new mask pixel */
unsigned char *result; /* Returned mask pointer */
/* Initialise returned values */
result = NULL;
/* Check inherited status. Also check that a map is being created. */
if( *status != SAI__OK || !dat || !dat->map ) return result;
/* Begin an AST context. */
astBegin;
/* Get the sub-keymap containing the configuration parameters for the
requested model. Also get a pointer to the mask array to use (there is
one for each maskable model)*/
if( mtype == SMF__COM ) {
modname = "COM";
mask = &(dat->com_mask);
} else if( mtype == SMF__AST ) {
modname = "AST";
mask = &(dat->ast_mask);
} else if( mtype == SMF__FLT ) {
modname = "FLT";
mask = &(dat->flt_mask);
} else {
modname = NULL;
mask = NULL;
*status = SAI__ERROR;
errRepf( " ", "smf_get_mask: Unsupported model type %d supplied - "
"must be COM, FLT or AST.", status, mtype );
}
subkm = NULL;
astMapGet0A( config, modname, &subkm );
/* Get the "ast.skip" value - when considering "zero_niter" and
"zero_freeze", we only count iterations for which the AST model
is subtracted (i.e. the ones following the initial "ast.skip"
iterations). */
astMapGet0A( config, "AST", &akm );
astMapGet0I( akm, "SKIP", &skip );
akm = astAnnul( akm );
/* Get the number of iterations over which the mask is to be applied. Zero
means all. Return with no mask if this number of iterations has
already been performed. */
zero_niter = 0;
astMapGet0I( subkm, "ZERO_NITER", &zero_niter );
if( zero_niter == 0 || dat->iter < zero_niter + skip ) {
/* Only return a mask if this is not the last iteration, or if ZERO_NOTLAST
is unset. */
zero_notlast = 0;
astMapGet0I( subkm, "ZERO_NOTLAST", &zero_notlast );
if( !( flags & SMF__DIMM_LASTITER ) || !zero_notlast ) {
/* Create a list of the mask types to be combined to get the final mask by
looking for non-default values for the corresponding configuration
parameters in the supplied KeyMap. Static masks (predefined, circles
or external NDFs) may be used on any iteration, but dynamic masks
(lowhits, snr) will only be avialable once the map has been determined
at the end of the first iteration. This means that when masking anything
but the AST model (which is determined after the map), the dynamic masks
cannot be used on the first iteration. Make a note if all masks being
used are static. */
isstatic = 1;
nmask = 0;
zero_lowhits = 0.0;
astMapGet0D( subkm, "ZERO_LOWHITS", &zero_lowhits );
if( zero_lowhits > 0.0 ) {
if( mtype == SMF__AST || !( flags & SMF__DIMM_FIRSTITER ) ) {
mask_types[ nmask++] = LOWHITS;
isstatic = 0;
}
} else if( zero_lowhits < 0.0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter %s.ZERO_LOWHITS (%g) - "
"it must not be negative.", status, modname, zero_lowhits );
}
if( astMapGet1D( subkm, "ZERO_CIRCLE", 3, &zero_c_n, zero_circle ) ) {
if( zero_c_n == 1 || zero_c_n == 3 ) {
mask_types[ nmask++] = CIRCLE;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad number of values (%d) for config parameter "
"%s.ZERO_CIRCLE - must be 1 or 3.", status, zero_c_n,
modname );
}
}
cval = NULL;
astMapGet0C( subkm, "ZERO_MASK", &cval );
if( cval ) {
if( !astChrMatch( cval, "REF" ) &&
!astChrMatch( cval, "MASK2" ) &&
!astChrMatch( cval, "MASK3" ) ) {
astMapGet0I( subkm, "ZERO_MASK", &zero_mask );
cval = ( zero_mask > 0 ) ? "REF" : NULL;
}
if( cval ) {
strcpy( refparam, cval );
astChrCase( NULL, refparam, 1, 0 );
mask_types[ nmask++] = REFNDF;
}
}
zero_snr = 0.0;
astMapGet0D( subkm, "ZERO_SNR", &zero_snr );
if( zero_snr > 0.0 ) {
if( mtype == SMF__AST || !( flags & SMF__DIMM_FIRSTITER ) ) {
mask_types[ nmask++] = SNR;
isstatic = 0;
}
} else if( zero_snr < 0.0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter %s.ZERO_SNR (%g) - "
"it must not be negative.", status, modname, zero_snr );
}
if( astMapHasKey( subkm, "ZERO_MASK_POINTER" ) ) {
astMapGet0P( subkm, "ZERO_MASK_POINTER", (void **) &predef );
if( predef ) mask_types[ nmask++] = PREDEFINED;
}
/* No need to create a mask if no masking was requested or possible. */
if( nmask > 0 ) {
/* Decide if we are using the union or intersection of the masks. */
astMapGet0I( subkm, "ZERO_UNION", &munion );
/* Note if a mask existed on entry. If not, create a mask now, and
initialise it to hold the mask defined by the initial sky map. */
if( *mask == NULL ) {
have_mask = 0;
if( dat->initqual ) {
*mask = astMalloc( dat->msize*sizeof( **mask ) );
if( *mask ) {
pm = *mask;
pq = dat->initqual;
for( i = 0; i < dat->msize; i++ ) {
*(pm++) = ( *(pq++) != 0 );
}
}
} else{
*mask = astCalloc( dat->msize, sizeof( **mask ) );
}
} else {
have_mask = 1;
}
/* If we are combining more than one mask, we need work space to hold
an individual mask independently of the total mask. If we are using
only one mask, then just use the main mask array. */
if( nmask > 1 ) {
newmask = astMalloc( dat->msize*sizeof( *newmask ) );
} else {
newmask = *mask;
}
/* Get the number of iterations after which the mask is to be frozen.
Zero means "never freeze the mask". */
int zero_freeze = 0;
astMapGet0I( subkm, "ZERO_FREEZE", &zero_freeze );
/* Loop round each type of mask to be used. */
for( imask = 0; imask < nmask && *status == SAI__OK; imask++ ){
/* If the mask is now frozen, we just return the existing mask. So leave the
loop. */
if( zero_freeze != 0 && dat->iter > zero_freeze + skip ) {
break;
/* Low hits masking... */
} else if( mask_types[ imask ] == LOWHITS ) {
/* Set hits pixels with 0 hits to VAL__BADI so that stats1 ignores them */
ph = dat->hitsmap;
for( i = 0; i < dat->msize; i++,ph++ ) {
if( *ph == 0 ) *ph = VAL__BADI;
}
/* Find the mean hits in the map */
smf_stats1I( dat->hitsmap, 1, dat->msize, NULL, 0, 0, &meanhits,
NULL, NULL, &ngood, status );
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: mean hits = %lf, ngood "
"= %zd", status, meanhits, ngood );
/* Create the mask */
thresh = meanhits*zero_lowhits;
ph = dat->hitsmap;
pn = newmask;
for( i = 0; i < dat->msize; i++,ph++ ) {
*(pn++) = ( *ph != VAL__BADI && *ph < thresh ) ? 1 : 0;
}
/* Report masking info. */
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: masking %s "
"model at hits = %d.", status, modname, thresh );
/* Circle masking... */
} else if( mask_types[ imask ] == CIRCLE ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the circle
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* If only one parameter supplied it is radius, assume reference
LON/LAT from the frameset to get the centre. If the SkyFrame
represents offsets from the reference position (i.e. the source is
moving), assume the circle is to be centred on the origin. */
if( zero_c_n == 1 ) {
zero_circle[ 2 ] = zero_circle[ 0 ];
skyrefis = astGetC( dat->outfset, "SkyRefIs" );
if( skyrefis && !strcmp( skyrefis, "Origin" ) ) {
zero_circle[ 0 ] = 0.0;
zero_circle[ 1 ] = 0.0;
} else {
zero_circle[ 0 ] = astGetD( dat->outfset, "SkyRef(1)" );
zero_circle[ 1 ] = astGetD( dat->outfset, "SkyRef(2)" );
}
zero_circle[ 0 ] *= AST__DR2D;
zero_circle[ 1 ] *= AST__DR2D;
}
/* The supplied bounds are for pixel coordinates... we need bounds for grid
coordinates which have an offset */
lbnd_grid[ 0 ] = 1;
lbnd_grid[ 1 ] = 1;
ubnd_grid[ 0 ] = dat->ubnd_out[ 0 ] - dat->lbnd_out[ 0 ] + 1;
ubnd_grid[ 1 ] = dat->ubnd_out[ 1 ] - dat->lbnd_out[ 1 ] + 1;
/* Coordinates & radius of the circular region converted from degrees
to radians */
centre[ 0 ] = zero_circle[ 0 ]*AST__DD2R;
centre[ 1 ] = zero_circle[ 1 ]*AST__DD2R;
radius[ 0 ] = zero_circle[ 2 ]*AST__DD2R;
/* Create the Circle, defined in the current Frame of the FrameSet (i.e.
the sky frame). */
circle = astCircle( astGetFrame( dat->outfset, AST__CURRENT), 1,
centre, radius, NULL, " " );
/* Fill the mask with zeros. */
memset( newmask, 0, sizeof( *newmask )*dat->msize );
/* Get the mapping from the sky frame (current) to the grid frame (base),
and then set the mask to 1 for all of the values outside this circle */
astMaskUB( circle, astGetMapping( dat->outfset, AST__CURRENT,
AST__BASE ),
0, 2, lbnd_grid, ubnd_grid, newmask, 1 );
/* Report masking info. */
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model will"
" be masked %s using a circle of "
"radius %g arc-secs, centred at %s=%s, %s=%s.",
status, modname, words, radius[0]*AST__DR2D*3600,
astGetC( dat->outfset, "Symbol(1)" ),
astFormat( dat->outfset, 1, centre[ 0 ] ),
astGetC( dat->outfset, "Symbol(2)" ),
astFormat( dat->outfset, 2, centre[ 1 ] ) );
}
/* Reference NDF masking... */
} else if( mask_types[ imask ] == REFNDF ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the NDF
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* Begin an NDF context. */
ndfBegin();
/* Get an identifier for the NDF using the associated ADAM parameter. */
ndfAssoc( refparam, "READ", &indf1, status );
/* Get a section from this NDF that matches the bounds of the map. */
ndfSect( indf1, 2, dat->lbnd_out, dat->ubnd_out, &indf2,
status );
/* Map the section. */
ndfMap( indf2, "DATA", "_DOUBLE", "READ", (void **) &ptr,
&nel, status );
/* Check we can use the pointer safely. */
if( *status == SAI__OK ) {
/* Find bad pixels in the NDF and set those pixels to 1 in the mask. */
pn = newmask;
for( i = 0; i < dat->msize; i++ ) {
*(pn++) = ( *(ptr++) == VAL__BADD ) ? 1 : 0;
}
/* Report masking info. */
ndfMsg( "N", indf2 );
msgSetc( "M", modname );
if( zero_niter == 0 ) {
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The ^M "
"model will be masked on each iteration "
"using the bad pixels in NDF '^N'.",
status );
} else {
msgSeti( "I", zero_niter );
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The ^M "
"model will be masked for ^I iterations "
"using the bad pixels in NDF '^N'.",
status );
}
}
/* End the NDF context. */
ndfEnd( status );
}
/* SNR masking... */
} else if( mask_types[ imask ] == SNR ) {
/* Get the lower SNR limit. */
zero_snrlo = 0.0;
astMapGet0D( subkm, "ZERO_SNRLO", &zero_snrlo );
if( zero_snrlo <= 0.0 ) {
zero_snrlo = zero_snr;
} else if( zero_snrlo > zero_snr && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter "
"%s.ZERO_SNRLO (%g) - it must not be higher "
"than %s.ZERO_SNR (%g).", status, modname,
zero_snrlo, modname, zero_snr );
}
/* If the higher and lower SNR limits are equal, just do a simple
threshold on the SNR values to get the mask. */
if( zero_snr == zero_snrlo ) {
pd = dat->map;
pv = dat->mapvar;
pn = newmask;
for( i = 0; i < dat->msize; i++,pd++,pv++ ) {
*(pn++) = ( *pd != VAL__BADD && *pv != VAL__BADD &&
*pv >= 0.0 && *pd < zero_snr*sqrt( *pv ) ) ? 1 : 0;
}
/* Report masking info. */
if( !have_mask ) {
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using an SNR limit of %g.",
status, modname, words, zero_snr );
}
/* If the higher and lower SNR limits are different, create an initial
mask by thresholding at the ZERO_SNR value, and then extend the source
areas within the mask down to an SNR limit of ZERO_SNRLO. */
} else {
smf_snrmask( wf, dat->map, dat->mapvar, dat->mdims,
zero_snr, zero_snrlo, newmask, status );
/* Report masking info. */
if( !have_mask ) {
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using an SNR limit of %g "
"extended down to %g.", status, modname,
words, zero_snr, zero_snrlo );
}
}
/* Predefined masking... */
} else if( mask_types[ imask ] == PREDEFINED ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* Find bad pixels in the predefined array and set those pixels to 1 in
the mask. */
pn = newmask;
for( i = 0; i < dat->msize; i++ ) {
*(pn++) = ( *(predef++) == VAL__BADD ) ? 1 : 0;
}
/* Report masking info. */
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using a smoothed form of "
"the final mask created with the previous map.",
status, modname, words );
}
}
/* If required, add the new mask into the returned mask. If this is the
first mask, we just copy the new mask to form the returned mask.
Otherwise, we combine it with the existing returned mask. */
if( ! have_mask || ! isstatic ) {
if( nmask > 1 ) {
if( imask == 0 ) {
memcpy( *mask, newmask, dat->msize*sizeof(*newmask));
} else {
pm = *mask;
pn = newmask;
if( munion ) {
for( i = 0; i < dat->msize; i++,pm++ ) {
if( *(pn++) == 0 ) *pm = 0;
}
} else {
for( i = 0; i < dat->msize; i++,pm++ ) {
if( *(pn++) == 1 ) *pm = 1;
}
}
}
}
}
}
/* Free the individual mask work array if it was used. */
if( nmask > 1 ) newmask = astFree( newmask );
/* Check that the mask has some source pixels (i.e. pixels that have non-bad data values -
we do not also check variance values since they are not available until the second
iteration). */
if( *status == SAI__OK ) {
nsource = 0;
pm = *mask;
pd = dat->map;
for( i = 0; i < dat->msize; i++,pd++,pv++,pm++ ) {
if( *pd != VAL__BADD && *pm == 0 ) nsource++;
}
if( nsource < 5 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", "The %s mask being used has fewer than 5 "
"source pixels.", status, modname );
if( zero_snr > 0.0 ) {
errRepf( "", "Maybe your zero_snr value (%g) is too high?",
status, zero_snr );
}
}
}
/* Return the mask pointer if all has gone well. */
if( *status == SAI__OK ) result = *mask;
}
}
}
/* End the AST context, annulling all AST Objects created in the context. */
astEnd;
/* Return the pointer to the boolean mask. */
return result;
}
void smf_subip( ThrWorkForce *wf, smfArray *res, smfArray *lut, int *lbnd_out,
int *ubnd_out, AstKeyMap *keymap, AstFrameSet *outfs, int *status ) {
/* Local Variables: */
HDSLoc *loc = NULL;
HDSLoc *sloc = NULL;
SmfSubIPData *job_data = NULL;
SmfSubIPData *pdata;
char ipref[200];
char subname[10];
const char *ipdata;
const char *qu;
dim_t bolostep;
dim_t nbolo;
dim_t ntslice;
double *angcdata;
double *c0data;
double *imapdata;
double *ipang;
double *p0data;
double *p1data;
int angcndf;
int c0ndf;
int imapndf;
int iw;
int nmap;
int nw;
int p0ndf;
int p1ndf;
size_t bstride;
size_t idx;
size_t tstride;
smfData *data;
smf_qual_t *qua_data;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Check if we have pol2 data, and see if it is Q or U. */
qu = NULL;
for( idx = 0; idx < res->ndat; idx++ ) {
data = res->sdata[idx];
if( !strcmp( data->hdr->dlabel, "Q" ) ){
if( !qu ) {
qu = "Q";
} else if( strcmp( qu, "Q" ) ) {
*status = SAI__ERROR;
break;
}
} else if( !strcmp( data->hdr->dlabel, "U" ) ) {
if( !qu ) {
qu = "U";
} else if( strcmp( qu, "U" ) ) {
*status = SAI__ERROR;
break;
}
} else if( qu ) {
*status = SAI__ERROR;
qu = NULL;
break;
}
}
/* Report an error if there is a mix of pol2 and non-pol2, or a mix of Q
and U. */
if( *status != SAI__OK ) {
if( qu ) {
errRep( "", "smf_subip: Input data contains mix of Q and U "
"data", status );
} else {
errRep( "", "smf_subip: Input data contains mix of POL2 and "
"non-POL2 data", status );
}
/* If we have pol2 data, get the path to the total intensity image that
is to be used to define the level of IP correction required. If no
value is supplied, annul the error and set "qu" NULL to indicate we should
leave immediately. */
} else if( qu && *status == SAI__OK ) {
parGet0c( "IPREF", ipref, sizeof(ipref), status );
if( *status == PAR__NULL ) {
errAnnul( status );
qu = NULL;
}
}
/* If we are applying IP correction... */
if( qu && *status == SAI__OK ) {
msgOutf( "", "smf_subip: applying instrumental polarisation %s "
"correction based on total intensity map `%s'", status,
qu, ipref );
/* Get an identifier for the IPREF NDF. */
ndfFind( NULL, ipref, &imapndf, status );
/* Resample the NDFs data values onto the output map grid. */
imapdata = smf_alignndf( imapndf, outfs, lbnd_out, ubnd_out,
status );
/* Annul the NDF identifier. */
ndfAnnul( &imapndf, status );
/* Create structures used to pass information to the worker threads. */
nw = wf ? wf->nworker : 1;
job_data = astMalloc( nw*sizeof( *job_data ) );
/* Get the path to the container file holding the IP model parameters. */
ipdata = "$STARLINK_DIR/share/smurf/ipdata.sdf";
astMapGet0C( keymap, "IPDATA", &ipdata );
/* Open the container file. */
hdsOpen( ipdata, "READ", &loc, status );
/* Do the IP correction for each subarray (s8a, s8b, etc) in turn. */
for( idx = 0; idx < res->ndat && *status == SAI__OK; idx++ ) {
data = res->sdata[idx];
/* Get an array holding the angle (rad.s) from north to focal plane Y,
measured positive in the sense of rotation from focal plane Y to focal
plane X, for every bolometer sample in the smfData. The values are bolo
ordered so that "bstride" is 1 and "tstsride" is nbolo. */
ipang = smf1_calcang( data, status );
/* Get the number of bolometers and time slices for the current subarray,
together with the strides between adjacent bolometers and adjacent
time slices. */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &bstride,
&tstride, status );
/* Get a locator for the structure holding the IP parameters for the
current subarray */
smf_find_subarray( data->hdr, subname, sizeof( subname ), NULL,
status );
datFind( loc, subname, &sloc, status );
/* Begin an NDF context. */
ndfBegin();
/* Get identifiers for the NDFs holding the individual parameters. Each
NDF holds a parameter value for each bolometer. */
ndfFind( sloc, "P0", &p0ndf, status );
ndfFind( sloc, "P1", &p1ndf, status );
ndfFind( sloc, "C0", &c0ndf, status );
ndfFind( sloc, "ANGC", &angcndf, status );
/* Map them. Check each one has the expected number of elements. */
ndfMap( p0ndf, "DATA", "_DOUBLE", "READ", (void **) &p0data,
&nmap, status );
if( nmap != (int) nbolo && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", p0ndf );
errRep( "", "smf_subip: Bad dimensions for ^N - should be 32x40.", status );
}
ndfMap( p1ndf, "DATA", "_DOUBLE", "READ", (void **) &p1data,
&nmap, status );
if( nmap != (int) nbolo && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", p1ndf );
errRep( "", "smf_subip: Bad dimensions for ^N - should be 32x40.", status );
}
ndfMap( c0ndf, "DATA", "_DOUBLE", "READ", (void **) &c0data,
&nmap, status );
if( nmap != (int) nbolo && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", c0ndf );
errRep( "", "smf_subip: Bad dimensions for ^N - should be 32x40.", status );
}
ndfMap( angcndf, "DATA", "_DOUBLE", "READ", (void **) &angcdata,
&nmap, status );
if( nmap != (int) nbolo && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", angcndf );
errRep( "", "smf_subip: Bad dimensions for ^N - should be 32x40.", status );
}
/* Get a pointer to the quality array for the residuals. */
qua_data = smf_select_qualpntr( data, NULL, status );
/* See how many bolometers to process in each thread. */
bolostep = nbolo/nw;
if( bolostep == 0 ) bolostep = 1;
/* Create jobs to apply the IP correction to a range of bolometers. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
/* Set the range of bolometers (b1 to b2) to be processed by the current
job. */
pdata->b1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->b2 = pdata->b1 + bolostep - 1;
} else {
pdata->b2 = nbolo - 1 ;
}
/* Store the other info needed by the worker thread. */
pdata->ntslice = ntslice;
pdata->nbolo = nbolo;
pdata->res_data = res->sdata[idx]->pntr[0];
pdata->lut_data = lut->sdata[idx]->pntr[0];
pdata->qua_data = qua_data;
pdata->ipang = ipang;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->imapdata = imapdata;
pdata->qu = qu;
pdata->p0data = p0data;
pdata->p1data = p1data;
pdata->c0data = c0data;
pdata->angcdata = angcdata;
pdata->allstate = data->hdr->allState;
/* Submit the job for execution by the next available thread. */
thrAddJob( wf, 0, pdata, smf1_subip, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* End the NDF context, thus unmapping and freeing all NDF identifiers
created since the context was started. */
ndfEnd( status );
/* Free locator for subarray IP parameters. */
datAnnul( &sloc, status );
ipang = astFree( ipang );
}
/* Free resources. */
datAnnul( &loc, status );
imapdata = astFree( imapdata );
job_data = astFree( job_data );
}
}
/* Main entry point. */
void smf_calcmodel_smo( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel,
int flags __attribute__((unused)),
int *status) {
/* Local Variables */
size_t bstride; /* bolo stride */
dim_t boxcar = 0; /* size of boxcar smooth window */
smf_filt_t filter_type; /* The type of smoothing to perform */
size_t i; /* Loop counter */
dim_t idx=0; /* Index within subgroup */
int iworker; /* Owkrer index */
smfCalcmodelSmoJobData *job_data=NULL; /* Pointer to all job data structures */
AstKeyMap *kmap=NULL; /* Pointer to PLN-specific keys */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
double *model_data_copy=NULL; /* Copy of model_data for one bolo */
dim_t nbolo=0; /* Number of bolometers */
dim_t ndata=0; /* Total number of data points */
int notfirst=0; /* flag for delaying until after 1st iter */
dim_t ntslice=0; /* Number of time slices */
int nworker; /* No. of worker threads in supplied Workforce */
smfCalcmodelSmoJobData *pdata=NULL; /* Pointer to current data structure */
smfArray *qua=NULL; /* Pointer to QUA at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to quality data */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
int step; /* Number of bolometers per thread */
size_t tstride; /* Time slice stride in data array */
const char * typestr = NULL; /* smo.type value */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
/* Obtain pointer to sub-keymap containing PLN parameters. Something will
always be available.*/
astMapGet0A( keymap, "SMO", &kmap );
/* Are we skipping the first iteration? */
astMapGet0I(kmap, "NOTFIRST", ¬first);
if( notfirst && (flags & SMF__DIMM_FIRSTITER) ) {
msgOutif( MSG__VERB, "", FUNC_NAME
": skipping SMO this iteration", status );
return;
}
/* Get the boxcar size */
if( kmap ) smf_get_nsamp( kmap, "BOXCAR", res->sdata[0], &boxcar, status );
/* Get the type of smoothing filter to use. Anthing that is not "MEDIAN" is mean */
filter_type = SMF__FILT_MEAN;
if (astMapGet0C( kmap, "TYPE", &typestr ) ) {
if (strncasecmp( typestr, "MED", 3 ) == 0 ) {
filter_type = SMF__FILT_MEDIAN;
}
}
/* Assert bolo-ordered data */
smf_model_dataOrder( wf, dat, allmodel, chunk, SMF__RES|SMF__QUA,
0, status );
smf_get_dims( res->sdata[0], NULL, NULL, NULL, &ntslice,
&ndata, NULL, NULL, status);
model = allmodel[chunk];
msgOutiff(MSG__VERB, "",
" Calculating smoothed model using boxcar of width %" DIM_T_FMT " time slices",
status, boxcar);
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
/* Loop over index in subgrp (subarray) and put the previous iteration
of the filtered component back into the residual before calculating
and removing the new filtered component */
for( idx=0; (*status==SAI__OK)&&(idx<res->ndat); idx++ ) {
/* Obtain dimensions of the data */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice,
&ndata, &bstride, &tstride, status);
/* Get pointers to data/quality/model */
res_data = (res->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Uncomment to aid debugging */
/*
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_in",
NULL, 0, 0, MSG__VERB, 0, status );
*/
if( *status == SAI__OK ) {
/* Place last iteration back into residual if this is a smoothable section of the time series */
for (i=0; i< ndata; i++) {
if ( !(qua_data[i]&SMF__Q_FIT) && res_data[i] != VAL__BADD && model_data[i] != VAL__BADD ) {
res_data[i] += model_data[i];
}
}
}
/* Uncomment to aid debugging */
/*
smf_write_smfData( model->sdata[idx], NULL, qua_data, "model_b4",
NULL, 0, 0, MSG__VERB, 0, status );
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_b4",
NULL, 0, 0, MSG__VERB, 0, status );
*/
/* Determine which bolometers are to be processed by which threads. */
step = nbolo/nworker;
if( step < 1 ) step = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->b1 = iworker*step;
pdata->b2 = pdata->b1 + step - 1;
}
/* Ensure that the last thread picks up any left-over bolometers */
pdata->b2 = nbolo - 1;
/* Store all the other info needed by the worker threads, and submit the
jobs to apply the smoothing. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->boxcar = boxcar;
pdata->bstride = bstride;
pdata->bstride = bstride;
pdata->filter_type = filter_type;
pdata->model_data = model_data;
pdata->nbolo = nbolo;
pdata->nbolo = nbolo;
pdata->ntslice = ntslice;
pdata->ntslice = ntslice;
pdata->qua_data = qua_data;
pdata->qua_data = qua_data;
pdata->res_data = res_data;
pdata->res_data = res_data;
pdata->tstride = tstride;
pdata->tstride = tstride;
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_calcmodel_smo_job,
0, NULL, status );
}
thrWait( wf, status );
/* Uncomment to aid debugging */
/*
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_af",
NULL, 0, 0, MSG__VERB, 0, status );
smf_write_smfData( model->sdata[idx], NULL, qua_data, "model_af",
NULL, 0, 0, MSG__VERB, 0, status );
*/
}
}
/* Free work space (astFree returns without action if a NULL pointer is
supplied). */
model_data_copy = astFree( model_data_copy );
job_data = astFree( job_data );
/* Annul AST Object pointers (astAnnul reports an error if a NULL pointer
is supplied). */
if( kmap ) kmap = astAnnul( kmap );
}
F77_SUBROUTINE(configecho)( INTEGER(STATUS) ){
/*
*+
* Name:
* CONFIGECHO
* Purpose:
* Displays one or more configuration parameters.
* Language:
* C (designed to be called from Fortran)
* Type of Module:
* ADAM A-task
* Invocation:
* CALL CONFIGECHO( STATUS )
* Arguments:
* STATUS = INTEGER (Given and Returned)
* The global status.
* Description:
* This application displays the name and value of one or all
* configuration parameters, specified using Parameters CONFIG or
* NDF. If a single parameter is displayed, its value is also
* written to an output parameter. If the parameter value is not
* specified by the CONFIG, NDF or DEFAULTS parameter, then the
* value supplied for DEFVAL is displayed.
*
* If an input NDF is supplied then configuration parameters
* are read from its history (see Parameters NDF and APPLICATION).
*
* If values are supplied for both CONFIG and NDF, then the
* differences between the two sets of configuration parameters
* are displayed (see Parameter NDF).
* Usage:
* configecho name config [defaults] [select] [defval]
* ADAM Parameters:
* APPLICATION = LITERAL (Read)
* When reading configuration parameters from the history
* of an NDF, this parameter specifies the name of the application
* to find in the history. There must be a history component
* corresponding to the value of this parameter, and it must
* include a CONFIG group. [current value]
* CONFIG = GROUP (Read)
* Specifies values for the configuration parameters. If the string
* "def" (case-insensitive) or a null (!) value is supplied, the
* configuration parameters are obtained using Parameter NDF. If
* a null value is also supplied for NDF, a set of default
* configuration parameter values will be used, as specified by
* Parameter DEFAULTS.
*
* The supplied value should be either a comma-separated list of
* strings or the name of a text file preceded by an up-arrow
* character "^", containing one or more comma-separated lists of
* strings. Each string is either a "keyword=value" setting, or the
* name of a text file preceded by an up-arrow character "^". Such
* text files should contain further comma-separated lists which
* will be read and interpreted in the same manner (any blank lines
* or lines beginning with "#" are ignored). Within a text file,
* newlines can be used as delimiters, as well as commas. Settings
* are applied in the order in which they occur within the list,
* with later settings overriding any earlier settings given for
* the same keyword.
*
* Each individual setting should be of the form "<keyword>=<value>".
* If a non-null value is supplied for Parameter DEFAULTS, an error
* will be reported if CONFIG includes values for any parameters
* that are not included in DEFAULTS.
* DEFAULTS = LITERAL (Read)
* The path to a file containing the default value for every
* allowed configuration parameter. If null (!) is supplied, no
* defaults will be supplied for parameters that are not specified
* by CONFIG, and no tests will be performed on the validity of
* paramter names supplied by CONFIG. [!]
* DEFVAL = LITERAL (Read)
* The value to return if no value can be obtained for the named
* parameter, or if the value is "<undef>". [<***>]
* NAME = LITERAL (Read)
* The name of the configuration parameter to display. If set to
* null (!), then all parameters defined in the configuration are
* displayed.
* NDF = NDF (Read)
* An NDF file containing history entries which include
* configuration parameters. If not null (!) the history
* of the NDF will be searched for a component corresponding
* to the Parameter APPLICATION. The Parameter CONFIG
* is then optional, but if it too is not null (!) then
* the output will show the differences between the configuration
* stored in the NDF history and the given configuration:
* new parameters and those different from the reference
* configuration (given by Parameter CONFIG) are prefixed
* with "+" and those which are the same as the reference
* configuration are prefixed with "-". [!]
* SELECT = GROUP (Read)
* A group that specifies any alternative prefixes that can be
* included at the start of any parameter name. For instance, if
* this group contains the two entries "450=1" and "850=0", then
* either CONFIG or DEFAULTS can specify two values for any single
* parameter -- one for the parameter prefixed by "450." and another
* for the parameter prefixed by "850.". Thus, for instance, if
* DEFAULTS defines a parameter called "filter", it could include
* "450.filter=300" and "850.filter=600". The CONFIG parameter could
* then either set the filter parameter for a specific prefix (as
* in "450.filter=234"); or it could leave the prefix unspecified,
* in which case the prefix used is the first one with a
* non-zero value in SELECT (450 in the case of this example - 850
* has a value zero in SELECT). Thus the names of the items in
* SELECT define the set of allowed alternative prefixes, and the
* values indicate which one of these alternatives is to be used
* (the first one with non-zero value). [!]
* SORT = _LOGICAL (Read)
* If TRUE then sort the listed parameters in to alphabetical order.
* Otherwise, retain the order they have in the supplied
* configuration. Only used if a null (!) value is supplied for
* Parameter NAME. [FALSE]
* VALUE = LITERAL (Write)
* The value of the configuration parameter, or "<***>" if the
* parameter has no value in CONFIG and DEFAULTS.
* Examples:
* configecho m81 ^myconf
* Report the value of configuration parameter "m81" defined within
* the file "myconf". If the file does not contain a value for
* "m81", then "<***>" is displayed.
* configecho type ^myconf select="m57=0,m31=1,m103=0"
* Report the value of configuration parameter "type" defined within
* the file "myconf". If the file does not contain a value for
* "type", then the value of "m31.type" will be reported instead. If
* neither is present, then "<***>" is displayed.
* configecho flt.filt_edge_largescale \
* config=^/star/share/smurf/dimmconfig.lis \
* defaults=/star/bin/smurf/smurf_makemap.def \
* select="450=1,850=0"
* Report the value of configuration parameter "flt.filt_edge_largescale"
* defined within the file "/star/share/smurf/dimmconfig.lis", using
* defaults from the file "/star/bin/smurf/smurf_makemap.def". If
* dimmconfig.lis does not contain a value for "flt.filt_edge_largescale"
* then it is searched for "450.flt.filt_edge_largescale" instead. An
* error is reported if dimmconfig.lis contains values for any
* items that are not defined in smurf_makemap.def.
* configecho ndf=omc1 config=^/star/share/smurf/dimmconfig.lis \
* defaults=/star/bin/smurf/smurf_makemap.def \
* application=makemap name=! sort select="450=0,850=1"
* Show how the configuration used to generate the 850um map
* of OMC1 differs from the basic dimmconfig.lis file.
* Copyright:
* Copyright (C) 2012-3 Science & Technology Facilities Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either Version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
* Authors:
* DSB: David S. Berry
* GSB: Graham S. Bell
* {enter_new_authors_here}
* History:
* 10-DEC-2012 (DSB):
* Original version.
* 6-FEB-2013 (DSB):
* Added parameter DEFVAL.
* 11-FEB-2013 (DSB):
* Added parameter SORT and allow all parameters to be listed by
* providing a null value for NAME.
* 11-FEB-2013 (GSB):
* Added ability to read configuration from history entries.
* 13-FEB-2013 (DSB):
* Nullify AST object pointers when the objects are annulled,
* to avoid re-use of dead pointers.
* 14-FEB-2013 (DSB):
* Allow the SELECT feature to be used even if no DEFAULTS file is
* supplied (see the new entry in the "Examples:" section).
* 15-FEB-2013 (DSB):
* Expand the prologue docs, and use NULL in place of zero for pointers.
* 22-FEB-2013 (DSB):
* Guard against seg fault in HistoryKeymap when the NDF does
* not contain the required CONFIG entry in the History
* component.
* {enter_further_changes_here}
*-
*/
GENPTR_INTEGER(STATUS)
/* Local Variables: */
AstKeyMap *keymap2;
AstKeyMap *keymap;
Grp *grp = NULL;
char *dot;
char *pname;
char defs[250];
char defval[250];
char name[250];
const char *value;
const char *historyValue = NULL;
int showall;
int sort;
size_t size;
int indf = 0;
int nrec;
int i;
char application[NDF__SZAPP];
char applicationi[NDF__SZAPP];
/* Abort if an error has already occurred. */
if( *STATUS != SAI__OK ) return;
/* Begin an AST context */
astBegin;
/* Get the value to return if no value can be obtained for the named
parameter, of it it has a value of <undef>. */
parGet0c( "DEFVAL", defval, sizeof(defval), STATUS );
/* Get any defaults file, annuling the error if null (!) is supplied. */
if( *STATUS == SAI__OK ) {
parGet0c( "DEFAULTS", defs, sizeof(defs), STATUS );
if( *STATUS == PAR__NULL ) {
errAnnul( STATUS );
defs[0] = 0;
}
}
/* Get the NDF identifier if requested. */
ndfBegin();
if (*STATUS == SAI__OK) {
ndfAssoc("NDF", "READ", &indf, STATUS);
if (*STATUS == PAR__NULL) {
errAnnul(STATUS);
indf = 0;
}
else {
parGet0c("APPLICATION", application, sizeof(application), STATUS);
/* Check now for error because the block below allowing an undefined
* CONFIG clears this status otherwise. */
if (*STATUS != SAI__OK) goto L999;
}
}
/* See if any alternate keyword prefixes are allowed, and if so determine
which of the alternatices is to be displayed. */
kpg1Gtgrp( "SELECT", &grp, &size, STATUS );
if( *STATUS == PAR__NULL ) {
grpDelet( &grp, STATUS );
errAnnul( STATUS );
keymap2 = NULL;
} else {
kpg1Kymap( grp, &keymap2, STATUS );
grpDelet( &grp, STATUS );
}
/* Create a KeyMap holding the selected alternative for each keyword, and
also supply defaults for any missing values (if a defaults file was
supplied by the user). */
keymap = kpg1Config( "CONFIG", defs[0]?defs:NULL, keymap2, 0, STATUS );
/* Allow it to be NULL if we're reading an NDF because we'll replace
keymap with historyConfig later if necessary. */
if( indf && *STATUS == PAR__NULL ) {
errAnnul(STATUS);
keymap = NULL;
}
/* Abort if an error has occurred. */
if( *STATUS != SAI__OK ) goto L999;
/* Get the name of the required parameter, and convert to upper case (if
supplied). If not supplied, set a flag indicating that all parameters
should be displayed. */
parGet0c( "NAME", name, sizeof(name), STATUS );
if( *STATUS == PAR__NULL ) {
errAnnul( STATUS );
showall = 1;
} else {
showall = 0;
astChrCase( NULL, name, 1, 0 );
}
/* Attempt to find the NDF's corresponding history record. */
if (indf && *STATUS == SAI__OK) {
ndfHnrec(indf, &nrec, STATUS);
for (i = 0; i < nrec; i ++) {
ndfHinfo(indf, "APPLICATION", i + 1, applicationi,
sizeof(applicationi), STATUS);
if (! strncasecmp(application, applicationi, strlen(application))) {
ndfHout(indf, i + 1, HistoryKeyMap, STATUS);
break;
}
}
if (*STATUS == SAI__OK && ! historyConfig) {
*STATUS = SAI__ERROR;
errRepf("CONFIGECHO_ERR", "CONFIGECHO: Failed to find %s "
"configuration in NDF history.", STATUS, application);
}
else if (! keymap) {
keymap = historyConfig;
historyConfig = NULL;
}
}
if( *STATUS == SAI__OK ) {
/* First deal with cases where we are displaying a single parameter
value. */
if( !showall ) {
/* Loop round each section of the name that ends with a dot. */
value = defval;
pname = name;
dot = strchr( pname, '.' );
while( dot && keymap ) {
/* Get a nested keymap with the name that occurs prior to the dot. If
found, use it in place of the parent keymap. */
pname[ dot - pname ] = 0;
if( astMapGet0A( keymap, pname, &keymap2 ) ) {
astAnnul( keymap );
keymap = keymap2;
} else {
keymap = astAnnul( keymap );
}
/* If historyConfig exists, do the same there. */
if (historyConfig) {
if (astMapGet0A(historyConfig, pname, &keymap2)) {
astAnnul(historyConfig);
historyConfig = keymap2;
}
else {
historyConfig = astAnnul(historyConfig);
}
}
/* Re-instate the original dot, and move on to find the next dot. */
pname[ dot - pname ] = '.';
pname = dot + 1;
dot = strchr( pname, '.' );
}
/* Ensure no error is reported if the parameter is not found in the
KeyMap. */
if( keymap ) {
astClear( keymap, "KeyError" );
/* Get the parameter value as a string. */
astMapGet0C( keymap, pname, &value );
}
if (historyConfig) {
astClear(historyConfig, "KeyError");
astMapGet0C(historyConfig, pname, &historyValue);
/* In NDF history mode we only want to return a value if it
was found in the configuration from the history. */
if (historyValue) {
if (strcmp(value, historyValue)) {
msgOutf("", "+ %s", STATUS, historyValue);
}
else {
msgOutf("", "- %s", STATUS, historyValue);
}
parPut0c("VALUE", historyValue, STATUS);
}
}
else {
/* Display it. */
msgOut( "", value, STATUS );
/* Write it to the output parameter. */
parPut0c( "VALUE", value, STATUS );
}
/* Now deal with cases were we are displaying all parameter values. */
} else {
/* See if the values should be sorted. */
parGet0l( "SORT", &sort, STATUS );
/* Display them. */
if (historyConfig) {
DisplayKeyMap( historyConfig , sort, "", keymap, STATUS );
}
else {
DisplayKeyMap( keymap, sort, "", NULL, STATUS );
}
}
}
/* Tidy up. */
L999:;
/* End the AST context */
astEnd;
/* Close the NDF if open. */
ndfEnd(STATUS);
/* If an error has occurred, issue another error report identifying the
program which has failed (i.e. this one). */
if( *STATUS != SAI__OK ) {
errRep( "CONFIGECHO_ERR", "CONFIGECHO: Failed to echo configuration "
"parameters.", STATUS );
}
}
