void smf_find_science(const Grp * ingrp, Grp **outgrp, int reverttodark,
Grp **darkgrp, Grp **flatgrp, int reducedark,
int calcflat, smf_dtype darktype, smfArray ** darks,
smfArray **fflats, AstKeyMap ** heateffmap,
double * meanstep, int * status ) {
smfSortInfo *alldarks; /* array of sort structs for darks */
smfSortInfo *allfflats; /* array of fast flat info */
Grp * dgrp = NULL; /* Internal dark group */
double duration_darks = 0.0; /* total duration of all darks */
double duration_sci = 0.0; /* Duration of all science observations */
size_t dkcount = 0; /* Dark counter */
size_t ffcount = 0; /* Fast flat counter */
Grp * fgrp = NULL; /* Fast flat group */
size_t i; /* loop counter */
smfData *infile = NULL; /* input file */
size_t insize; /* number of input files */
size_t nsteps_dark = 0; /* Total number of steps for darks */
size_t nsteps_sci = 0; /* Total number of steps for science */
AstKeyMap * heatermap = NULL; /* Heater efficiency map */
AstKeyMap * obsmap = NULL; /* Info from all observations */
AstKeyMap * objmap = NULL; /* All the object names used */
AstKeyMap * scimap = NULL; /* All non-flat obs indexed by unique key */
Grp *ogrp = NULL; /* local copy of output group */
size_t sccount = 0; /* Number of accepted science files */
struct timeval tv1; /* Timer */
struct timeval tv2; /* Timer */
if (meanstep) *meanstep = VAL__BADD;
if (outgrp) *outgrp = NULL;
if (darkgrp) *darkgrp = NULL;
if (darks) *darks = NULL;
if (fflats) *fflats = NULL;
if (heateffmap) *heateffmap = NULL;
if (*status != SAI__OK) return;
/* Sanity check to make sure we return some information */
if ( outgrp == NULL && darkgrp == NULL && darks == NULL && fflats == NULL) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": Must have some non-NULL arguments"
" (possible programming error)", status);
return;
}
/* Start a timer to see how long this takes */
smf_timerinit( &tv1, &tv2, status );
/* Create new group for output files */
ogrp = smf_grp_new( ingrp, "Science", status );
/* and a new group for darks */
dgrp = smf_grp_new( ingrp, "DarkFiles", status );
/* and for fast flats */
fgrp = smf_grp_new( ingrp, "FastFlats", status );
/* and also create a keymap for the observation description */
obsmap = astKeyMap( "KeyError=1" );
/* and an object map */
objmap = astKeyMap( "KeyError=1" );
/* This keymap contains the sequence counters for each related
subarray/obsidss/heater/shutter combination and is used to decide
if a bad flat is relevant */
scimap = astKeyMap( "KeyError=1,KeyCase=0" );
/* This keymap is used to contain relevant heater efficiency data */
heatermap = astKeyMap( "KeyError=1,KeyCase=0" );
/* Work out how many input files we have and allocate sufficient sorting
space */
insize = grpGrpsz( ingrp, status );
alldarks = astCalloc( insize, sizeof(*alldarks) );
allfflats = astCalloc( insize, sizeof(*allfflats) );
/* check each file in turn */
for (i = 1; i <= insize; i++) {
int seqcount = 0;
char keystr[100]; /* Key for scimap entry */
/* open the file but just to get the header */
smf_open_file( ingrp, i, "READ", SMF__NOCREATE_DATA, &infile, status );
if (*status != SAI__OK) break;
/* Fill in the keymap with observation details */
smf_obsmap_fill( infile, obsmap, objmap, status );
/* Find the heater efficiency map if required */
if (*status == SAI__OK && heateffmap) {
char arrayidstr[32];
smf_fits_getS( infile->hdr, "ARRAYID", arrayidstr, sizeof(arrayidstr),
status );
if (!astMapHasKey( heatermap, arrayidstr ) ) {
smfData * heateff = NULL;
dim_t nbolos = 0;
smf_flat_params( infile, "RESIST", NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, &heateff, status );
smf_get_dims( heateff, NULL, NULL, &nbolos, NULL, NULL, NULL, NULL,
status );
if (heateff) astMapPut0P( heatermap, arrayidstr, heateff, NULL );
}
}
/* Get the sequence counter for the file. We do not worry about
duplicate sequence counters (at the moment) */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* The key identifying this subarray/obsidss/heater/shutter combo */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
if (smf_isdark( infile, status )) {
/* Store the sorting information */
dkcount = smf__addto_sortinfo( infile, alldarks, i, dkcount, "Dark", status );
smf__addto_durations( infile, &duration_darks, &nsteps_dark, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
} else {
/* compare sequence type with observation type and drop it (for now)
if they differ */
if ( infile->hdr->obstype == infile->hdr->seqtype ) {
/* Sanity check the header for corruption. Compare RTS_NUM with SEQSTART
and SEQEND. The first RTS_NUM must either be SEQSTART or else between
SEQSTART and SEQEND (if someone has giving us a section) */
int seqstart = 0;
int seqend = 0;
int firstnum = 0;
JCMTState *tmpState = NULL;
smf_getfitsi( infile->hdr, "SEQSTART", &seqstart, status );
smf_getfitsi( infile->hdr, "SEQEND", &seqend, status );
tmpState = infile->hdr->allState;
if( tmpState ) {
firstnum = (tmpState[0]).rts_num;
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
if ( firstnum >= seqstart && firstnum <= seqend ) {
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif(MSG__DEBUG, " ", "Non-dark file: ^F",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
} else {
msgOutif( MSG__QUIET, "",
"File ^F has a corrupt FITS header. Ignoring it.",
status );
}
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif( MSG__DEBUG, " ",
"File ^F lacks JCMTState: assuming it is non-dark",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
}
} else if (infile->hdr->seqtype == SMF__TYP_FASTFLAT ) {
ffcount = smf__addto_sortinfo( infile, allfflats, i, ffcount, "Fast flat", status );
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
msgOutif(MSG__DEBUG, " ", "Sequence type mismatch with observation type: ^F",status);
}
}
/* close the file */
smf_close_file( &infile, status );
}
/* Store output group in return variable or else free it */
if (outgrp) {
*outgrp = ogrp;
} else {
grpDelet( &ogrp, status );
}
/* process flatfields if necessary */
if (ffcount > 0 && fflats ) {
smfArray * array = NULL;
/* sort flats into order */
qsort( allfflats, ffcount, sizeof(*allfflats), smf_sort_bydouble);
if (fflats) array = smf_create_smfArray( status );
/* now open the flats and store them if requested */
if (*status == SAI__OK && array && ffcount) {
size_t start_ffcount = ffcount;
AstKeyMap * flatmap = NULL;
if (calcflat) {
/* Use AgeUp so that we get the keys out in the sorted order
that allfflats used */
flatmap = astKeyMap( "KeyCase=0,KeyError=1,SortBy=AgeDown" );
}
/* Read each flatfield. Calculate a responsivity image and a flatfield
solution. Store these in a keymap along with related information
which is itself stored in a keymap indexed by a string made of
OBSIDSS, reference heater value, shutter and subarray.
*/
for (i = 0; i < start_ffcount; i++ ) {
size_t ori_index = (allfflats[i]).index;
smfData * outfile = NULL;
char keystr[100];
AstKeyMap * infomap = astKeyMap( "KeyError=1" );
int oplen = 0;
char thisfile[MSG__SZMSG];
int seqcount = 0;
/* read filename from group */
infile = NULL;
smf_open_file( ingrp, ori_index, "READ", 0, &infile, status );
if ( *status != SAI__OK ) {
/* This should not happen because we have already opened
the file. If it does happen we abort with error. */
if (infile) smf_close_file( &infile, status );
break;
}
/* Calculate the key for this observation */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
/* Get the file name for error messages */
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>" );
msgLoad( "", "^F", thisfile, sizeof(thisfile), &oplen, status );
/* And the sequence counter to link against science observations */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* Prefill infomap */
astMapPut0C( infomap, "FILENAME", thisfile, "");
astMapPut0I( infomap, "SEQCOUNT", seqcount, "");
/* Collapse it */
if (*status == SAI__OK) {
smf_flat_fastflat( infile, &outfile, status );
if (*status == SMF__BADFLAT) {
errFlush( status );
if (calcflat) {
/* Need to generate an outfile like smf_flat_fastflat
and one heater setting will force smf_flat_calcflat to fail */
smf_flat_malloc( 1, infile, NULL, &outfile, status );
} else {
if (outfile) smf_close_file( &outfile, status );
if (infile) smf_close_file( &infile, status );
infomap = astAnnul( infomap );
ffcount--;
continue;
}
}
}
if (outfile && *status == SAI__OK) {
smf_close_file( &infile, status );
infile = outfile;
if (calcflat) {
size_t ngood = 0;
smfData * curresp = NULL;
int utdate;
if (*status == SAI__OK) {
ngood = smf_flat_calcflat( MSG__VERB, NULL, "RESIST",
"FLATMETH", "FLATORDER", NULL, "RESPMASK",
"FLATSNR", NULL, infile, &curresp, status );
if (*status != SAI__OK) {
/* if we failed to calculate a flatfield we continue but force the
flatfield to be completely bad. This will force the science data associated
with the flatfield to be correctly blanked. We do not annul though
if we have a SUBPAR error telling us that we have failed to define
our parameters properly. */
if (*status != SUBPAR__NOPAR) errAnnul(status);
/* parameters of flatfield */
ngood = 0;
/* Generate a blank flatfield and blank responsivity image */
smf_flat_badflat( infile, &curresp, status );
}
/* Retrieve the UT date so we can decide whether to compare
flatfields */
smf_getfitsi( infile->hdr, "UTDATE", &utdate, status );
/* Store the responsivity data for later on and the processed
flatfield until we have vetted it */
astMapPut0P( infomap, "CALCFLAT", infile, "" );
astMapPut0P( infomap, "RESP", curresp, "" );
astMapPut0I( infomap, "UTDATE", utdate, "" );
astMapPut0I( infomap, "ISGOOD", 1, "" );
astMapPut0I( infomap, "NGOOD", ngood, "" );
astMapPut0I( infomap, "GRPINDEX", ori_index, "" );
astMapPut0I( infomap, "SMFTYP", infile->hdr->obstype, "" );
astMapPutElemA( flatmap, keystr, -1, infomap );
}
} else { /* if (calcflat) */
/* Store the collapsed flatfield - the processed flat is not stored here yet */
smf_addto_smfArray( array, infile, status );
/* Copy the group info */
ndgCpsup( ingrp, ori_index, fgrp, status );
}
} /* if (outfile) */
/* Annul the keymap (will be fine if it is has been stored in another keymap) */
infomap = astAnnul( infomap );
} /* End loop over flatfields */
/* Now we have to loop over the related flatfields to disable
bolometers that are not good and also decide whether we
need to set status to bad. */
if (*status == SAI__OK && calcflat ) {
size_t nkeys = astMapSize( flatmap );
for (i = 0; i < nkeys; i++ ) {
const char *key = astMapKey( flatmap, i );
int nf = 0;
AstKeyMap ** kmaps = NULL;
int nelem = astMapLength( flatmap, key );
kmaps = astMalloc( sizeof(*kmaps) * nelem );
astMapGet1A( flatmap, key, nelem, &nelem, kmaps );
for ( nf = 0; nf < nelem && *status == SAI__OK; nf++ ) {
AstKeyMap * infomap = kmaps[nf];
int isgood = 0;
astMapGet0I( infomap, "ISGOOD", &isgood );
if (isgood) {
/* The flatfield worked */
size_t ngood = 0;
int itemp;
int utdate = 0;
int ratioFlats = 0;
/* Get the UT date - we do not compare flatfields after
the time we enabled heater tracking at each sequence. */
astMapGet0I( infomap, "UTDATE", &utdate );
/* Get the number of good bolometers at this point */
astMapGet0I( infomap, "NGOOD", &itemp );
ngood = itemp;
/* Decide if we want to do the ratio test. We default to
not doing it between 20110901 and 20120827 which is
the period when we did mini-heater tracks before each
flat. ! indicates that we choose based on date. */
if (*status == SAI__OK) {
parGet0l( "FLATUSENEXT", &ratioFlats, status );
if ( *status == PAR__NULL ) {
errAnnul( status );
if (utdate >= 20110901 || utdate <= 20120827 ) {
ratioFlats = 0;
} else {
ratioFlats = 1;
}
}
}
/* Can we compare with the next flatfield? */
if (ngood < SMF__MINSTATSAMP || !ratioFlats ) {
/* no point doing all the ratio checking for this */
} else if ( nelem - nf >= 2 ) {
AstKeyMap * nextmap = kmaps[nf+1];
const char *nextfname = NULL;
const char *fname = NULL;
smfData * curresp = NULL;
smfData * nextresp = NULL;
smfData * curflat = NULL;
void *tmpvar = NULL;
size_t bol = 0;
smfData * ratio = NULL;
double *in1 = NULL;
double *in2 = NULL;
double mean = VAL__BADD;
size_t nbolo;
double *out = NULL;
double sigma = VAL__BADD;
float clips[] = { 5.0, 5.0 }; /* 5.0 sigma iterative clip */
size_t ngoodz = 0;
astMapGet0C( nextmap, "FILENAME", &nextfname );
astMapGet0C( infomap, "FILENAME", &fname );
/* Retrieve the responsivity images from the keymap */
astMapGet0P( infomap, "RESP", &tmpvar );
curresp = tmpvar;
astMapGet0P( nextmap, "RESP", &tmpvar );
nextresp = tmpvar;
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
nbolo = (curresp->dims)[0] * (curresp->dims)[1];
/* get some memory for the ratio if we have not already.
We could get some memory once assuming each flat has the
same number of bolometers... */
ratio = smf_deepcopy_smfData( curresp, 0, 0, 0, 0, status );
if( *status == SAI__OK ) {
/* divide: smf_divide_smfData ? */
in1 = (curresp->pntr)[0];
in2 = (nextresp->pntr)[0];
out = (ratio->pntr)[0];
for (bol=0; bol<nbolo;bol++) {
if ( in1[bol] != VAL__BADD && in1[bol] != 0.0 &&
in2[bol] != VAL__BADD && in2[bol] != 0.0 ) {
out[bol] = in1[bol] / in2[bol];
} else {
out[bol] = VAL__BADD;
}
}
}
/* find some statistics */
smf_clipped_stats1D( out, 2, clips, 1, nbolo, NULL, 0, 0, &mean,
&sigma, NULL, 0, &ngoodz, status );
if (*status == SMF__INSMP) {
errAnnul(status);
msgOutiff( MSG__QUIET, "",
"Flatfield ramp ratio of %s with %s had too few bolometers (%zu < %d).",
status, fname, nextfname, ngoodz, SMF__MINSTATSAMP );
ngood = ngoodz; /* Must be lower or equal to original ngood */
} else if (*status == SAI__OK && mean != VAL__BADD && sigma != VAL__BADD && curflat->da) {
/* Now flag the flatfield as bad for bolometers that have changed
more than n%. We expect the variation to be 1+/-a small bit */
const double pmrange = 0.10;
double thrlo = 1.0 - pmrange;
double thrhi = 1.0 + pmrange;
size_t nmasked = 0;
double *flatcal = curflat->da->flatcal;
msgOutiff( MSG__DEBUG, "", "Flatfield fast ramp ratio mean = %g +/- %g (%zu bolometers)",
status, mean, sigma, ngood);
/* we can just set the first slice of the flatcal to bad. That should
be enough to disable the entire bolometer. We have just read these
data so they should be in ICD order. */
for (bol=0; bol<nbolo;bol++) {
if ( out[bol] != VAL__BADD &&
(out[bol] < thrlo || out[bol] > thrhi ) ) {
flatcal[bol] = VAL__BADD;
nmasked++;
} else if ( in1[bol] != VAL__BADD && in2[bol] == VAL__BADD ) {
/* A bolometer is bad next time but good now so we must set it bad now */
flatcal[bol] = VAL__BADD;
nmasked++;
}
}
if ( nmasked > 0 ) {
msgOutiff( MSG__NORM, "", "Masked %zu bolometers in %s from unstable flatfield",
status, nmasked, fname );
/* update ngood to take into account the masking */
ngood -= nmasked;
}
}
smf_close_file( &ratio, status );
} /* End of flatfield responsivity comparison */
/* if we only have a few bolometers left we now consider this
a bad flat unless it is actually an engineering measurement
where expect some configurations to give zero bolometers */
if (ngood < SMF__MINSTATSAMP) {
const char *fname = NULL;
void * tmpvar = NULL;
int smftyp = 0;
smfData * curflat = NULL;
astMapGet0I( infomap, "SMFTYP", &smftyp );
astMapGet0C( infomap, "FILENAME", &fname );
if (smftyp != SMF__TYP_NEP) {
msgOutiff( MSG__QUIET, "",
"Flatfield %s has %zu good bolometer%s.%s",
status, fname, ngood, (ngood == 1 ? "" : "s"),
( ngood == 0 ? "" : " Keeping none.") );
isgood = 0;
/* Make sure that everything is blanked. */
if (ngood > 0) {
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
if (curflat && curflat->da) {
size_t bol;
size_t nbolo = (curflat->dims)[0] * (curflat->dims)[1];
double *flatcal = curflat->da->flatcal;
for (bol=0; bol<nbolo; bol++) {
/* Just need to set the first element to bad */
flatcal[bol] = VAL__BADD;
}
}
}
} else {
msgOutiff( MSG__NORM, "",
"Flatfield ramp file %s has %zu good bolometer%s. Eng mode.",
status, fname, ngood, (ngood == 1 ? "" : "s") );
}
}
/* We do not need the responsivity image again */
{
void *tmpvar = NULL;
smfData * resp = NULL;
astMapGet0P( infomap, "RESP", &tmpvar );
resp = tmpvar;
if (resp) smf_close_file( &resp, status );
astMapRemove( infomap, "RESP" );
}
} /* End of isgood comparison */
/* We are storing flats even if they failed. Let the downstream
software worry about it */
{
int ori_index;
smfData * flatfile = NULL;
void *tmpvar = NULL;
/* Store in the output group */
astMapGet0I( infomap, "GRPINDEX", &ori_index );
ndgCpsup( ingrp, ori_index, fgrp, status );
/* And store in the smfArray */
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
astMapRemove( infomap, "CALCFLAT" );
flatfile = tmpvar;
smf_addto_smfArray( array, flatfile, status );
}
/* Free the object as we go */
kmaps[nf] = astAnnul( kmaps[nf] );
} /* End of loop over this obsidss/subarray/heater */
kmaps = astFree( kmaps );
}
}
if (array->ndat) {
if (fflats) *fflats = array;
} else {
smf_close_related(&array, status );
if (fflats) *fflats = NULL;
}
}
}
/* no need to do any more if neither darks nor darkgrp are defined or we might
be wanting to revert to darks. */
if (dkcount > 0 && (darks || darkgrp || reverttodark ) ) {
smfArray * array = NULL;
/* sort darks into order */
qsort( alldarks, dkcount, sizeof(*alldarks), smf_sort_bydouble);
if (darks) array = smf_create_smfArray( status );
/* now open the darks and store them if requested */
if (*status == SAI__OK) {
for (i = 0; i < dkcount; i++ ) {
size_t ori_index = (alldarks[i]).index;
/* Store the entry in the output group */
ndgCpsup( ingrp, ori_index, dgrp, status );
if (darks) {
/* read the value from the new group */
smf_open_file( dgrp, i+1, "READ", 0, &infile, status );
/* do we have to process these darks? */
if (reducedark) {
smfData *outfile = NULL;
smf_reduce_dark( infile, darktype, &outfile, status );
if (outfile) {
smf_close_file( &infile, status );
infile = outfile;
}
}
smf_addto_smfArray( array, infile, status );
}
}
if (darks) *darks = array;
}
}
/* free memory */
alldarks = astFree( alldarks );
allfflats = astFree( allfflats );
if( reverttodark && outgrp && (grpGrpsz(*outgrp,status)==0) &&
(grpGrpsz(dgrp,status)>0) ) {
/* If outgrp requested but no science observations were found, and
dark observations were found, return darks in outgrp and set
flatgrp and darkgrp to NULL. This is to handle cases where we
want to process data taken in the dark like normal science
data. To activate this behaviour set reverttodark */
msgOutiff( MSG__NORM, "", "Treating the dark%s as science data",
status, ( dkcount > 1 ? "s" : "" ) );
*outgrp = dgrp;
if( darkgrp ){
*darkgrp = NULL;
}
if( flatgrp ) {
*flatgrp = NULL;
}
grpDelet( &ogrp, status);
grpDelet( &fgrp, status);
if (meanstep && nsteps_dark > 0) *meanstep = duration_darks / nsteps_dark;
/* Have to clear the darks smfArray as well */
if (darks) smf_close_related( darks, status );
} else {
/* Store the output groups in the return variable or free it */
if (darkgrp) {
*darkgrp = dgrp;
} else {
grpDelet( &dgrp, status);
}
if (flatgrp) {
*flatgrp = fgrp;
} else {
grpDelet( &fgrp, status);
}
if (meanstep && nsteps_sci > 0) *meanstep = duration_sci / nsteps_sci;
}
msgSeti( "ND", sccount );
msgSeti( "DK", dkcount );
msgSeti( "FF", ffcount );
msgSeti( "TOT", insize );
if ( insize == 1 ) {
if (dkcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a dark",
status);
} else if (ffcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a fast flatfield",
status);
} else if (sccount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was accepted (observation type same as sequence type)",
status);
} else {
msgOutif( MSG__VERB, " ", "Single input file was not accepted.",
status);
}
} else {
if (dkcount == 1) {
msgSetc( "DKTXT", "was a dark");
} else {
msgSetc( "DKTXT", "were darks");
}
if (ffcount == 1) {
msgSetc( "FFTXT", "was a fast flat");
} else {
msgSetc( "FFTXT", "were fast flats");
}
if (sccount == 1) {
msgSetc( "NDTXT", "was science");
} else {
msgSetc( "NDTXT", "were science");
}
/* This might be a useful message */
msgOutif( MSG__NORM, " ", "Out of ^TOT input files, ^DK ^DKTXT, ^FF ^FFTXT "
"and ^ND ^NDTXT", status );
}
if (meanstep && *meanstep != VAL__BADD) {
msgOutiff( MSG__VERB, "", "Mean step time for input files = %g sec",
status, *meanstep );
}
/* Store the heater efficiency map */
if (*status != SAI__OK) heatermap = smf_free_effmap( heatermap, status );
if (heateffmap) *heateffmap = heatermap;
/* Now report the details of the observation */
smf_obsmap_report( MSG__NORM, obsmap, objmap, status );
obsmap = astAnnul( obsmap );
objmap = astAnnul( objmap );
scimap = astAnnul( scimap );
msgOutiff( SMF__TIMER_MSG, "",
"Took %.3f s to find science observations",
status, smf_timerupdate( &tv1, &tv2, status ) );
return;
}
int smf_fix_metadata_scuba2 ( msglev_t msglev, smfData * data, int have_fixed, int *ncards, int * status ) {
AstFitsChan * fits = NULL; /* FITS header (FitsChan) */
struct FitsHeaderStruct fitsvals; /* Quick access Fits header struct */
smfHead *hdr = NULL; /* Data header struct */
AstKeyMap * obsmap = NULL; /* Info from all observations */
AstKeyMap * objmap = NULL; /* All the object names used */
if (*status != SAI__OK) return have_fixed;
/* Validate arguments - need smfFile and smfHead */
smf_validate_smfData( data, 1, 1, status );
if (*status != SAI__OK) return have_fixed;
hdr = data->hdr;
smf_validate_smfHead( hdr, 1, 1, status );
if (*status != SAI__OK) return have_fixed;
fits = hdr->fitshdr;
if (hdr->instrument != INST__SCUBA2) {
if (*status != SAI__OK) {
*status = SAI__ERROR;
errRep("", " Attempting to fix metadata using SCUBA-2 algorithms but this is not SCUBA-2 data",
status );
}
return have_fixed;
}
/* Update units string to something that is FITS standard compliant
- we used "DAC units" for a while but in FITS land this becomes
"decacoulomb * units" */
if ( strncmp( hdr->units, "DAC", 3) == 0 ) {
one_strlcpy( hdr->units, "adu", SMF__CHARLABEL, status );
}
/* Clock jitter and readout efficiencies mean we need to recalculate STEPTIME from the data.
This is possible because we know that we have a continuous sequence in each file (unlike
ACSIS). */
if (hdr->allState) {
/* it will be odd if it is not there */
size_t nframes = hdr->nframes;
size_t istart = 0;
double start_time = (hdr->allState)[istart].rts_end;
while( start_time == VAL__BADD && ++istart < nframes ) {
start_time = (hdr->allState)[istart].rts_end;
}
size_t iend = nframes - 1;
double end_time = (hdr->allState)[iend].rts_end;
while( end_time == VAL__BADD && iend-- > 0 ) {
end_time = (hdr->allState)[iend].rts_end;
}
double steptime = VAL__BADD;
double newstep;
smf_getfitsd( hdr, "STEPTIME", &steptime, status );
newstep = steptime;
/* it is possible for a file to contain only one step since
the DA just dumps every N-steps. We can not recalculate the
step time in that case. */
nframes = iend - istart + 1;
if (nframes > 1) {
/* duration of file in days */
newstep = end_time - start_time;
/* convert to seconds */
newstep *= SPD;
/* Convert to step time */
newstep /= (nframes - 1);
} else if( nframes > 0 ) {
/* work it out from RTS_END and TCS_TAI */
JCMTState * onlystate = &((hdr->allState)[istart]);
if ( onlystate->tcs_tai != VAL__BADD &&
onlystate->tcs_tai != onlystate->rts_end) {
/* TCS_TAI is in the middle of the step */
newstep = 2.0 * ( onlystate->rts_end - onlystate->tcs_tai ) * SPD;
}
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
if( data->file ) {
smf_smfFile_msg( data->file, "N", 1, "<unknown>" );
errRep("", "No valid RTS_END values found in NDF '^N'.", status );
} else {
errRep("", "No valid RTS_END values found.", status );
}
}
if (steptime != newstep) {
msgOutiff( msglev, "", INDENT "Recalculated step time as %g sec from JCMTSTATE (was %g sec)",
status, newstep, steptime);
smf_fits_updateD( hdr, "STEPTIME", newstep, NULL, status );
have_fixed |= SMF__FIXED_FITSHDR;
}
}
/* Read some FITS headers, intialising the struct first */
fitsvals.utdate = VAL__BADI;
*(fitsvals.instrume) = '\0';
smf_getfitsi( hdr, "UTDATE", &(fitsvals.utdate), status );
smf_getfitss( hdr, "INSTRUME", fitsvals.instrume, sizeof(fitsvals.instrume), status );
/* Print out summary of this observation - this may get repetitive if multiple files come
from the same observation in one invocation but it seems better to describe each fix up
separately and in context. */
obsmap = astKeyMap( " " );
objmap = astKeyMap( " " );
smf_obsmap_fill( data, obsmap, objmap, status );
smf_obsmap_report( msglev, obsmap, objmap, status );
obsmap = astAnnul( obsmap );
objmap = astAnnul( objmap );
/* First we need to look for a BACKEND header which we do not write to the raw
data files but CADC would like to see equal to INSTRUME */
if (!astTestFits( fits, "BACKEND", NULL ) ) {
have_fixed |= SMF__FIXED_FITSHDR;
smf_fits_updateS( hdr, "BACKEND", fitsvals.instrume, "Name of the backend", status );
msgOutif( msglev, "", INDENT "Setting backend for SCUBA-2 observation.", status);
}
/* BASETEMP was reading MUXTEMP for pre-20091101 data */
if ( fitsvals.utdate < 20091101 ) {
double muxtemp = 0.0;
have_fixed |= SMF__FIXED_FITSHDR;
smf_getfitsd( hdr, "BASETEMP", &muxtemp, status );
smf_fits_updateU( hdr, "BASETEMP", "[K] Base temperature", status );
smf_fits_updateD( hdr, "MUXTEMP", muxtemp, "[K] Mux temperature", status );
msgOutif( msglev, "", INDENT "Mux temperature is being read from BASETEMP header.", status );
}
/* Sometime before 20091119 the SHUTTER keyword was written as a string
OPEN or CLOSED. Rewrite those as numbers */
if (fitsvals.utdate < 20091119) {
double shutval = 0.0;
/* Try to read as a double. */
smf_fits_getD( hdr, "SHUTTER", &shutval, status );
/* Old data was a string. Convert to a double */
if (*status == AST__FTCNV) {
char shutter[100];
errAnnul( status );
smf_fits_getS( hdr, "SHUTTER", shutter, sizeof(shutter), status);
if (strcmp(shutter, "CLOSED") == 0) {
shutval = 0.0;
} else {
shutval = 1.0;
}
/* update the value */
have_fixed |= SMF__FIXED_FITSHDR;
smf_fits_updateD( hdr, "SHUTTER", shutval, "shutter position 0-Closed 1-Open", status );
msgOutif( msglev, "", INDENT "Forcing SHUTTER header to be numeric", status );
}
}
/* Engineering data with just SCUBA2 and no RTS left the RTS_NUM field
filled with zeroes. Just assume that a zero in RTS_NUM is always
indicative of a private sequence. */
if (fitsvals.utdate < 20110401) {
size_t nframes = hdr->nframes;
JCMTState * curstate = &((hdr->allState)[0]);
JCMTState * endstate = &((hdr->allState)[nframes-1]);
if (curstate->rts_num == 0 && endstate->rts_num == 0) {
/* have to set the values from the SEQSTART and SEQEND headers
since those were set correctly (although any value would
do of course apart from the sanity check in smf_find_science. */
size_t i;
int seqnum = 1;
smf_fits_getI( hdr, "SEQSTART", &seqnum, status );
for ( i=0; i<nframes; i++) {
curstate = &((hdr->allState)[i]);
curstate->rts_num = seqnum;
seqnum++;
}
have_fixed |= SMF__FIXED_JCMTSTATE;
msgOutif( msglev, "", INDENT "Private RTS sequence. Fixing RTS_NUM.", status );
}
}
/* work out if this is a fast flat observation taken before May 2010 */
if (fitsvals.utdate > 20100218 && fitsvals.utdate < 20100501) {
char buff[100];
/* need to know whether this is a FASTFLAT */
smf_getfitss( hdr, "SEQ_TYPE", buff, sizeof(buff), status );
if (strcmp( buff, "FASTFLAT" ) == 0 ) {
/* Fast flats had incorrect SHUTTER settings for one night */
if (fitsvals.utdate == 20100223) {
have_fixed |= SMF__FIXED_FITSHDR;
smf_fits_updateD( hdr, "SHUTTER", 1.0, "shutter position 0-Closed 1-Open", status );
msgOutif( msglev, "", INDENT "Shutter was open for fast flatfield ramp. Correcting.", status );
}
/* Need to fix up SC2_HEAT ramps */
/* the problem is that the data were assumed to be taken with 3 measurements
in each heater setting. What actually happened was that the first 5 were
done at the reference setting and then the data were grouped in threes
finishing with a single value at the reference setting again.
For example the heater values and the actual values look something like
Stored 1 1 1 2 2 2 3 3 3 4 4 4 5 5 5
Actual 0 0 0 0 0 2 2 2 3 3 3 4 4 4 5
So we can correct for this by starting at the end and copying in the value
two slots further down until we get to position #4. Then replacing that with
the PIXHEAT number.
*/
{
size_t i;
int pixheat = 0;
size_t nframes = hdr->nframes;
smf_getfitsi( hdr, "PIXHEAT", &pixheat, status );
/* shift everything up by 2 */
for (i=nframes-1; i > 4; i--) {
JCMTState * curstate = &((hdr->allState)[i]);
JCMTState * prevstate = &((hdr->allState)[i-2]);
curstate->sc2_heat = prevstate->sc2_heat;
}
/* fill in the first 5 slots with the same value */
for (i=0; i<5; i++) {
JCMTState * curstate = &((hdr->allState)[i]);
curstate->sc2_heat = pixheat;
}
have_fixed |= SMF__FIXED_JCMTSTATE;
}
}
}
/* We always recalculate the WVM start and end tau values so that the header
reflects something approximating the value that was actually used in the
extinction correction.
Note that smf_calc_smoothedwvm can do a better job because it has multiple
subarrays to get all the values from. We just have to try with what we
have from a single subarray. We do step into the time series until we
find something good.
The header values should mostly agree with the recalculated values if the
WVM code at the time matches the code in SMURF for that date. This has not
been true in cases where we have retrospectively realised that there has been
a calibration error in the WVM. So that we do not have to keep track of those
times explicitly we currently recalculate every time. If this recalculation
becomes a problem (smf_calc_wvm has a cache now to minimize this) it should
be possible to disable this recalculation if the file is less than, say,
30 minutes old to indicate we are running in near realtime.
As a special case we do not recalculate the headers for FOCUS observations
as the WVM reading is somewhat irrelevant and simply slows things down.
*/
if( *status == SAI__OK ){
/* Have not parsed header yet to extract type so do it explicitly here */
char obstype[100];
smf_getfitss( hdr, "OBS_TYPE", obstype, sizeof(obstype), status );
if (strcasecmp( obstype, "focus") != 0) {
size_t i;
size_t nframes = hdr->nframes;
double starttau = VAL__BADD;
double starttime = VAL__BADD;
double endtau = VAL__BADD;
double endtime = VAL__BADD;
/* Create a TimeFrame that can be used to format MJD values into ISO
date-time strings, including a "T" separator between time and date. */
AstTimeFrame *tf = astTimeFrame( "Format=iso.0T" );
for (i=0; i < nframes && *status == SAI__OK; i++) {
smf__calc_wvm_index( hdr, "AMSTART", i, &starttau, &starttime, status );
if (starttau != VAL__BADD) break;
if (*status == SAI__ERROR) errAnnul( status );
}
/* if we did not get a start tau we are not going to get an end tau */
if (starttau != VAL__BADD) {
for (i=0; i < nframes && *status == SAI__OK; i++) {
smf__calc_wvm_index( hdr, "AMEND", nframes - 1 - i, &endtau, &endtime, status );
if (endtau != VAL__BADD) break;
if (*status == SAI__ERROR) errAnnul( status );
}
}
/* If we could not find any WVM readings then we have a bit of a problem.
Do we clear the FITS headers or do we leave them untouched? Leave them
alone for now. */
if (starttau != VAL__BADD && starttime != VAL__BADD) {
smf_fits_updateD( hdr, "WVMTAUST", starttau, "186GHz Tau from JCMT WVM at start", status );
/* Convert starttime MJD to ISO format and update the value in the
FITS header. */
smf_fits_updateS( hdr, "WVMDATST", astFormat( tf, 1, starttime ),
"Time of WVMTAUST", status );
have_fixed |= SMF__FIXED_FITSHDR;
}
if (endtau != VAL__BADD && endtime != VAL__BADD) {
smf_fits_updateD( hdr, "WVMTAUEN", endtau, "186GHz Tau from JCMT WVM at end", status );
/* Convert endtime MJD to ISO format and update the value in the
FITS header. */
smf_fits_updateS( hdr, "WVMDATEN", astFormat( tf, 1, endtime ),
"Time of WVMTAUEN", status );
have_fixed |= SMF__FIXED_FITSHDR;
}
/* Free the TimeFrame. */
tf = astAnnul( tf );
}
}
/* SEQ_TYPE header turned up in 20091125. Before that date the SEQ_TYPE only
had two values. If the shutter was open then SEQ_TYPE is just OBS_TYPE. In the
dark only a FLATFIELD sometimes finished with a noise but in that case CALCFLAT
doesn't care so we just call it a flatfield sequence anyhow. We could look at
the OBSEND flag but I'm not sure it makes a difference. */
if ( fitsvals.utdate < 20091125 ) {
char obstype[100];
char seqtype[100];
double shutval = 0.0;
/* need to know what type of observation this is */
smf_getfitss( hdr, "OBS_TYPE", obstype, sizeof(obstype), status );
/* and the shutter status */
smf_fits_getD( hdr, "SHUTTER", &shutval, status );
if (shutval == 0.0 && strcasecmp( obstype, "flatfield" ) != 0 ) {
/* flatfield was the only non-noise observation in the dark */
one_strlcpy( seqtype, "NOISE", sizeof(seqtype), status );
msgOutif( msglev, "", INDENT "Setting sequence type to NOISE", status );
} else {
/* Shutter was open so SEQ_TYPE is just OBS_TYPE */
one_strlcpy( seqtype, obstype, sizeof(seqtype), status );
msgOutif( msglev, "", INDENT "Setting sequence type to obs type", status);
}
smf_fits_updateS( hdr, "SEQ_TYPE", seqtype, "Type of sequence", status );
have_fixed |= SMF__FIXED_FITSHDR;
}
/* The telescope goes crazy at the end of observation 56 on 20110530. Null
the telescope data for subscans 30, 31 and 32 */
if (fitsvals.utdate == 20110530) {
char obsid[81];
smf_getobsidss( hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status);
if (strcmp(obsid, "scuba2_00056_20110530T135530") == 0 ) {
int subscan;
smf_getfitsi( hdr, "NSUBSCAN", &subscan, status );
if (subscan == 30 || subscan == 31 || subscan == 32) {
size_t nframes = hdr->nframes;
JCMTState * curstate;
size_t i;
for ( i=0; i<nframes; i++ ) {
curstate = &((hdr->allState)[i]);
curstate->jos_drcontrol |= DRCNTRL__PTCS_BIT;
}
msgOutif( msglev, "", INDENT "Blanking telescope data due to extreme excursion", status );
have_fixed |= SMF__FIXED_JCMTSTATE;
}
}
}
/* The second half of observation 14 on 20111215 (scuba2_00014_20111215T061536)
has a elevation pointing shift */
if (fitsvals.utdate == 20111215) {
char obsid[81];
const char fitskey[] = "FIXPCORR";
smf_getobsidss( hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status);
if (strcmp(obsid, "scuba2_00014_20111215T061536") == 0 ) {
int seqcount;
smf_getfitsi( hdr, "SEQCOUNT", &seqcount, status );
if (seqcount == 5) {
int have_fixed_pntg = 0;
smf_fits_getL( hdr, fitskey, &have_fixed_pntg, status );
if (*status == SMF__NOKWRD) {
have_fixed = 0;
errAnnul( status );
}
if (!have_fixed_pntg) {
size_t nframes = hdr->nframes;
size_t i;
const double dlon = 0.0;
const double dlat = -16.83; /* From making maps of each half */
/* Correct the pointing */
msgOutif( msglev, "", INDENT "Applying pointing anomaly correction", status );
for (i=0;i<nframes;i++) {
JCMTState * curstate = &((hdr->allState)[i]);
/* This is an AZEL correction */
smf_add_smu_pcorr( curstate, 1, dlon, dlat, status );
}
smf_fits_updateL(hdr, fitskey, 1, "Applied internal pointing correction", status);
have_fixed |= SMF__FIXED_JCMTSTATE;
}
}
}
}
/* For POL-2 data prior to 18-JAN-2013, the POL_CRD header was always
"FPLANE" in reality, even if the POL_CRD value in JCMTSTATE said
something else. */
if ( fitsvals.utdate < 20130118 ) {
char polcrd[80] = "<unset>";
smf_getfitss( hdr, "POL_CRD", polcrd, sizeof(polcrd), status );
if (*status == SMF__NOKWRD) {
errAnnul( status );
} else if( !strcmp( polcrd, "TRACKING" ) || !strcmp( polcrd, "AZEL" ) ) {
msgOutiff( msglev, "", INDENT "Changing POL_CRD from %s to FPLANE", status, polcrd);
smf_fits_updateS( hdr, "POL_CRD", "FPLANE",
"Coordinate system of polarimeter", status );
have_fixed |= SMF__FIXED_FITSHDR;
}
}
return have_fixed;
}
