void
smf_flat_badflat( ThrWorkForce *wf, smfData * refdata, smfData ** resp, int * status ) {
smfData * bolrefd = NULL;
smfData * powrefd = NULL;
size_t nheat = 1;
if (*status != SAI__OK) return;
if (!refdata) return;
/* See if we have heater values */
if (refdata->da && refdata->da->nheat > 0) {
nheat = refdata->da->nheat;
}
/* Need to create bogus powref and bolref arrays */
smf_flat_malloc( nheat, refdata, &powrefd, &bolrefd, status );
smf__fill_smfData ( powrefd, VAL__BADD, status );
smf__fill_smfData ( bolrefd, VAL__BADD, status );
/* Create a responsivity image and fill with bad values */
if (resp) {
smf_create_bolfile( wf, NULL, 0, refdata, "Responsivity",
"A/W", SMF__MAP_VAR, resp, status );
smf__fill_smfData ( *resp, VAL__BADD, status );
}
/* Attach the bad flatfield data */
smf_flat_assign( 0, SMF__FLATMETH_POLY, 3.0, powrefd, bolrefd,
refdata, status );
}
void smf_flat_smfData ( const smfData *data, smf_flatmeth * flatmethod,
double * refres, smfData ** powval, smfData **bolval,
int *status ) {
smfDA * da = NULL;
*powval = NULL;
*bolval = NULL;
*flatmethod = SMF__FLATMETH_NULL;
if (*status != SAI__OK) return;
if (!data) return;
if (!(data->da)) return;
da = data->da;
/* we need to malloc space to make sure that we do not free the pointers
twice. There really needs to be a way to tell smf_close_file to free
everything except the pointers to the data. */
smf_flat_malloc( da->nflat, data, powval, bolval, status );
/* flatpar is powval */
if (*status == SAI__OK) {
size_t nelem = (*bolval)->dims[0] * (*bolval)->dims[1] * (*bolval)->dims[2];
memcpy( (*powval)->pntr[0], da->flatpar, da->nflat * sizeof(*(da->flatpar)) );
memcpy( (*bolval)->pntr[0], da->flatcal, nelem * sizeof(*(da->flatcal)) );
}
*flatmethod = da->flatmeth;
*refres = da->refres;
if (*status != SAI__OK) {
if (*bolval) smf_close_file( bolval, status );
if (*powval) smf_close_file( powval, status );
}
return;
}
void
smf_flat_standardpow( const smfData * bolvald, double refohms,
const double resistance[],
smfData ** powrefd, smfData ** bolrefd, int * status) {
double current; /* current through heaters */
double *heatframe = NULL; /* Pointer to flatfield heater data */
double *heatframevar = NULL; /* Pointer to flatfield heatframe variance */
double * heatref = NULL; /* local copy of heater settings */
size_t i; /* loop counter */
size_t j; /* loop counter */
size_t k; /* loop counter */
size_t nheat; /* Number of input frames */
size_t numbol; /* Number of bolometers */
double *powbol = NULL; /* pointer to individual bolometer powers */
double s; /* interpolation slope factor */
double * powref = NULL; /* Data array inside powrefd */
double * bolref = NULL; /* Data array inside bolrefd */
double * bolrefvar = NULL; /* Variance inside bolrefd */
*powrefd = NULL;
*bolrefd = NULL;
if (*status != SAI__OK) return;
if ( !smf_dtype_check_fatal( bolvald, NULL, SMF__DOUBLE, status ) ) return;
if (!bolvald->da) {
*status = SAI__ERROR;
errRep( "", "Unable to proceed with flatfield calculation. Heater settings not "
"present in smfData", status);
return;
}
nheat = (bolvald->dims)[2];
numbol = (bolvald->dims)[0] * (bolvald->dims)[1];
heatref = bolvald->da->heatval;
/* Create a smfData for powref and bolref */
smf_flat_malloc( nheat, bolvald, powrefd, bolrefd, status );
if (*status == SAI__OK) {
powref = (*powrefd)->pntr[0];
bolref = (*bolrefd)->pntr[0];
bolrefvar = (*bolrefd)->pntr[1];
}
/* Get some memory -
bolometer power per heater setting */
powbol = astCalloc( nheat, sizeof(*powbol) );
/* pointers to the flatfield data and variance */
heatframe = (bolvald->pntr)[0];
heatframevar = (bolvald->pntr)[1];
/* Choose the reference heater powers to be the actual heater settings acting on
the adopted reference resistance. */
for ( j=0; j<nheat; j++ ) {
current = heatref[j] * SC2FLAT__DTOI;
powref[j] = SIMULT * current * current * refohms;
}
if (*status != SAI__OK) goto CLEANUP;
/* For each bolometer interpolate to find the measurement it would report at the
standard power input values */
for ( i=0; i<numbol; i++ )
{
/* Calculate the actual power seen by each bolometer at each heater setting */
for ( j=0; j<nheat; j++ )
{
current = heatref[j] * SC2FLAT__DTOI;
if ( resistance[i] == VAL__BADD )
{
powbol[j] = VAL__BADD;
}
else
{
powbol[j] = SIMULT * current * current * resistance[i];
}
}
/* Trap bolometers that have any bad values in their measurements */
for ( j=0; j<nheat; j++) {
if ( heatframe[i + j*numbol] == VAL__BADD) {
powbol[0] = VAL__BADD;
break;
}
}
/* Trap bolometers which are unresponsive */
if ( powbol[0] != VAL__BADD &&
(heatframe[i] == VAL__BADD || heatframe[i + (nheat-1)*numbol] == VAL__BADD ||
(fabs ( heatframe[i] - heatframe[i + (nheat-1)*numbol] ) < 1.0) ) )
{
powbol[0] = VAL__BADD;
}
for ( j=0; j<nheat; j++ )
{
if ( powbol[0] == VAL__BADD )
{
bolref[j*numbol+i] = VAL__BADD;
bolrefvar[j*numbol+i] = VAL__BADD;
}
else
{
if ( powref[j] < powbol[0] )
{
double var = VAL__BADD;
/* Standard point is below actual measured range for this bolometer,
extrapolate */
s = ( heatframe[i + 1*numbol] - heatframe[i + 0*numbol] ) /
( powbol[1] - powbol[0] );
/* calculate error in gradient using standard rules */
if (heatframevar && heatframevar[i+ 1*numbol] != VAL__BADD &&
heatframevar[i + 0*numbol] != VAL__BADD) {
var = ( heatframevar[i + 1*numbol] + heatframevar[i + 0*numbol] )
/ pow( powbol[1] - powbol[0], 2 );
}
bolref[j*numbol+i] = heatframe[i+ 0*numbol] +
s * ( powref[j] - powbol[0] );
if (var != VAL__BADD) {
bolrefvar[j*numbol+i] = heatframevar[i + 0*numbol] +
( var * pow( powref[j] - powbol[0], 2 ));
} else {
bolrefvar[j*numbol+i] = VAL__BADD;
}
}
else if ( powref[j] > powbol[nheat-1] )
{
double var = VAL__BADD;
/* Standard point is above actual measured range for this bolometer,
extrapolate */
s = ( heatframe[i + (nheat-1)*numbol] - heatframe[i + (nheat-2)*numbol] ) /
( powbol[nheat-1] - powbol[nheat-2] );
/* calculate error in gradient using standard rules */
if (heatframevar && heatframevar[i + (nheat-1)*numbol] != VAL__BADD &&
heatframevar[i + (nheat-2)*numbol] != VAL__BADD) {
var = ( heatframevar[i + (nheat-1)] + heatframevar[i + (nheat-2)*numbol] )
/ pow( powbol[nheat-1] - powbol[nheat-2], 2 );
}
bolref[j*numbol+i] = heatframe[i + (nheat-2)*numbol] +
s * ( powref[j] - powbol[nheat-2] );
if (var != VAL__BADD) {
bolrefvar[j*numbol+i] = heatframevar[i +(nheat-2)*numbol] +
( var * pow( powref[j] - powbol[nheat-2], 2 ));
} else {
bolrefvar[j*numbol+i] = VAL__BADD;
}
}
else
{
/* Standard point is within actual measured range for this bolometer,
search for the points to interpolate */
for ( k=1; k<nheat; k++ )
{
if ( powref[j] <= powbol[k] )
{
double var = VAL__BADD;
s = ( heatframe[i + k*numbol] -
heatframe[i + (k-1)*numbol] ) /
( powbol[k] - powbol[k-1] );
/* calculate error in gradient using standard rules */
if (heatframevar && heatframevar[i + k*numbol] != VAL__BADD &&
heatframevar[i +(k-1)*numbol] != VAL__BADD) {
var = ( heatframevar[i + k*numbol] + heatframevar[i + (k-1)*numbol] )
/ pow( powbol[k] - powbol[k-1], 2 );
}
bolref[j*numbol+i] = heatframe[i + (k-1)*numbol] +
s * ( powref[j] - powbol[k-1] );
if (var != VAL__BADD) {
bolrefvar[j*numbol+i] = heatframevar[i +(k-1)*numbol] +
( var * pow( powref[j] - powbol[k-1], 2 ));
} else {
bolrefvar[j*numbol+i] = VAL__BADD;
}
break;
}
}
}
}
}
}
CLEANUP:
if (powbol) powbol = astFree( powbol );
if (*status != SAI__OK) {
if (*bolrefd) smf_close_file( NULL, bolrefd, status );
if (*powrefd) smf_close_file( NULL, powrefd, status );
}
}
void smf_find_science(const Grp * ingrp, Grp **outgrp, int reverttodark,
Grp **darkgrp, Grp **flatgrp, int reducedark,
int calcflat, smf_dtype darktype, smfArray ** darks,
smfArray **fflats, AstKeyMap ** heateffmap,
double * meanstep, int * status ) {
smfSortInfo *alldarks; /* array of sort structs for darks */
smfSortInfo *allfflats; /* array of fast flat info */
Grp * dgrp = NULL; /* Internal dark group */
double duration_darks = 0.0; /* total duration of all darks */
double duration_sci = 0.0; /* Duration of all science observations */
size_t dkcount = 0; /* Dark counter */
size_t ffcount = 0; /* Fast flat counter */
Grp * fgrp = NULL; /* Fast flat group */
size_t i; /* loop counter */
smfData *infile = NULL; /* input file */
size_t insize; /* number of input files */
size_t nsteps_dark = 0; /* Total number of steps for darks */
size_t nsteps_sci = 0; /* Total number of steps for science */
AstKeyMap * heatermap = NULL; /* Heater efficiency map */
AstKeyMap * obsmap = NULL; /* Info from all observations */
AstKeyMap * objmap = NULL; /* All the object names used */
AstKeyMap * scimap = NULL; /* All non-flat obs indexed by unique key */
Grp *ogrp = NULL; /* local copy of output group */
size_t sccount = 0; /* Number of accepted science files */
struct timeval tv1; /* Timer */
struct timeval tv2; /* Timer */
if (meanstep) *meanstep = VAL__BADD;
if (outgrp) *outgrp = NULL;
if (darkgrp) *darkgrp = NULL;
if (darks) *darks = NULL;
if (fflats) *fflats = NULL;
if (heateffmap) *heateffmap = NULL;
if (*status != SAI__OK) return;
/* Sanity check to make sure we return some information */
if ( outgrp == NULL && darkgrp == NULL && darks == NULL && fflats == NULL) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": Must have some non-NULL arguments"
" (possible programming error)", status);
return;
}
/* Start a timer to see how long this takes */
smf_timerinit( &tv1, &tv2, status );
/* Create new group for output files */
ogrp = smf_grp_new( ingrp, "Science", status );
/* and a new group for darks */
dgrp = smf_grp_new( ingrp, "DarkFiles", status );
/* and for fast flats */
fgrp = smf_grp_new( ingrp, "FastFlats", status );
/* and also create a keymap for the observation description */
obsmap = astKeyMap( "KeyError=1" );
/* and an object map */
objmap = astKeyMap( "KeyError=1" );
/* This keymap contains the sequence counters for each related
subarray/obsidss/heater/shutter combination and is used to decide
if a bad flat is relevant */
scimap = astKeyMap( "KeyError=1,KeyCase=0" );
/* This keymap is used to contain relevant heater efficiency data */
heatermap = astKeyMap( "KeyError=1,KeyCase=0" );
/* Work out how many input files we have and allocate sufficient sorting
space */
insize = grpGrpsz( ingrp, status );
alldarks = astCalloc( insize, sizeof(*alldarks) );
allfflats = astCalloc( insize, sizeof(*allfflats) );
/* check each file in turn */
for (i = 1; i <= insize; i++) {
int seqcount = 0;
char keystr[100]; /* Key for scimap entry */
/* open the file but just to get the header */
smf_open_file( ingrp, i, "READ", SMF__NOCREATE_DATA, &infile, status );
if (*status != SAI__OK) break;
/* Fill in the keymap with observation details */
smf_obsmap_fill( infile, obsmap, objmap, status );
/* Find the heater efficiency map if required */
if (*status == SAI__OK && heateffmap) {
char arrayidstr[32];
smf_fits_getS( infile->hdr, "ARRAYID", arrayidstr, sizeof(arrayidstr),
status );
if (!astMapHasKey( heatermap, arrayidstr ) ) {
smfData * heateff = NULL;
dim_t nbolos = 0;
smf_flat_params( infile, "RESIST", NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, &heateff, status );
smf_get_dims( heateff, NULL, NULL, &nbolos, NULL, NULL, NULL, NULL,
status );
if (heateff) astMapPut0P( heatermap, arrayidstr, heateff, NULL );
}
}
/* Get the sequence counter for the file. We do not worry about
duplicate sequence counters (at the moment) */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* The key identifying this subarray/obsidss/heater/shutter combo */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
if (smf_isdark( infile, status )) {
/* Store the sorting information */
dkcount = smf__addto_sortinfo( infile, alldarks, i, dkcount, "Dark", status );
smf__addto_durations( infile, &duration_darks, &nsteps_dark, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
} else {
/* compare sequence type with observation type and drop it (for now)
if they differ */
if ( infile->hdr->obstype == infile->hdr->seqtype ) {
/* Sanity check the header for corruption. Compare RTS_NUM with SEQSTART
and SEQEND. The first RTS_NUM must either be SEQSTART or else between
SEQSTART and SEQEND (if someone has giving us a section) */
int seqstart = 0;
int seqend = 0;
int firstnum = 0;
JCMTState *tmpState = NULL;
smf_getfitsi( infile->hdr, "SEQSTART", &seqstart, status );
smf_getfitsi( infile->hdr, "SEQEND", &seqend, status );
tmpState = infile->hdr->allState;
if( tmpState ) {
firstnum = (tmpState[0]).rts_num;
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
if ( firstnum >= seqstart && firstnum <= seqend ) {
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif(MSG__DEBUG, " ", "Non-dark file: ^F",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
} else {
msgOutif( MSG__QUIET, "",
"File ^F has a corrupt FITS header. Ignoring it.",
status );
}
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif( MSG__DEBUG, " ",
"File ^F lacks JCMTState: assuming it is non-dark",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
}
} else if (infile->hdr->seqtype == SMF__TYP_FASTFLAT ) {
ffcount = smf__addto_sortinfo( infile, allfflats, i, ffcount, "Fast flat", status );
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
msgOutif(MSG__DEBUG, " ", "Sequence type mismatch with observation type: ^F",status);
}
}
/* close the file */
smf_close_file( &infile, status );
}
/* Store output group in return variable or else free it */
if (outgrp) {
*outgrp = ogrp;
} else {
grpDelet( &ogrp, status );
}
/* process flatfields if necessary */
if (ffcount > 0 && fflats ) {
smfArray * array = NULL;
/* sort flats into order */
qsort( allfflats, ffcount, sizeof(*allfflats), smf_sort_bydouble);
if (fflats) array = smf_create_smfArray( status );
/* now open the flats and store them if requested */
if (*status == SAI__OK && array && ffcount) {
size_t start_ffcount = ffcount;
AstKeyMap * flatmap = NULL;
if (calcflat) {
/* Use AgeUp so that we get the keys out in the sorted order
that allfflats used */
flatmap = astKeyMap( "KeyCase=0,KeyError=1,SortBy=AgeDown" );
}
/* Read each flatfield. Calculate a responsivity image and a flatfield
solution. Store these in a keymap along with related information
which is itself stored in a keymap indexed by a string made of
OBSIDSS, reference heater value, shutter and subarray.
*/
for (i = 0; i < start_ffcount; i++ ) {
size_t ori_index = (allfflats[i]).index;
smfData * outfile = NULL;
char keystr[100];
AstKeyMap * infomap = astKeyMap( "KeyError=1" );
int oplen = 0;
char thisfile[MSG__SZMSG];
int seqcount = 0;
/* read filename from group */
infile = NULL;
smf_open_file( ingrp, ori_index, "READ", 0, &infile, status );
if ( *status != SAI__OK ) {
/* This should not happen because we have already opened
the file. If it does happen we abort with error. */
if (infile) smf_close_file( &infile, status );
break;
}
/* Calculate the key for this observation */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
/* Get the file name for error messages */
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>" );
msgLoad( "", "^F", thisfile, sizeof(thisfile), &oplen, status );
/* And the sequence counter to link against science observations */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* Prefill infomap */
astMapPut0C( infomap, "FILENAME", thisfile, "");
astMapPut0I( infomap, "SEQCOUNT", seqcount, "");
/* Collapse it */
if (*status == SAI__OK) {
smf_flat_fastflat( infile, &outfile, status );
if (*status == SMF__BADFLAT) {
errFlush( status );
if (calcflat) {
/* Need to generate an outfile like smf_flat_fastflat
and one heater setting will force smf_flat_calcflat to fail */
smf_flat_malloc( 1, infile, NULL, &outfile, status );
} else {
if (outfile) smf_close_file( &outfile, status );
if (infile) smf_close_file( &infile, status );
infomap = astAnnul( infomap );
ffcount--;
continue;
}
}
}
if (outfile && *status == SAI__OK) {
smf_close_file( &infile, status );
infile = outfile;
if (calcflat) {
size_t ngood = 0;
smfData * curresp = NULL;
int utdate;
if (*status == SAI__OK) {
ngood = smf_flat_calcflat( MSG__VERB, NULL, "RESIST",
"FLATMETH", "FLATORDER", NULL, "RESPMASK",
"FLATSNR", NULL, infile, &curresp, status );
if (*status != SAI__OK) {
/* if we failed to calculate a flatfield we continue but force the
flatfield to be completely bad. This will force the science data associated
with the flatfield to be correctly blanked. We do not annul though
if we have a SUBPAR error telling us that we have failed to define
our parameters properly. */
if (*status != SUBPAR__NOPAR) errAnnul(status);
/* parameters of flatfield */
ngood = 0;
/* Generate a blank flatfield and blank responsivity image */
smf_flat_badflat( infile, &curresp, status );
}
/* Retrieve the UT date so we can decide whether to compare
flatfields */
smf_getfitsi( infile->hdr, "UTDATE", &utdate, status );
/* Store the responsivity data for later on and the processed
flatfield until we have vetted it */
astMapPut0P( infomap, "CALCFLAT", infile, "" );
astMapPut0P( infomap, "RESP", curresp, "" );
astMapPut0I( infomap, "UTDATE", utdate, "" );
astMapPut0I( infomap, "ISGOOD", 1, "" );
astMapPut0I( infomap, "NGOOD", ngood, "" );
astMapPut0I( infomap, "GRPINDEX", ori_index, "" );
astMapPut0I( infomap, "SMFTYP", infile->hdr->obstype, "" );
astMapPutElemA( flatmap, keystr, -1, infomap );
}
} else { /* if (calcflat) */
/* Store the collapsed flatfield - the processed flat is not stored here yet */
smf_addto_smfArray( array, infile, status );
/* Copy the group info */
ndgCpsup( ingrp, ori_index, fgrp, status );
}
} /* if (outfile) */
/* Annul the keymap (will be fine if it is has been stored in another keymap) */
infomap = astAnnul( infomap );
} /* End loop over flatfields */
/* Now we have to loop over the related flatfields to disable
bolometers that are not good and also decide whether we
need to set status to bad. */
if (*status == SAI__OK && calcflat ) {
size_t nkeys = astMapSize( flatmap );
for (i = 0; i < nkeys; i++ ) {
const char *key = astMapKey( flatmap, i );
int nf = 0;
AstKeyMap ** kmaps = NULL;
int nelem = astMapLength( flatmap, key );
kmaps = astMalloc( sizeof(*kmaps) * nelem );
astMapGet1A( flatmap, key, nelem, &nelem, kmaps );
for ( nf = 0; nf < nelem && *status == SAI__OK; nf++ ) {
AstKeyMap * infomap = kmaps[nf];
int isgood = 0;
astMapGet0I( infomap, "ISGOOD", &isgood );
if (isgood) {
/* The flatfield worked */
size_t ngood = 0;
int itemp;
int utdate = 0;
int ratioFlats = 0;
/* Get the UT date - we do not compare flatfields after
the time we enabled heater tracking at each sequence. */
astMapGet0I( infomap, "UTDATE", &utdate );
/* Get the number of good bolometers at this point */
astMapGet0I( infomap, "NGOOD", &itemp );
ngood = itemp;
/* Decide if we want to do the ratio test. We default to
not doing it between 20110901 and 20120827 which is
the period when we did mini-heater tracks before each
flat. ! indicates that we choose based on date. */
if (*status == SAI__OK) {
parGet0l( "FLATUSENEXT", &ratioFlats, status );
if ( *status == PAR__NULL ) {
errAnnul( status );
if (utdate >= 20110901 || utdate <= 20120827 ) {
ratioFlats = 0;
} else {
ratioFlats = 1;
}
}
}
/* Can we compare with the next flatfield? */
if (ngood < SMF__MINSTATSAMP || !ratioFlats ) {
/* no point doing all the ratio checking for this */
} else if ( nelem - nf >= 2 ) {
AstKeyMap * nextmap = kmaps[nf+1];
const char *nextfname = NULL;
const char *fname = NULL;
smfData * curresp = NULL;
smfData * nextresp = NULL;
smfData * curflat = NULL;
void *tmpvar = NULL;
size_t bol = 0;
smfData * ratio = NULL;
double *in1 = NULL;
double *in2 = NULL;
double mean = VAL__BADD;
size_t nbolo;
double *out = NULL;
double sigma = VAL__BADD;
float clips[] = { 5.0, 5.0 }; /* 5.0 sigma iterative clip */
size_t ngoodz = 0;
astMapGet0C( nextmap, "FILENAME", &nextfname );
astMapGet0C( infomap, "FILENAME", &fname );
/* Retrieve the responsivity images from the keymap */
astMapGet0P( infomap, "RESP", &tmpvar );
curresp = tmpvar;
astMapGet0P( nextmap, "RESP", &tmpvar );
nextresp = tmpvar;
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
nbolo = (curresp->dims)[0] * (curresp->dims)[1];
/* get some memory for the ratio if we have not already.
We could get some memory once assuming each flat has the
same number of bolometers... */
ratio = smf_deepcopy_smfData( curresp, 0, 0, 0, 0, status );
if( *status == SAI__OK ) {
/* divide: smf_divide_smfData ? */
in1 = (curresp->pntr)[0];
in2 = (nextresp->pntr)[0];
out = (ratio->pntr)[0];
for (bol=0; bol<nbolo;bol++) {
if ( in1[bol] != VAL__BADD && in1[bol] != 0.0 &&
in2[bol] != VAL__BADD && in2[bol] != 0.0 ) {
out[bol] = in1[bol] / in2[bol];
} else {
out[bol] = VAL__BADD;
}
}
}
/* find some statistics */
smf_clipped_stats1D( out, 2, clips, 1, nbolo, NULL, 0, 0, &mean,
&sigma, NULL, 0, &ngoodz, status );
if (*status == SMF__INSMP) {
errAnnul(status);
msgOutiff( MSG__QUIET, "",
"Flatfield ramp ratio of %s with %s had too few bolometers (%zu < %d).",
status, fname, nextfname, ngoodz, SMF__MINSTATSAMP );
ngood = ngoodz; /* Must be lower or equal to original ngood */
} else if (*status == SAI__OK && mean != VAL__BADD && sigma != VAL__BADD && curflat->da) {
/* Now flag the flatfield as bad for bolometers that have changed
more than n%. We expect the variation to be 1+/-a small bit */
const double pmrange = 0.10;
double thrlo = 1.0 - pmrange;
double thrhi = 1.0 + pmrange;
size_t nmasked = 0;
double *flatcal = curflat->da->flatcal;
msgOutiff( MSG__DEBUG, "", "Flatfield fast ramp ratio mean = %g +/- %g (%zu bolometers)",
status, mean, sigma, ngood);
/* we can just set the first slice of the flatcal to bad. That should
be enough to disable the entire bolometer. We have just read these
data so they should be in ICD order. */
for (bol=0; bol<nbolo;bol++) {
if ( out[bol] != VAL__BADD &&
(out[bol] < thrlo || out[bol] > thrhi ) ) {
flatcal[bol] = VAL__BADD;
nmasked++;
} else if ( in1[bol] != VAL__BADD && in2[bol] == VAL__BADD ) {
/* A bolometer is bad next time but good now so we must set it bad now */
flatcal[bol] = VAL__BADD;
nmasked++;
}
}
if ( nmasked > 0 ) {
msgOutiff( MSG__NORM, "", "Masked %zu bolometers in %s from unstable flatfield",
status, nmasked, fname );
/* update ngood to take into account the masking */
ngood -= nmasked;
}
}
smf_close_file( &ratio, status );
} /* End of flatfield responsivity comparison */
/* if we only have a few bolometers left we now consider this
a bad flat unless it is actually an engineering measurement
where expect some configurations to give zero bolometers */
if (ngood < SMF__MINSTATSAMP) {
const char *fname = NULL;
void * tmpvar = NULL;
int smftyp = 0;
smfData * curflat = NULL;
astMapGet0I( infomap, "SMFTYP", &smftyp );
astMapGet0C( infomap, "FILENAME", &fname );
if (smftyp != SMF__TYP_NEP) {
msgOutiff( MSG__QUIET, "",
"Flatfield %s has %zu good bolometer%s.%s",
status, fname, ngood, (ngood == 1 ? "" : "s"),
( ngood == 0 ? "" : " Keeping none.") );
isgood = 0;
/* Make sure that everything is blanked. */
if (ngood > 0) {
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
if (curflat && curflat->da) {
size_t bol;
size_t nbolo = (curflat->dims)[0] * (curflat->dims)[1];
double *flatcal = curflat->da->flatcal;
for (bol=0; bol<nbolo; bol++) {
/* Just need to set the first element to bad */
flatcal[bol] = VAL__BADD;
}
}
}
} else {
msgOutiff( MSG__NORM, "",
"Flatfield ramp file %s has %zu good bolometer%s. Eng mode.",
status, fname, ngood, (ngood == 1 ? "" : "s") );
}
}
/* We do not need the responsivity image again */
{
void *tmpvar = NULL;
smfData * resp = NULL;
astMapGet0P( infomap, "RESP", &tmpvar );
resp = tmpvar;
if (resp) smf_close_file( &resp, status );
astMapRemove( infomap, "RESP" );
}
} /* End of isgood comparison */
/* We are storing flats even if they failed. Let the downstream
software worry about it */
{
int ori_index;
smfData * flatfile = NULL;
void *tmpvar = NULL;
/* Store in the output group */
astMapGet0I( infomap, "GRPINDEX", &ori_index );
ndgCpsup( ingrp, ori_index, fgrp, status );
/* And store in the smfArray */
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
astMapRemove( infomap, "CALCFLAT" );
flatfile = tmpvar;
smf_addto_smfArray( array, flatfile, status );
}
/* Free the object as we go */
kmaps[nf] = astAnnul( kmaps[nf] );
} /* End of loop over this obsidss/subarray/heater */
kmaps = astFree( kmaps );
}
}
if (array->ndat) {
if (fflats) *fflats = array;
} else {
smf_close_related(&array, status );
if (fflats) *fflats = NULL;
}
}
}
/* no need to do any more if neither darks nor darkgrp are defined or we might
be wanting to revert to darks. */
if (dkcount > 0 && (darks || darkgrp || reverttodark ) ) {
smfArray * array = NULL;
/* sort darks into order */
qsort( alldarks, dkcount, sizeof(*alldarks), smf_sort_bydouble);
if (darks) array = smf_create_smfArray( status );
/* now open the darks and store them if requested */
if (*status == SAI__OK) {
for (i = 0; i < dkcount; i++ ) {
size_t ori_index = (alldarks[i]).index;
/* Store the entry in the output group */
ndgCpsup( ingrp, ori_index, dgrp, status );
if (darks) {
/* read the value from the new group */
smf_open_file( dgrp, i+1, "READ", 0, &infile, status );
/* do we have to process these darks? */
if (reducedark) {
smfData *outfile = NULL;
smf_reduce_dark( infile, darktype, &outfile, status );
if (outfile) {
smf_close_file( &infile, status );
infile = outfile;
}
}
smf_addto_smfArray( array, infile, status );
}
}
if (darks) *darks = array;
}
}
/* free memory */
alldarks = astFree( alldarks );
allfflats = astFree( allfflats );
if( reverttodark && outgrp && (grpGrpsz(*outgrp,status)==0) &&
(grpGrpsz(dgrp,status)>0) ) {
/* If outgrp requested but no science observations were found, and
dark observations were found, return darks in outgrp and set
flatgrp and darkgrp to NULL. This is to handle cases where we
want to process data taken in the dark like normal science
data. To activate this behaviour set reverttodark */
msgOutiff( MSG__NORM, "", "Treating the dark%s as science data",
status, ( dkcount > 1 ? "s" : "" ) );
*outgrp = dgrp;
if( darkgrp ){
*darkgrp = NULL;
}
if( flatgrp ) {
*flatgrp = NULL;
}
grpDelet( &ogrp, status);
grpDelet( &fgrp, status);
if (meanstep && nsteps_dark > 0) *meanstep = duration_darks / nsteps_dark;
/* Have to clear the darks smfArray as well */
if (darks) smf_close_related( darks, status );
} else {
/* Store the output groups in the return variable or free it */
if (darkgrp) {
*darkgrp = dgrp;
} else {
grpDelet( &dgrp, status);
}
if (flatgrp) {
*flatgrp = fgrp;
} else {
grpDelet( &fgrp, status);
}
if (meanstep && nsteps_sci > 0) *meanstep = duration_sci / nsteps_sci;
}
msgSeti( "ND", sccount );
msgSeti( "DK", dkcount );
msgSeti( "FF", ffcount );
msgSeti( "TOT", insize );
if ( insize == 1 ) {
if (dkcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a dark",
status);
} else if (ffcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a fast flatfield",
status);
} else if (sccount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was accepted (observation type same as sequence type)",
status);
} else {
msgOutif( MSG__VERB, " ", "Single input file was not accepted.",
status);
}
} else {
if (dkcount == 1) {
msgSetc( "DKTXT", "was a dark");
} else {
msgSetc( "DKTXT", "were darks");
}
if (ffcount == 1) {
msgSetc( "FFTXT", "was a fast flat");
} else {
msgSetc( "FFTXT", "were fast flats");
}
if (sccount == 1) {
msgSetc( "NDTXT", "was science");
} else {
msgSetc( "NDTXT", "were science");
}
/* This might be a useful message */
msgOutif( MSG__NORM, " ", "Out of ^TOT input files, ^DK ^DKTXT, ^FF ^FFTXT "
"and ^ND ^NDTXT", status );
}
if (meanstep && *meanstep != VAL__BADD) {
msgOutiff( MSG__VERB, "", "Mean step time for input files = %g sec",
status, *meanstep );
}
/* Store the heater efficiency map */
if (*status != SAI__OK) heatermap = smf_free_effmap( heatermap, status );
if (heateffmap) *heateffmap = heatermap;
/* Now report the details of the observation */
smf_obsmap_report( MSG__NORM, obsmap, objmap, status );
obsmap = astAnnul( obsmap );
objmap = astAnnul( objmap );
scimap = astAnnul( scimap );
msgOutiff( SMF__TIMER_MSG, "",
"Took %.3f s to find science observations",
status, smf_timerupdate( &tv1, &tv2, status ) );
return;
}
void smf_flat_mergedata( const smfArray * heatframes,
const double heatval[], smfData ** bolvald,
int * status ) {
double * bolval = NULL; /* Output bolometer data */
double * bolvalvar = NULL; /* Variance */
size_t i;
size_t nheat; /* Number of input frames */
size_t numbol; /* Number of bolometers */
if (bolvald) *bolvald = NULL;
if (*status != SAI__OK) return;
if (!bolvald) {
*status = SAI__ERROR;
errRep( "", "Must provide a pointer for returned smfData"
" (possible programming error)", status );
return;
}
nheat = heatframes->ndat;
numbol = (heatframes->sdata)[0]->dims[0] *
(heatframes->sdata)[0]->dims[1];
/* Create a smfData for powref and bolref */
smf_flat_malloc( nheat, (heatframes->sdata)[0], NULL, bolvald, status );
if (*status == SAI__OK) {
bolval = (*bolvald)->pntr[0];
bolvalvar = (*bolvald)->pntr[1];
/* copy the data from the individual frames to the new smfData */
for (i = 0; i<nheat; i++) {
double * datpntr = &(bolval[ i * numbol ]);
double * varpntr = &(bolvalvar[ i * numbol ] );
memcpy( datpntr, (heatframes->sdata)[i]->pntr[0],
numbol * smf_dtype_sz( (heatframes->sdata)[i]->dtype, status ) );
/* just in case we are missing variance */
if ( (heatframes->sdata)[i]->pntr[1] ) {
memcpy( varpntr, (heatframes->sdata)[i]->pntr[1],
numbol * smf_dtype_sz( (heatframes->sdata)[i]->dtype, status ) );
} else {
size_t j;
for (j = 0; j < numbol; j++) {
varpntr[j] = VAL__BADD;
}
}
}
}
/* Get the representative jcmtstate information */
if (*status == SAI__OK) {
JCMTState * state = astMalloc( nheat*sizeof(*state) );
for (i = 0; i < nheat; i++) {
smfData * frame = (heatframes->sdata)[i];
memcpy( &(state[i]), &(frame->hdr->allState)[0], sizeof(*state) );
}
(*bolvald)->hdr->allState = state;
}
/* Store the heater values in a smfDA */
if (*status == SAI__OK) {
smfDA * da = NULL;
double * dheatval = astMalloc( nheat*sizeof(*heatval) );
memcpy( dheatval, heatval, nheat * sizeof(*heatval) );
da = smf_construct_smfDA( NULL, NULL, NULL, NULL,
SMF__FLATMETH_NULL, 0, VAL__BADD,
dheatval, nheat, status );
(*bolvald)->da = da;
}
return;
}
