void fts2_getmirrorpositions(smfData* data, double* positions, int* size, int* status)
{
if(*status != SAI__OK) { return; }
char ftsMode[SZFITSTR];
size_t count = 0; /* Size */
HDSLoc* hdsLoc = NULL; /* Pointer to HDS location */
HDSLoc* hdsLocPos = NULL; /* Pointer to mirror positions */
smf_fits_getS(data->hdr, "FTS_MODE", ftsMode, sizeof(ftsMode), status);
if(strncmp(ftsMode, "FSCAN", 5) == 0 ) {
hdsLoc = smf_get_xloc(data, "JCMTSTATE", "EXT", "READ", 0, 0, status);
datFind(hdsLoc, "FTS_POS", &hdsLocPos, status);
datSize(hdsLocPos, &count, status);
datGetVD(hdsLocPos, count, positions, &count, status);
*size = (int) count;
} else if(strncmp(ftsMode, "STEPINT", 7) == 0 ) {
*status = SAI__ERROR;
errRep(FUNC_NAME, "STEPINT mode is NOT supported!", status);
} else {
*status = SAI__ERROR;
errRep(FUNC_NAME, "Unknown Scan Mode is found!", status);
}
if(hdsLoc) { datAnnul(&hdsLoc, status); }
if(hdsLocPos) { datAnnul(&hdsLocPos, status); }
}
/* private routine for scanning a JCMTSTATE TAI or date FITS header */
static double smf__find_utc( const smfHead *hdr, int first, int *status) {
double utc = VAL__BADD;
AstTimeFrame *tf = NULL;
if (*status != SAI__OK) return VAL__BADD;
tf = astTimeFrame( " " );
if (hdr->allState) {
dim_t index;
if (first) {
index = 0;
} else {
index = hdr->nframes;
}
astSet( tf, "TimeScale=TAI" );
astSet( tf, "TimeOrigin=MJD %.*g", DBL_DIG, (hdr->allState)[index].rts_end);
astSet( tf, "TimeScale=UTC" ); /* we need UTC */
} else if (hdr->fitshdr) {
/* look for DATE-OBS */
char iso[81];
const char * fitscard = NULL;
astSet( tf, "TimeScale=UTC" );
if (first) {
fitscard = "DATE-OBS";
} else {
fitscard = "DATE-END";
}
smf_fits_getS( hdr, fitscard, iso, sizeof(iso), status );
astSet( tf, "TimeOrigin=%s", iso );
} else {
*status = SAI__ERROR;
errRep( " ","Can not find date of observation without FITS header "
"or JCMTSTATE extension", status );
}
/* now read back the TimeOrigin as an MJD */
if (*status == SAI__OK) utc = astGetD( tf, "TimeOrigin" );
/* Clean up */
tf = astAnnul( tf );
return utc;
}
void smf_calc_mode ( smfHead * hdr, int * status ) {
char sam_mode[SZFITSTR]; /* Value of SAM_MODE header */
char obs_type[SZFITSTR]; /* value of OBS_TYPE header */
char sw_mode[SZFITSTR]; /* value of SW_MODE header */
char seq_type[SZFITSTR]; /* value of SEQ_TYPE header */
char inbeamstr[SZFITSTR]; /* value of INBEAM header */
smf_obstype type = SMF__TYP_NULL; /* temporary type */
smf_obstype stype = SMF__TYP_NULL; /* temporary seq type */
smf_obsmode mode = SMF__OBS_NULL; /* temporary mode */
smf_swmode swmode = SMF__SWM_NULL; /* Switching mode */
smf_inbeam_t inbeam = SMF__INBEAM_NOTHING; /* what is in beam? */
if (*status != SAI__OK) return;
if (hdr == NULL) {
*status = SAI__ERROR;
errRep( " ", "Null pointer supplied to " FUNC_NAME, status );
return;
}
/* Proceed if we're using a valid instrument */
if( hdr->instrument != INST__NONE ) {
/* Read the relevant headers */
smf_fits_getS( hdr, "SAM_MODE", sam_mode, sizeof(sam_mode), status );
smf_fits_getS( hdr, "SW_MODE", sw_mode, sizeof(sw_mode), status );
smf_fits_getS( hdr, "OBS_TYPE", obs_type, sizeof(obs_type), status );
/* INBEAM can be undef */
inbeamstr[0] = '\0';
smf_getfitss( hdr, "INBEAM", inbeamstr, sizeof(inbeamstr), status );
/* SEQ_TYPE is "new" */
if ( *status == SAI__OK ) {
smf_fits_getS( hdr, "SEQ_TYPE", seq_type, sizeof(seq_type), status );
if (*status == SMF__NOKWRD ) {
errAnnul( status );
one_strlcpy( seq_type, obs_type, sizeof(seq_type), status );
}
}
/* start with sample type */
if (strcasecmp( sam_mode, "SCAN" ) == 0 ||
strcasecmp( sam_mode, "RASTER") == 0 ) {
mode = SMF__OBS_SCAN;
} else if (strcasecmp( sam_mode, "STARE" ) == 0) {
mode = SMF__OBS_STARE;
} else if (strcasecmp( sam_mode, "DREAM" ) == 0) {
mode = SMF__OBS_DREAM;
} else if (strcasecmp( sam_mode, "JIGGLE" ) == 0) {
mode = SMF__OBS_JIGGLE;
} else if (strcasecmp( sam_mode, "GRID" ) == 0) {
mode = SMF__OBS_GRID;
} else {
if (*status != SAI__OK) {
*status = SAI__ERROR;
msgSetc( "MOD", sam_mode );
errRep( " ", "Unrecognized observing mode '^MOD'", status );
}
}
/* switching mode: options are "none", "pssw", "chop", "freqsw", "self" */
if (strcasecmp( sw_mode, "NONE" ) == 0 ) {
swmode = SMF__SWM_NULL;
} else if (strcasecmp( sw_mode, "PSSW" ) == 0) {
swmode = SMF__SWM_PSSW;
} else if (strcasecmp( sw_mode, "CHOP" ) == 0) {
swmode = SMF__SWM_CHOP;
} else if (strcasecmp( sw_mode, "SELF" ) == 0) {
swmode = SMF__SWM_SELF;
} else if (strcasecmp( sw_mode, "FREQSW" ) == 0) {
swmode = SMF__SWM_FREQSW;
} else {
if (*status != SAI__OK) {
*status = SAI__ERROR;
msgSetc( "MOD", sw_mode );
errRep( " ", "Unrecognized switching mode '^MOD'", status );
}
}
/* obs type */
type = smf__parse_obstype( obs_type, status );
stype = smf__parse_obstype( seq_type, status );
/* in beam (convert to upper case to make it case insensitive) */
astChrCase( NULL, inbeamstr, 1, 0 );
if ( smf_pattern_extract( inbeamstr, "(POL)", NULL, NULL, 0, status ) ) {
inbeam |= SMF__INBEAM_POL;
}
if ( smf_pattern_extract( inbeamstr, "(FTS)", NULL, NULL, 0, status ) ) {
inbeam |= SMF__INBEAM_FTS;
}
if ( smf_pattern_extract( inbeamstr, "(BODY)", NULL, NULL, 0, status ) ) {
inbeam |= SMF__INBEAM_BLACKBODY;
/* We could consider ensuring FTS is not set in this case */
}
}
hdr->obstype = type;
hdr->seqtype = stype;
hdr->obsmode = mode;
hdr->swmode = swmode;
hdr->inbeam = inbeam;
}
void smf_find_science(const Grp * ingrp, Grp **outgrp, int reverttodark,
Grp **darkgrp, Grp **flatgrp, int reducedark,
int calcflat, smf_dtype darktype, smfArray ** darks,
smfArray **fflats, AstKeyMap ** heateffmap,
double * meanstep, int * status ) {
smfSortInfo *alldarks; /* array of sort structs for darks */
smfSortInfo *allfflats; /* array of fast flat info */
Grp * dgrp = NULL; /* Internal dark group */
double duration_darks = 0.0; /* total duration of all darks */
double duration_sci = 0.0; /* Duration of all science observations */
size_t dkcount = 0; /* Dark counter */
size_t ffcount = 0; /* Fast flat counter */
Grp * fgrp = NULL; /* Fast flat group */
size_t i; /* loop counter */
smfData *infile = NULL; /* input file */
size_t insize; /* number of input files */
size_t nsteps_dark = 0; /* Total number of steps for darks */
size_t nsteps_sci = 0; /* Total number of steps for science */
AstKeyMap * heatermap = NULL; /* Heater efficiency map */
AstKeyMap * obsmap = NULL; /* Info from all observations */
AstKeyMap * objmap = NULL; /* All the object names used */
AstKeyMap * scimap = NULL; /* All non-flat obs indexed by unique key */
Grp *ogrp = NULL; /* local copy of output group */
size_t sccount = 0; /* Number of accepted science files */
struct timeval tv1; /* Timer */
struct timeval tv2; /* Timer */
if (meanstep) *meanstep = VAL__BADD;
if (outgrp) *outgrp = NULL;
if (darkgrp) *darkgrp = NULL;
if (darks) *darks = NULL;
if (fflats) *fflats = NULL;
if (heateffmap) *heateffmap = NULL;
if (*status != SAI__OK) return;
/* Sanity check to make sure we return some information */
if ( outgrp == NULL && darkgrp == NULL && darks == NULL && fflats == NULL) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": Must have some non-NULL arguments"
" (possible programming error)", status);
return;
}
/* Start a timer to see how long this takes */
smf_timerinit( &tv1, &tv2, status );
/* Create new group for output files */
ogrp = smf_grp_new( ingrp, "Science", status );
/* and a new group for darks */
dgrp = smf_grp_new( ingrp, "DarkFiles", status );
/* and for fast flats */
fgrp = smf_grp_new( ingrp, "FastFlats", status );
/* and also create a keymap for the observation description */
obsmap = astKeyMap( "KeyError=1" );
/* and an object map */
objmap = astKeyMap( "KeyError=1" );
/* This keymap contains the sequence counters for each related
subarray/obsidss/heater/shutter combination and is used to decide
if a bad flat is relevant */
scimap = astKeyMap( "KeyError=1,KeyCase=0" );
/* This keymap is used to contain relevant heater efficiency data */
heatermap = astKeyMap( "KeyError=1,KeyCase=0" );
/* Work out how many input files we have and allocate sufficient sorting
space */
insize = grpGrpsz( ingrp, status );
alldarks = astCalloc( insize, sizeof(*alldarks) );
allfflats = astCalloc( insize, sizeof(*allfflats) );
/* check each file in turn */
for (i = 1; i <= insize; i++) {
int seqcount = 0;
char keystr[100]; /* Key for scimap entry */
/* open the file but just to get the header */
smf_open_file( ingrp, i, "READ", SMF__NOCREATE_DATA, &infile, status );
if (*status != SAI__OK) break;
/* Fill in the keymap with observation details */
smf_obsmap_fill( infile, obsmap, objmap, status );
/* Find the heater efficiency map if required */
if (*status == SAI__OK && heateffmap) {
char arrayidstr[32];
smf_fits_getS( infile->hdr, "ARRAYID", arrayidstr, sizeof(arrayidstr),
status );
if (!astMapHasKey( heatermap, arrayidstr ) ) {
smfData * heateff = NULL;
dim_t nbolos = 0;
smf_flat_params( infile, "RESIST", NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, &heateff, status );
smf_get_dims( heateff, NULL, NULL, &nbolos, NULL, NULL, NULL, NULL,
status );
if (heateff) astMapPut0P( heatermap, arrayidstr, heateff, NULL );
}
}
/* Get the sequence counter for the file. We do not worry about
duplicate sequence counters (at the moment) */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* The key identifying this subarray/obsidss/heater/shutter combo */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
if (smf_isdark( infile, status )) {
/* Store the sorting information */
dkcount = smf__addto_sortinfo( infile, alldarks, i, dkcount, "Dark", status );
smf__addto_durations( infile, &duration_darks, &nsteps_dark, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
} else {
/* compare sequence type with observation type and drop it (for now)
if they differ */
if ( infile->hdr->obstype == infile->hdr->seqtype ) {
/* Sanity check the header for corruption. Compare RTS_NUM with SEQSTART
and SEQEND. The first RTS_NUM must either be SEQSTART or else between
SEQSTART and SEQEND (if someone has giving us a section) */
int seqstart = 0;
int seqend = 0;
int firstnum = 0;
JCMTState *tmpState = NULL;
smf_getfitsi( infile->hdr, "SEQSTART", &seqstart, status );
smf_getfitsi( infile->hdr, "SEQEND", &seqend, status );
tmpState = infile->hdr->allState;
if( tmpState ) {
firstnum = (tmpState[0]).rts_num;
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
if ( firstnum >= seqstart && firstnum <= seqend ) {
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif(MSG__DEBUG, " ", "Non-dark file: ^F",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
} else {
msgOutif( MSG__QUIET, "",
"File ^F has a corrupt FITS header. Ignoring it.",
status );
}
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif( MSG__DEBUG, " ",
"File ^F lacks JCMTState: assuming it is non-dark",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
}
} else if (infile->hdr->seqtype == SMF__TYP_FASTFLAT ) {
ffcount = smf__addto_sortinfo( infile, allfflats, i, ffcount, "Fast flat", status );
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
msgOutif(MSG__DEBUG, " ", "Sequence type mismatch with observation type: ^F",status);
}
}
/* close the file */
smf_close_file( &infile, status );
}
/* Store output group in return variable or else free it */
if (outgrp) {
*outgrp = ogrp;
} else {
grpDelet( &ogrp, status );
}
/* process flatfields if necessary */
if (ffcount > 0 && fflats ) {
smfArray * array = NULL;
/* sort flats into order */
qsort( allfflats, ffcount, sizeof(*allfflats), smf_sort_bydouble);
if (fflats) array = smf_create_smfArray( status );
/* now open the flats and store them if requested */
if (*status == SAI__OK && array && ffcount) {
size_t start_ffcount = ffcount;
AstKeyMap * flatmap = NULL;
if (calcflat) {
/* Use AgeUp so that we get the keys out in the sorted order
that allfflats used */
flatmap = astKeyMap( "KeyCase=0,KeyError=1,SortBy=AgeDown" );
}
/* Read each flatfield. Calculate a responsivity image and a flatfield
solution. Store these in a keymap along with related information
which is itself stored in a keymap indexed by a string made of
OBSIDSS, reference heater value, shutter and subarray.
*/
for (i = 0; i < start_ffcount; i++ ) {
size_t ori_index = (allfflats[i]).index;
smfData * outfile = NULL;
char keystr[100];
AstKeyMap * infomap = astKeyMap( "KeyError=1" );
int oplen = 0;
char thisfile[MSG__SZMSG];
int seqcount = 0;
/* read filename from group */
infile = NULL;
smf_open_file( ingrp, ori_index, "READ", 0, &infile, status );
if ( *status != SAI__OK ) {
/* This should not happen because we have already opened
the file. If it does happen we abort with error. */
if (infile) smf_close_file( &infile, status );
break;
}
/* Calculate the key for this observation */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
/* Get the file name for error messages */
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>" );
msgLoad( "", "^F", thisfile, sizeof(thisfile), &oplen, status );
/* And the sequence counter to link against science observations */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* Prefill infomap */
astMapPut0C( infomap, "FILENAME", thisfile, "");
astMapPut0I( infomap, "SEQCOUNT", seqcount, "");
/* Collapse it */
if (*status == SAI__OK) {
smf_flat_fastflat( infile, &outfile, status );
if (*status == SMF__BADFLAT) {
errFlush( status );
if (calcflat) {
/* Need to generate an outfile like smf_flat_fastflat
and one heater setting will force smf_flat_calcflat to fail */
smf_flat_malloc( 1, infile, NULL, &outfile, status );
} else {
if (outfile) smf_close_file( &outfile, status );
if (infile) smf_close_file( &infile, status );
infomap = astAnnul( infomap );
ffcount--;
continue;
}
}
}
if (outfile && *status == SAI__OK) {
smf_close_file( &infile, status );
infile = outfile;
if (calcflat) {
size_t ngood = 0;
smfData * curresp = NULL;
int utdate;
if (*status == SAI__OK) {
ngood = smf_flat_calcflat( MSG__VERB, NULL, "RESIST",
"FLATMETH", "FLATORDER", NULL, "RESPMASK",
"FLATSNR", NULL, infile, &curresp, status );
if (*status != SAI__OK) {
/* if we failed to calculate a flatfield we continue but force the
flatfield to be completely bad. This will force the science data associated
with the flatfield to be correctly blanked. We do not annul though
if we have a SUBPAR error telling us that we have failed to define
our parameters properly. */
if (*status != SUBPAR__NOPAR) errAnnul(status);
/* parameters of flatfield */
ngood = 0;
/* Generate a blank flatfield and blank responsivity image */
smf_flat_badflat( infile, &curresp, status );
}
/* Retrieve the UT date so we can decide whether to compare
flatfields */
smf_getfitsi( infile->hdr, "UTDATE", &utdate, status );
/* Store the responsivity data for later on and the processed
flatfield until we have vetted it */
astMapPut0P( infomap, "CALCFLAT", infile, "" );
astMapPut0P( infomap, "RESP", curresp, "" );
astMapPut0I( infomap, "UTDATE", utdate, "" );
astMapPut0I( infomap, "ISGOOD", 1, "" );
astMapPut0I( infomap, "NGOOD", ngood, "" );
astMapPut0I( infomap, "GRPINDEX", ori_index, "" );
astMapPut0I( infomap, "SMFTYP", infile->hdr->obstype, "" );
astMapPutElemA( flatmap, keystr, -1, infomap );
}
} else { /* if (calcflat) */
/* Store the collapsed flatfield - the processed flat is not stored here yet */
smf_addto_smfArray( array, infile, status );
/* Copy the group info */
ndgCpsup( ingrp, ori_index, fgrp, status );
}
} /* if (outfile) */
/* Annul the keymap (will be fine if it is has been stored in another keymap) */
infomap = astAnnul( infomap );
} /* End loop over flatfields */
/* Now we have to loop over the related flatfields to disable
bolometers that are not good and also decide whether we
need to set status to bad. */
if (*status == SAI__OK && calcflat ) {
size_t nkeys = astMapSize( flatmap );
for (i = 0; i < nkeys; i++ ) {
const char *key = astMapKey( flatmap, i );
int nf = 0;
AstKeyMap ** kmaps = NULL;
int nelem = astMapLength( flatmap, key );
kmaps = astMalloc( sizeof(*kmaps) * nelem );
astMapGet1A( flatmap, key, nelem, &nelem, kmaps );
for ( nf = 0; nf < nelem && *status == SAI__OK; nf++ ) {
AstKeyMap * infomap = kmaps[nf];
int isgood = 0;
astMapGet0I( infomap, "ISGOOD", &isgood );
if (isgood) {
/* The flatfield worked */
size_t ngood = 0;
int itemp;
int utdate = 0;
int ratioFlats = 0;
/* Get the UT date - we do not compare flatfields after
the time we enabled heater tracking at each sequence. */
astMapGet0I( infomap, "UTDATE", &utdate );
/* Get the number of good bolometers at this point */
astMapGet0I( infomap, "NGOOD", &itemp );
ngood = itemp;
/* Decide if we want to do the ratio test. We default to
not doing it between 20110901 and 20120827 which is
the period when we did mini-heater tracks before each
flat. ! indicates that we choose based on date. */
if (*status == SAI__OK) {
parGet0l( "FLATUSENEXT", &ratioFlats, status );
if ( *status == PAR__NULL ) {
errAnnul( status );
if (utdate >= 20110901 || utdate <= 20120827 ) {
ratioFlats = 0;
} else {
ratioFlats = 1;
}
}
}
/* Can we compare with the next flatfield? */
if (ngood < SMF__MINSTATSAMP || !ratioFlats ) {
/* no point doing all the ratio checking for this */
} else if ( nelem - nf >= 2 ) {
AstKeyMap * nextmap = kmaps[nf+1];
const char *nextfname = NULL;
const char *fname = NULL;
smfData * curresp = NULL;
smfData * nextresp = NULL;
smfData * curflat = NULL;
void *tmpvar = NULL;
size_t bol = 0;
smfData * ratio = NULL;
double *in1 = NULL;
double *in2 = NULL;
double mean = VAL__BADD;
size_t nbolo;
double *out = NULL;
double sigma = VAL__BADD;
float clips[] = { 5.0, 5.0 }; /* 5.0 sigma iterative clip */
size_t ngoodz = 0;
astMapGet0C( nextmap, "FILENAME", &nextfname );
astMapGet0C( infomap, "FILENAME", &fname );
/* Retrieve the responsivity images from the keymap */
astMapGet0P( infomap, "RESP", &tmpvar );
curresp = tmpvar;
astMapGet0P( nextmap, "RESP", &tmpvar );
nextresp = tmpvar;
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
nbolo = (curresp->dims)[0] * (curresp->dims)[1];
/* get some memory for the ratio if we have not already.
We could get some memory once assuming each flat has the
same number of bolometers... */
ratio = smf_deepcopy_smfData( curresp, 0, 0, 0, 0, status );
if( *status == SAI__OK ) {
/* divide: smf_divide_smfData ? */
in1 = (curresp->pntr)[0];
in2 = (nextresp->pntr)[0];
out = (ratio->pntr)[0];
for (bol=0; bol<nbolo;bol++) {
if ( in1[bol] != VAL__BADD && in1[bol] != 0.0 &&
in2[bol] != VAL__BADD && in2[bol] != 0.0 ) {
out[bol] = in1[bol] / in2[bol];
} else {
out[bol] = VAL__BADD;
}
}
}
/* find some statistics */
smf_clipped_stats1D( out, 2, clips, 1, nbolo, NULL, 0, 0, &mean,
&sigma, NULL, 0, &ngoodz, status );
if (*status == SMF__INSMP) {
errAnnul(status);
msgOutiff( MSG__QUIET, "",
"Flatfield ramp ratio of %s with %s had too few bolometers (%zu < %d).",
status, fname, nextfname, ngoodz, SMF__MINSTATSAMP );
ngood = ngoodz; /* Must be lower or equal to original ngood */
} else if (*status == SAI__OK && mean != VAL__BADD && sigma != VAL__BADD && curflat->da) {
/* Now flag the flatfield as bad for bolometers that have changed
more than n%. We expect the variation to be 1+/-a small bit */
const double pmrange = 0.10;
double thrlo = 1.0 - pmrange;
double thrhi = 1.0 + pmrange;
size_t nmasked = 0;
double *flatcal = curflat->da->flatcal;
msgOutiff( MSG__DEBUG, "", "Flatfield fast ramp ratio mean = %g +/- %g (%zu bolometers)",
status, mean, sigma, ngood);
/* we can just set the first slice of the flatcal to bad. That should
be enough to disable the entire bolometer. We have just read these
data so they should be in ICD order. */
for (bol=0; bol<nbolo;bol++) {
if ( out[bol] != VAL__BADD &&
(out[bol] < thrlo || out[bol] > thrhi ) ) {
flatcal[bol] = VAL__BADD;
nmasked++;
} else if ( in1[bol] != VAL__BADD && in2[bol] == VAL__BADD ) {
/* A bolometer is bad next time but good now so we must set it bad now */
flatcal[bol] = VAL__BADD;
nmasked++;
}
}
if ( nmasked > 0 ) {
msgOutiff( MSG__NORM, "", "Masked %zu bolometers in %s from unstable flatfield",
status, nmasked, fname );
/* update ngood to take into account the masking */
ngood -= nmasked;
}
}
smf_close_file( &ratio, status );
} /* End of flatfield responsivity comparison */
/* if we only have a few bolometers left we now consider this
a bad flat unless it is actually an engineering measurement
where expect some configurations to give zero bolometers */
if (ngood < SMF__MINSTATSAMP) {
const char *fname = NULL;
void * tmpvar = NULL;
int smftyp = 0;
smfData * curflat = NULL;
astMapGet0I( infomap, "SMFTYP", &smftyp );
astMapGet0C( infomap, "FILENAME", &fname );
if (smftyp != SMF__TYP_NEP) {
msgOutiff( MSG__QUIET, "",
"Flatfield %s has %zu good bolometer%s.%s",
status, fname, ngood, (ngood == 1 ? "" : "s"),
( ngood == 0 ? "" : " Keeping none.") );
isgood = 0;
/* Make sure that everything is blanked. */
if (ngood > 0) {
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
if (curflat && curflat->da) {
size_t bol;
size_t nbolo = (curflat->dims)[0] * (curflat->dims)[1];
double *flatcal = curflat->da->flatcal;
for (bol=0; bol<nbolo; bol++) {
/* Just need to set the first element to bad */
flatcal[bol] = VAL__BADD;
}
}
}
} else {
msgOutiff( MSG__NORM, "",
"Flatfield ramp file %s has %zu good bolometer%s. Eng mode.",
status, fname, ngood, (ngood == 1 ? "" : "s") );
}
}
/* We do not need the responsivity image again */
{
void *tmpvar = NULL;
smfData * resp = NULL;
astMapGet0P( infomap, "RESP", &tmpvar );
resp = tmpvar;
if (resp) smf_close_file( &resp, status );
astMapRemove( infomap, "RESP" );
}
} /* End of isgood comparison */
/* We are storing flats even if they failed. Let the downstream
software worry about it */
{
int ori_index;
smfData * flatfile = NULL;
void *tmpvar = NULL;
/* Store in the output group */
astMapGet0I( infomap, "GRPINDEX", &ori_index );
ndgCpsup( ingrp, ori_index, fgrp, status );
/* And store in the smfArray */
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
astMapRemove( infomap, "CALCFLAT" );
flatfile = tmpvar;
smf_addto_smfArray( array, flatfile, status );
}
/* Free the object as we go */
kmaps[nf] = astAnnul( kmaps[nf] );
} /* End of loop over this obsidss/subarray/heater */
kmaps = astFree( kmaps );
}
}
if (array->ndat) {
if (fflats) *fflats = array;
} else {
smf_close_related(&array, status );
if (fflats) *fflats = NULL;
}
}
}
/* no need to do any more if neither darks nor darkgrp are defined or we might
be wanting to revert to darks. */
if (dkcount > 0 && (darks || darkgrp || reverttodark ) ) {
smfArray * array = NULL;
/* sort darks into order */
qsort( alldarks, dkcount, sizeof(*alldarks), smf_sort_bydouble);
if (darks) array = smf_create_smfArray( status );
/* now open the darks and store them if requested */
if (*status == SAI__OK) {
for (i = 0; i < dkcount; i++ ) {
size_t ori_index = (alldarks[i]).index;
/* Store the entry in the output group */
ndgCpsup( ingrp, ori_index, dgrp, status );
if (darks) {
/* read the value from the new group */
smf_open_file( dgrp, i+1, "READ", 0, &infile, status );
/* do we have to process these darks? */
if (reducedark) {
smfData *outfile = NULL;
smf_reduce_dark( infile, darktype, &outfile, status );
if (outfile) {
smf_close_file( &infile, status );
infile = outfile;
}
}
smf_addto_smfArray( array, infile, status );
}
}
if (darks) *darks = array;
}
}
/* free memory */
alldarks = astFree( alldarks );
allfflats = astFree( allfflats );
if( reverttodark && outgrp && (grpGrpsz(*outgrp,status)==0) &&
(grpGrpsz(dgrp,status)>0) ) {
/* If outgrp requested but no science observations were found, and
dark observations were found, return darks in outgrp and set
flatgrp and darkgrp to NULL. This is to handle cases where we
want to process data taken in the dark like normal science
data. To activate this behaviour set reverttodark */
msgOutiff( MSG__NORM, "", "Treating the dark%s as science data",
status, ( dkcount > 1 ? "s" : "" ) );
*outgrp = dgrp;
if( darkgrp ){
*darkgrp = NULL;
}
if( flatgrp ) {
*flatgrp = NULL;
}
grpDelet( &ogrp, status);
grpDelet( &fgrp, status);
if (meanstep && nsteps_dark > 0) *meanstep = duration_darks / nsteps_dark;
/* Have to clear the darks smfArray as well */
if (darks) smf_close_related( darks, status );
} else {
/* Store the output groups in the return variable or free it */
if (darkgrp) {
*darkgrp = dgrp;
} else {
grpDelet( &dgrp, status);
}
if (flatgrp) {
*flatgrp = fgrp;
} else {
grpDelet( &fgrp, status);
}
if (meanstep && nsteps_sci > 0) *meanstep = duration_sci / nsteps_sci;
}
msgSeti( "ND", sccount );
msgSeti( "DK", dkcount );
msgSeti( "FF", ffcount );
msgSeti( "TOT", insize );
if ( insize == 1 ) {
if (dkcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a dark",
status);
} else if (ffcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a fast flatfield",
status);
} else if (sccount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was accepted (observation type same as sequence type)",
status);
} else {
msgOutif( MSG__VERB, " ", "Single input file was not accepted.",
status);
}
} else {
if (dkcount == 1) {
msgSetc( "DKTXT", "was a dark");
} else {
msgSetc( "DKTXT", "were darks");
}
if (ffcount == 1) {
msgSetc( "FFTXT", "was a fast flat");
} else {
msgSetc( "FFTXT", "were fast flats");
}
if (sccount == 1) {
msgSetc( "NDTXT", "was science");
} else {
msgSetc( "NDTXT", "were science");
}
/* This might be a useful message */
msgOutif( MSG__NORM, " ", "Out of ^TOT input files, ^DK ^DKTXT, ^FF ^FFTXT "
"and ^ND ^NDTXT", status );
}
if (meanstep && *meanstep != VAL__BADD) {
msgOutiff( MSG__VERB, "", "Mean step time for input files = %g sec",
status, *meanstep );
}
/* Store the heater efficiency map */
if (*status != SAI__OK) heatermap = smf_free_effmap( heatermap, status );
if (heateffmap) *heateffmap = heatermap;
/* Now report the details of the observation */
smf_obsmap_report( MSG__NORM, obsmap, objmap, status );
obsmap = astAnnul( obsmap );
objmap = astAnnul( objmap );
scimap = astAnnul( scimap );
msgOutiff( SMF__TIMER_MSG, "",
"Took %.3f s to find science observations",
status, smf_timerupdate( &tv1, &tv2, status ) );
return;
}
void
smf_flat_params( const smfData * refdata, const char resistpar[],
const char methpar[], const char orderpar[], const char snrminpar[],
double * refohms, double **resistance, int * outrows,
int * outcols, smf_flatmeth *flatmeth,
int * order, double * snrmin, smfData ** heateff,
int * status ) {
dim_t datarows = 0; /* Number of rows in refdata */
dim_t datacols = 0; /* Number of columns in refdata */
size_t j = 0; /* Counter, index */
char method[SC2STORE_FLATLEN]; /* flatfield method string */
size_t nbols; /* Number of bolometers */
double refohmsval = 0.0; /* Internal version of refohms */
AstKeyMap * resmap = NULL; /* Resistor map */
AstKeyMap * subarrays = NULL; /* Subarray lookup table */
char thissub[32]; /* This sub-instrument string */
if (resistance) *resistance = NULL;
if (*status != SAI__OK) return;
if (!refdata) {
*status = SAI__ERROR;
errRep( "", "Must provide reference data file to calculate flatfield parameters"
" (possible programming error)", status );
return;
}
/* Based on refdata we now need to calculate the default reference
resistance and retrieve the correct heater efficiency file for each array.
We need the unique subarray string so that we can set up a look up keymap.
There is no code in SMURF to return all the known subarrays but
we need to know all the options in order to use kpg1Config. */
subarrays = astKeyMap( " " );
astMapPut0I( subarrays, "CG450MK2_M0907D0501", 0, NULL );
astMapPut0I( subarrays, "CG850MK2_M0904D0503", 0, NULL );
astMapPut0I( subarrays, "SG850_M0906D1005", 0, NULL );
astMapPut0I( subarrays, "SG850_M1002D1006", 0, NULL );
astMapPut0I( subarrays, "SG850_M1005D1007", 0, NULL );
astMapPut0I( subarrays, "SG850_M1003D1004", 0, NULL );
astMapPut0I( subarrays, "SG450_M1004D1000", 0, NULL );
astMapPut0I( subarrays, "SG450_M1007D1002", 0, NULL );
astMapPut0I( subarrays, "SG450_M1006D1003", 0, NULL );
astMapPut0I( subarrays, "SG450_M1009D1008", 0, NULL );
/* and indicate which subarray we are interested in (uppercased) */
smf_fits_getS( refdata->hdr, "ARRAYID", thissub, sizeof(thissub), status );
{ /* need to uppercase */
size_t l = strlen(thissub);
for (j=0;j<l;j++) {
thissub[j] = toupper(thissub[j]);
}
}
astMapPut0I( subarrays, thissub, 1, NULL );
/* Read the config file */
resmap = kpg1Config( resistpar, "$SMURF_DIR/smurf_calcflat.def",
subarrays, 1, status );
subarrays = astAnnul( subarrays );
if (*status != SAI__OK) goto CLEANUP;
/* Read the reference resistance */
astMapGet0D( resmap, "REFRES", &refohmsval );
if (refohms && *status == SAI__OK) {
*refohms = refohmsval;
msgOutiff(MSG__VERB, "",
"Read reference resistance for subarray %s of %g ohms\n",
status, thissub, *refohms );
}
/* We no longer want to read per-bolometer resistor values from the
config file. To retain backwards compatibility with the current
implementation of smf_flat_standardpow we simply fill the
per-bol resistance array with the reference resistance which
effectively disables smf_flat_standardpow */
smf_get_dims( refdata, &datarows, &datacols, NULL, NULL, NULL, NULL, NULL, status );
nbols = datacols * datarows;
if (*status == SAI__OK && resistance ) {
*resistance = astMalloc( nbols*sizeof(**resistance) );
for (j = 0; j < (size_t)nbols; j++) {
(*resistance)[j] = refohmsval;
}
}
/* Get the heater efficiency file */
if (heateff && astMapHasKey( resmap, "HEATEFF" ) ) {
const char * heateffstr = NULL;
if (astMapGet0C( resmap, "HEATEFF", &heateffstr )) {
Grp * heateffgrp = NULL;
smfData * heatefftmp = NULL;
heateffgrp = grpNew( "heateff", status );
grpPut1( heateffgrp, heateffstr, 0, status );
smf_open_file( NULL, heateffgrp, 1, "READ", SMF__NOTTSERIES|SMF__NOFIX_METADATA, &heatefftmp, status );
/* Divorce the smfData from the underlying file. This file stays open for the entire
duration of the data processing and can some times lead to issues when we attempt
to close it an hour after we opened it (it's usually on an NFS disk) */
if (*status == SAI__OK) {
*heateff = smf_deepcopy_smfData( NULL, heatefftmp, 0,
SMF__NOCREATE_FILE | SMF__NOCREATE_FTS |
SMF__NOCREATE_DA,
0, 0, status );
smf_close_file(NULL, &heatefftmp, status);
}
/* Check the dimensions */
if (*status == SAI__OK) {
dim_t heatrows = 0;
dim_t heatcols = 0;
smf_get_dims( *heateff, &heatrows, &heatcols, NULL, NULL, NULL, NULL, NULL, status );
if (*status == SAI__OK) {
if ( datarows != heatrows || datacols != heatcols ) {
*status = SAI__ERROR;
errRepf( "", "Dimensions of heater efficiency file %s are (%zu, %zu)"
" but flatfield has dimensions (%zu, %zu)",
status, heateffstr, (size_t)heatrows, (size_t)heatcols,
(size_t)datarows, (size_t)datacols);
}
}
if (*status == SAI__OK) {
smf_dtype_check_fatal( *heateff, NULL, SMF__DOUBLE, status );
if (*status == SMF__BDTYP) {
errRepf("", "Heater efficiency data in %s should be double precision",
status, heateffstr);
}
}
if (*status == SAI__OK) {
char heateffarrid[32];
smf_fits_getS( refdata->hdr, "ARRAYID", heateffarrid, sizeof(heateffarrid), status );
if (*status != SAI__OK) errAnnul( status );
if (strcasecmp( thissub, heateffarrid ) != 0 ) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRepf("", "Subarray associated with heater efficiency image (%s)"
" does not match that of the data to be flatfielded (%s)",
status, heateffarrid, thissub );
}
}
}
}
if (heateffgrp) grpDelet( &heateffgrp, status );
}
}
if (methpar && flatmeth) {
/* See if we want to use TABLE or POLYNOMIAL mode */
parChoic( methpar, "POLYNOMIAL", "POLYNOMIAL, TABLE", 1,
method, sizeof(method), status );
*flatmeth = smf_flat_methcode( method, status );
if (*flatmeth == SMF__FLATMETH_POLY) {
/* need an order for the polynomial */
if (order && orderpar) {
parGdr0i( orderpar, 1, 1, 3, 1, order, status );
/* and if the order is 1 then we can ask for the snr min */
if (snrminpar && *order == 1) {
parGet0d( snrminpar, snrmin, status );
}
}
} else {
/* need an snr min for table mode responsivities */
if (snrminpar) parGet0d( snrminpar, snrmin, status );
}
}
if (outrows) *outrows = datarows;
if (outcols) *outcols = datacols;
CLEANUP:
resmap = astAnnul( resmap );
if (*status != SAI__OK) {
if (resistance && *resistance) *resistance = astFree( *resistance );
if (heateff && *heateff) smf_close_file( NULL, heateff, status );
}
return;
}
void smurf_fts2_split(int* status)
{
if( *status != SAI__OK ) { return; }
const double STAGE_LENGTH = 450.0; /* mm */
int LR = 0; /* Treat as Low Resolution scan */
Grp* gIn = NULL; /* Input group */
Grp* gOut = NULL; /* Output group */
Grp* gTmp = NULL; /* Temporary group */
smfData* inData = NULL; /* Pointer to input data */
smfData* outData = NULL; /* Pointer to output data */
double* outData_pntr = NULL; /* Pointer to output data values array */
int nMirPos = 0; /* Number of frames where the mirror actually moves */
int nStart = 0; /* Frame index where the mirror starts moving */
int nStartNext = 0; /* Frame index where the mirror starts moving in the next scan */
int nStop = 0; /* Frame index where the mirror stops */
int lrStart = 0; /* Frame index where low resolution mirror limit starts */
int hrStop = 0; /* Frame index where high resolution mirror limit stops */
int hrStart = 0; /* Frame index where high resolution mirror limit starts */
int lrStop = 0; /* Frame index where low resolution mirror limit stops */
int lrCentre = 0; /* Frame index at centre of low resolution mirror positions */
int i = 0; /* Counter */
int j = 0; /* Counter */
int k = 0; /* Counter */
int n = 0; /* Counter */
double fNyquist = 0.0; /* Nyquist frequency */
double dz = 0.0; /* Step size in evenly spaced OPD grid */
double* MIRPOS = NULL; /* Mirror positions */
size_t nFiles = 0; /* Size of the input group */
size_t nOutFiles = 0; /* Size of the output group */
size_t fIndex = 0; /* File index */
size_t nWidth = 0; /* Data cube width */
size_t nHeight = 0; /* Data cube height */
size_t nFrames = 0; /* Data cube depth in input file */
size_t nFramesOut = 0; /* Data cube depth in output file */
size_t nFramesOutPrev = 0; /* Data cube depth in previous output file */
size_t hrFramesOut = 0; /* Data cube depth in high res output file */
size_t hrFramesOutPrev = 0; /* Data cube depth in previous high res output file */
size_t lrFramesOut = 0; /* Data cube depth in low res output file */
size_t lrFramesOutPrev = 0; /* Data cube depth in previous low res output file */
size_t nPixels = 0; /* Number of bolometers in the subarray */
char object[SZFITSTR];
char subarray[SZFITSTR];
char obsID[SZFITSTR];
char scanMode[SZFITSTR];
double scanVel = 0.0; /* Mirror speed in mm/sec */
double stepTime = 0.0; /* RTS step time, average sample rate */
double minOPD = 0; /* OPD minimum */
double maxOPD = 0; /* OPD maximum */
double ZPD = 0;
double lrmmBandPass = 0.0; /* low res mm +/- offset from centre */
int lrBandPassFrames = 0; /* Number of low res band pass frames from centre +/- length of lrmmBandPass */
int nTmp = 0;
int nMax = 0;
int nOPD = 0;
int bolIndex = 0;
int index = 0;
int indexIn = 0;
int indexOut = 0;
int badPixel = 0;
int k0 = 0;
int indexZPD = 0;
int done = 0; /* Track completion of extracting multiple scans */
int outDataCount = 0; /* The number of output data files being written */
double lenLeft,
lenRight,
minLenLeft,
minLenRight,
minLen,
minZPD,
maxZPD,
midZPD = 0.0; /* Mirror position half side measures */
int midZPDPos = 0; /* Middle ZPD position in mirror position array */
double EPSILON = 0.0;
char fileName[SMF_PATH_MAX+1];
char scanNumStr[5+1]; /* String form of scan number of the input file */
int scanNum = 0; /* Scan number of the input file */
int conNum = 0; /* Concatenation number of the input file (left shifted scanNum) */
int scanDir = 0; /* Scan direction: 1 -> back to front (positive), -1 -> front to back (negative) */
JCMTState *allState = NULL; /* Temporary buffer for reduced header allState array data */
/* Get Input, Output groups */
kpg1Rgndf("IN", 0, 1, "", &gIn, &nFiles, status);
kpg1Wgndf("OUT", gOut, nFiles, nFiles, "More output files expected!", &gOut, &nOutFiles, status);
/* Read in ADAM parameters */
parGet0d("BANDPASS", &lrmmBandPass, status); /* Low res mm band +/- offset from centre */
/* Treat as Low Resolution scan? */
if(lrmmBandPass > 0) {
LR = 1;
}
/* Eliminate the first record in the output group, since it will be replaced later */
gTmp = grpCopy(gOut, 1, 1, 1, status);
grpDelet(&gOut, status);
gOut = gTmp;
/* BEGIN NDF */
ndfBegin();
/* Loop through each input file */
for(fIndex = 1; fIndex <= nFiles; fIndex++) {
/* Open Observation file */
smf_open_file(gIn, fIndex, "READ", 0, &inData, status);
if(*status != SAI__OK) {
*status = SAI__ERROR;
errRep(FUNC_NAME, "Unable to open the source file!", status);
goto CLEANUP;
}
smf_fits_getS(inData->hdr, "OBJECT", object, sizeof(object), status);
smf_fits_getS(inData->hdr, "SUBARRAY", subarray, sizeof(subarray), status);
smf_fits_getS(inData->hdr, "OBSID", obsID, sizeof(obsID), status);
smf_fits_getS(inData->hdr, "FTS_MODE", scanMode, sizeof(scanMode), status);
smf_fits_getD(inData->hdr, "SCANVEL", &scanVel, status);
smf_fits_getD(inData->hdr, "STEPTIME", &stepTime, status);
/* Nyquist frequency */
fNyquist = 10.0 / (8.0 * scanVel * stepTime);
dz = 1.0 / (2.0 * fNyquist);
EPSILON = scanVel * stepTime / 2;
/* Extract the scan number from the input file to be incremented in the output files */
one_strlcpy(scanNumStr, &(inData->file->name[strlen(inData->file->name) - 8]),
astMIN(SMF_PATH_MAX + 1, 5), status);
if (*status == ONE__TRUNC) {
errRep(FUNC_NAME, "Error extracting scanNumStr!", status);
errAnnul(status);
}
/* Create a temporary base file name from input file name */
one_strlcpy(fileName, inData->file->name,
astMIN(SMF_PATH_MAX + 1, strlen(inData->file->name) - 7), status);
if (*status == ONE__TRUNC) {
errRep(FUNC_NAME, "Error extracting base fileName!", status);
errAnnul(status);
}
scanNum = (int) one_strtod(scanNumStr, status);
if (*status != SAI__OK) {
errRep(FUNC_NAME, "Error extracting scanNum!", status);
errAnnul(status);
}
/* Left shift scanNum to conNum as a prefix to make output scan number unique */
if(scanNum < 100) {
conNum = scanNum * 100;
} else if(scanNum < 1000) {
conNum = scanNum * 10;
}
/*printf("%s: Processing file: %s, having basename: %s and scanNumStr: %s, scanNum: %04d\n",
TASK_NAME, inData->file->name, fileName, scanNumStr, scanNum);*/
/* Data cube dimensions */
nWidth = inData->dims[0];
nHeight = inData->dims[1];
nFrames = inData->dims[2];
nPixels = nWidth * nHeight;
/* Mirror positions in mm */
nTmp = nFrames;
MIRPOS = astCalloc(nFrames, sizeof(*MIRPOS));
fts2_getmirrorpositions(inData, MIRPOS, &nTmp, status); // (mm)
if(*status != SAI__OK) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Unable to get the mirror positions!", status);
goto CLEANUP;
}
nStart = -1;
nStop = -1;
nStartNext = 0;
hrStart = -1;
hrStop = -1;
lrStart = -1;
lrStop = -1;
outDataCount = 0;
done = 0;
do {
/* Find the next range of single scan mirror positions for which to extract corresponding NDF data */
for(n=nStartNext; n<nFrames-1; n++){
if(hrStart < 0 && fabs(MIRPOS[n+1] - MIRPOS[n]) >= EPSILON) {
nStart = n;
hrStart = n;
/*printf("%s: Split nStart=%d\n", TASK_NAME, nStart);*/
}
if(hrStart >= 0 && hrStop < 0 && (fabs(MIRPOS[n+1] - MIRPOS[n]) < EPSILON || n+1 == nFrames-1)) {
hrStop = n+1;
hrFramesOutPrev = hrFramesOut;
hrFramesOut = abs(hrStop - hrStart) + 1;
outDataCount++;
nStop = hrStop;
nFramesOutPrev = hrFramesOutPrev;
nFramesOut = hrFramesOut;
/*printf("%s: Split: %d of %d frames found at hrStart=%d, hrStop=%d\n",
TASK_NAME, outDataCount, hrFramesOut, hrStart, hrStop);*/
break;
}
}
/* Determine scan direction */
if(MIRPOS[hrStart] < MIRPOS[hrStop]) {
scanDir = 1; /* Positive */
} else {
scanDir = -1; /* Negative */
}
/* Limit to specified mirror position range */
if(LR) {
/* Calculate how many frames correspond to the given +/- mm of LR bandpass */
lrBandPassFrames = lrmmBandPass / dz;
/* Find the centre of the current scan */
lrCentre = floor((abs(hrStop-hrStart)+1)/2);
/* Set low res start and stop values at corresponding frame offsets from centre */
lrStart = lrCentre - lrBandPassFrames;
lrStop = lrCentre + lrBandPassFrames;
lrFramesOutPrev = lrFramesOut;
lrFramesOut = abs(lrStop - lrStart) + 1;
nStart = lrStart;
nStop = lrStop;
nFramesOutPrev = lrFramesOutPrev;
nFramesOut = lrFramesOut;
/*printf("%s: LR Split: %d of %d frames found at lrStart=%d, lrStop=%d\n",
TASK_NAME, outDataCount, lrFramesOut, lrStart, lrStop);*/
}
/* Check for end of data condition */
if(hrStop < hrStart || hrStop >= nFrames-1) {
hrStop = nFrames-1;
done = 1;
}
/* Output scan if there is a start and stop position found,
and for the last scan if it's the only one
and if it's not too short (compared to the previous one) */
/*printf("%s: nStart=%d, nStop=%d, nFramesOutPrev=%d, nFramesOut=%d\n", TASK_NAME, nStart, nStop, nFramesOutPrev, nFramesOut);*/
if(nStart >=0 && nStop > 0 &&
(nFramesOutPrev == 0 ||
(nFramesOutPrev > 0 && nFramesOut > 0 && (double)hrFramesOut/(double)hrFramesOutPrev >= 0.5))) {
/* Copy single scan NDF data from input to output */
outData = smf_deepcopy_smfData(inData, 0, SMF__NOCREATE_DATA | SMF__NOCREATE_FTS, 0, 0, status);
outData->dtype = SMF__DOUBLE;
outData->ndims = 3;
outData->dims[0] = nWidth;
outData->dims[1] = nHeight;
outData->dims[2] = nFramesOut;
outData_pntr = (double*) astMalloc((nPixels * nFramesOut) * sizeof(*outData_pntr));
outData->pntr[0] = outData_pntr;
outData->hdr->nframes = nFramesOut;
for(i=0; i<nWidth; i++) {
for(j=0; j<nHeight; j++) {
bolIndex = i + j * nWidth;
for(k=nStart; k<=nStop; k++) {
indexIn = bolIndex + k * nPixels;
indexOut = bolIndex + (k-nStart) * nPixels;
*((double*)(outData->pntr[0]) + indexOut) = *((double*)(inData->pntr[0]) + indexIn);
}
}
}
/* Update the FITS headers */
outData->fts = smf_create_smfFts(status);
/* Update FITS component */
smf_fits_updateD(outData->hdr, "FNYQUIST", fNyquist, "Nyquist frequency (cm^-1)", status);
smf_fits_updateI(outData->hdr, "MIRSTART", 1, "Frame index in which the mirror starts moving", status);
smf_fits_updateI(outData->hdr, "MIRSTOP", nFramesOut, "Frame index in which the mirror stops moving", status);
smf_fits_updateI(outData->hdr, "SCANDIR", scanDir, "Scan direction", status);
smf_fits_updateD(outData->hdr, "OPDMIN", 0.0, "Minimum OPD", status);
smf_fits_updateD(outData->hdr, "OPDSTEP", 0.0, "OPD step size", status);
/* Update the JCMTSTATE header */
/* Reallocate outData header array memory to reduced size */
allState = (JCMTState*) astRealloc(outData->hdr->allState, nFramesOut * sizeof(*(outData->hdr->allState)));
if(*status == SAI__OK && allState) {
outData->hdr->allState = allState;
} else {
errRepf(TASK_NAME, "Error reallocating allState JCMTState header", status);
goto CLEANUP;
}
for(k=nStart; k<=nStop; k++) {
/* Copy over JCMTstate */
/*printf("%s: memcpy allState: %d to: %p from: %p size: %d\n",TASK_NAME, k,
(void *) &(outData->hdr->allState[k-nStart]), (void *) &(inData->hdr->allState[k]), sizeof(*(outData->hdr->allState)) );*/
memcpy( (void *) &(outData->hdr->allState[k-nStart]), (void *) &(inData->hdr->allState[k]), sizeof(*(outData->hdr->allState)) );
/*printf("%s: Scan: %d index: %d rts_num: %d\n", TASK_NAME, outDataCount, k-nStart, outData->hdr->allState[k-nStart].rts_num);*/
/*printf("%s: Scan: %d index: %d fts_pos: %f\n", TASK_NAME, outDataCount, k-nStart, outData->hdr->allState[k-nStart].fts_pos);*/
}
/* Write output */
/* Append unique suffix to fileName */
/* This must be modified by the concatenation file scan number to improve uniqueness */
n = one_snprintf(outData->file->name, SMF_PATH_MAX, "%s%04d_scn.sdf", status, fileName, conNum+outDataCount);
/*printf("%s: Writing outData->file->name: %s\n", TASK_NAME, outData->file->name);*/
if(n < 0 || n >= SMF_PATH_MAX) {
errRepf(TASK_NAME, "Error creating outData->file->name", status);
goto CLEANUP;
}
/* Update the list of output _scn file names */
grpPut1(gOut, outData->file->name, 0, status);
if(*status != SAI__OK) {
errRepf(TASK_NAME, "Error saving outData file name", status);
goto CLEANUP;
}
smf_write_smfData(outData, NULL, outData->file->name, gOut, fIndex, 0, MSG__VERB, 0, status);
if(*status != SAI__OK) {
errRepf(TASK_NAME, "Error writing outData file", status);
goto CLEANUP;
}
smf_close_file(&outData, status);
if(*status != SAI__OK) {
errRepf(TASK_NAME, "Error closing outData file", status);
goto CLEANUP;
}
if(*status != SAI__OK) {
errRepf(TASK_NAME, "Error closing outData file", status);
goto CLEANUP;
}
}/* else {
if(!(nStart >=0 && nStop)) printf("%s: Output scan condition failed: nStart(%d) >= nStop(%d) is FALSE\n",TASK_NAME, nStart, nStop);
if(!(nFramesOutPrev == 0 ||
(nFramesOutPrev > 0 && nFramesOut > 0 && (double)nFramesOut/(double)nFramesOutPrev >= 0.5))) printf("%s: Output scan condition failed: nFramesOutPrev(%d) == 0 || (nFramesOutPrev(%d) > 0 && nFramesOut(%d) > 0 && nFramesOut/nFramesOutPrev (%f) >= 0.5) is FALSE\n", TASK_NAME, nFramesOutPrev, nFramesOutPrev, nFramesOut, (double)nFramesOut/(double)nFramesOutPrev);
}*/
/* Prepare for next iteration */
nStartNext = hrStop + 1;
hrStart = -1;
hrStop = -1;
} while (!done);
/* Deallocate memory used by arrays */
if(MIRPOS) { MIRPOS = astFree(MIRPOS); }
/* Close the file */
smf_close_file(&inData, status);
}
CLEANUP:
/* Deallocate memory used by arrays */
if(inData) { smf_close_file(&inData, status); }
if(outData) { smf_close_file(&outData, status); }
/* END NDF */
ndfEnd(status);
/* Write out the list of output NDF names, annulling the error if a null
parameter value is supplied. */
if( *status == SAI__OK && gOut ) {
grpList( "OUTFILES", 0, 0, NULL, gOut, status );
if( *status == PAR__NULL ) {
errRep(FUNC_NAME, "Error writing OUTFILES!", status);
errAnnul( status );
}
}
/* Delete groups */
if(gIn) { grpDelet(&gIn, status); }
if(gOut) { grpDelet(&gOut, status); }
}
void smurf_unmakemap( int *status ) {
/* Local Variables */
AstFrameSet *wcsin = NULL; /* WCS Frameset for input cube */
AstMapping *skymap; /* GRID->SkyFrame Mapping from input WCS */
AstSkyFrame *abskyfrm; /* Input SkyFrame (always absolute) */
AstSkyFrame *skyfrm = NULL;/* SkyFrame from the input WCS Frameset */
Grp *igrp1 = NULL; /* Group of input sky files */
Grp *igrp2 = NULL; /* Group of input template files */
Grp *igrpc = NULL; /* Group of input COM files */
Grp *igrpg = NULL; /* Group of input GAI files */
Grp *igrpq = NULL; /* Group of input Q sky files */
Grp *igrpu = NULL; /* Group of input U sky files */
Grp *ogrp = NULL; /* Group containing output file */
HDSLoc *cloc = NULL; /* HDS locator for component ipdata structure */
HDSLoc *iploc = NULL; /* HDS locator for top level ipdata structure */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
char ipdata[ 200 ]; /* Text buffer for IPDATA value */
char pabuf[ 10 ]; /* Text buffer for parameter value */
char subarray[ 5 ]; /* Name of SCUBA-2 subarray (s8a,s8b,etc) */
dim_t iel; /* Index of next element */
dim_t ndata; /* Number of elements in array */
dim_t ntslice; /* Number of time slices in array */
double *ang_data = NULL; /* Pointer to the FP orientation angles */
double *angc_data = NULL; /* Pointer to the instrumental ANGC data */
double *c0_data = NULL; /* Pointer to the instrumental C0 data */
double *gai_data = NULL; /* Pointer to the input GAI map */
double *in_data = NULL; /* Pointer to the input I sky map */
double *inc_data = NULL; /* Pointer to the input COM data */
double *inq_data = NULL; /* Pointer to the input Q sky map */
double *inu_data = NULL; /* Pointer to the input U sky map */
double *outq_data = NULL; /* Pointer to the Q time series data */
double *outu_data = NULL; /* Pointer to the U time series data */
double *p0_data = NULL; /* Pointer to the instrumental P0 data */
double *p1_data = NULL; /* Pointer to the instrumental P1 data */
double *pd; /* Pointer to next element */
double *pq = NULL; /* Pointer to next Q time series value */
double *pu = NULL; /* Pointer to next U time series value */
double *qinst_data = NULL; /* Pointer to the instrumental Q data */
double *uinst_data = NULL; /* Pointer to the instrumental U data */
double amp16; /* Amplitude of 16 Hz signal */
double amp2; /* Amplitude of 2 Hz signal */
double amp4; /* Amplitude of 4 Hz signal */
double angrot; /* Angle from focal plane X axis to fixed analyser */
double paoff; /* WPLATE value corresponding to POL_ANG=0.0 */
double params[ 4 ]; /* astResample parameters */
double phase16; /* Phase of 16 Hz signal */
double phase2; /* Phase of 2 Hz signal */
double phase4; /* Phase of 4 Hz signal */
double sigma; /* Standard deviation of noise to add to output */
int alignsys; /* Align data in the map's system? */
int cdims[ 3 ]; /* Common-mode NDF dimensions */
int dims[ NDF__MXDIM ]; /* NDF dimensions */
int flag; /* Was the group expression flagged? */
int gdims[ 3 ]; /* GAI model NDF dimensions */
int harmonic; /* The requested harmonic */
int ifile; /* Input file index */
int indf; /* Input sky map NDF identifier */
int indfangc; /* IP ANGC values NDF identifier */
int indfc0; /* IP C0 values NDF identifier */
int indfc; /* Input COM NDF identifier */
int indfcs; /* NDF identifier for matching section of COM */
int indfg; /* Input GAI NDF identifier */
int indfin; /* Input template cube NDF identifier */
int indfiq; /* Input instrumental Q NDF */
int indfiu; /* Input instrumental U NDF */
int indfout; /* Output cube NDF identifier */
int indfp0; /* IP P0 values NDF identifier */
int indfp1; /* IP P1 values NDF identifier */
int indfq; /* Input Q map NDF identifier */
int indfu; /* Input U map NDF identifier */
int interp = 0; /* Pixel interpolation method */
int lbndc[ 3 ]; /* Array of lower bounds of COM NDF */
int moving; /* Is the telescope base position changing? */
int ndim; /* Number of pixel axes in NDF */
int ndimc; /* Number of pixel axes in common-mode NDF */
int ndimg; /* Number of pixel axes in GAI NDF */
int nel; /* Number of elements in array */
int nelc; /* Number of elements in COM array */
int nelg; /* Number of elements in GAI array */
int nelqu; /* Number of elements in Q or U array */
int ngood; /* No. of good values in putput cube */
int nparam = 0; /* No. of parameters required for interpolation scheme */
int pasign; /* Indicates sense of POL_ANG value */
int sdim[ 2 ]; /* Array of significant pixel axes */
int slbnd[ 2 ]; /* Array of lower bounds of input map */
int subnd[ 2 ]; /* Array of upper bounds of input map */
int ubndc[ 3 ]; /* Array of upper bounds of COM NDF */
size_t ncom; /* Number of com files */
size_t ngai; /* Number of gai files */
size_t nskymap; /* Number of supplied sky cubes */
size_t outsize; /* Number of files in output group */
size_t size; /* Number of files in input group */
smfData *odata = NULL; /* Pointer to output data struct */
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Begin an AST context */
astBegin;
/* Begin an NDF context. */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get an identifier for the input NDF. We use NDG (via kpg1Rgndf)
instead of calling ndfAssoc directly since NDF/HDS has problems with
file names containing spaces, which NDG does not have. */
kpg1Rgndf( "IN", 1, 1, "", &igrp1, &nskymap, status );
ndgNdfas( igrp1, 1, "READ", &indf, status );
/* Map the data array in the input sky map. */
ndfMap( indf, "DATA", "_DOUBLE", "READ", (void **) &in_data, &nel,
status );
/* Get the WCS FrameSet from the sky map, together with its pixel index
bounds. */
kpg1Asget( indf, 2, 0, 1, 1, sdim, slbnd, subnd, &wcsin, status );
/* Check the current Frame is a SKY frame. */
skyfrm = astGetFrame( wcsin, AST__CURRENT );
if( !astIsASkyFrame( skyfrm ) && *status == SAI__OK ) {
ndfMsg( "N", indf );
*status = SAI__ERROR;
errRep( " ", " Current Frame in ^N is not a SKY Frame.", status );
}
/* Get a copy of the current frame that represents absolute coords rather
than offsets. We assume the target is moving if the map represents
offsets. */
moving = ( *status == SAI__OK &&
!strcmp( astGetC( skyfrm, "SkyRefIs" ), "Origin" ) ) ? 1 : 0;
abskyfrm = astCopy( skyfrm );
astClear( abskyfrm, "SkyRefIs" );
/* If the ALIGNSYS parameter is TRUE then we align the raw data with the
map in the current system of the map, rather than the default ICRS. */
parGet0l( "ALIGNSYS", &alignsys, status );
if( alignsys ) astSetC( abskyfrm, "AlignSystem", astGetC( abskyfrm,
"System" ) );
/* Get the Mapping from the Sky Frame to grid axis in the iput map. */
skymap = astGetMapping( wcsin, AST__CURRENT, AST__BASE );
/* Get the pixel interpolation scheme to use. */
parChoic( "INTERP", "NEAREST", "NEAREST,LINEAR,SINC,"
"SINCSINC,SINCCOS,SINCGAUSS,SOMB,SOMBCOS",
1, pabuf, 10, status );
if( !strcmp( pabuf, "NEAREST" ) ) {
interp = AST__NEAREST;
nparam = 0;
} else if( !strcmp( pabuf, "LINEAR" ) ) {
interp = AST__LINEAR;
nparam = 0;
} else if( !strcmp( pabuf, "SINC" ) ) {
interp = AST__SINC;
nparam = 1;
} else if( !strcmp( pabuf, "SINCSINC" ) ) {
interp = AST__SINCSINC;
nparam = 2;
} else if( !strcmp( pabuf, "SINCCOS" ) ) {
interp = AST__SINCCOS;
nparam = 2;
} else if( !strcmp( pabuf, "SINCGAUSS" ) ) {
interp = AST__SINCGAUSS;
nparam = 2;
} else if( !strcmp( pabuf, "SOMB" ) ) {
interp = AST__SOMB;
nparam = 1;
} else if( !strcmp( pabuf, "SOMBCOS" ) ) {
interp = AST__SOMBCOS;
nparam = 2;
} else if( *status == SAI__OK ) {
nparam = 0;
*status = SAI__ERROR;
msgSetc( "V", pabuf );
errRep( "", "Support not available for INTERP = ^V (programming "
"error)", status );
}
/* Get an additional parameter vector if required. */
if( nparam > 0 ) parExacd( "PARAMS", nparam, params, status );
/* Get a group of reference time series files to use as templates for
the output time series files.*/
ndgAssoc( "REF", 1, &igrp2, &size, &flag, status );
/* Get output file(s) */
kpg1Wgndf( "OUT", igrp2, size, size, "More output files required...",
&ogrp, &outsize, status );
/* Get he noise level to add to the output data. */
parGet0d( "SIGMA", &sigma, status );
/* Get any Q and U input maps. */
if( *status == SAI__OK ) {
kpg1Rgndf( "QIN", 1, 1, "", &igrpq, &nskymap, status );
ndgNdfas( igrpq, 1, "READ", &indfq, status );
ndfMap( indfq, "DATA", "_DOUBLE", "READ", (void **) &inq_data, &nelqu,
status );
if( nelqu != nel && *status == SAI__OK ) {
ndfMsg( "Q", indfq );
*status = SAI__ERROR;
errRep( "", "Q image '^Q' is not the same size as the I image.",
status );
}
kpg1Rgndf( "UIN", 1, 1, "", &igrpu, &nskymap, status );
ndgNdfas( igrpu, 1, "READ", &indfu, status );
ndfMap( indfu, "DATA", "_DOUBLE", "READ", (void **) &inu_data, &nelqu,
status );
if( nelqu != nel && *status == SAI__OK ) {
ndfMsg( "U", indfu );
*status = SAI__ERROR;
errRep( "", "U image '^U' is not the same size as the I image.",
status );
}
if( *status == PAR__NULL ) {
ndfAnnul( &indfq, status );
ndfAnnul( &indfu, status );
inq_data = NULL;
inu_data = NULL;
errAnnul( status );
} else {
parGet0d( "ANGROT", &angrot, status );
parGet0d( "PAOFF", &paoff, status );
parGet0l( "PASIGN", &pasign, status );
}
}
/* Get any common-mode files. */
if( *status == SAI__OK ) {
kpg1Rgndf( "COM", size, size, "", &igrpc, &ncom, status );
if( *status == PAR__NULL ) {
errAnnul( status );
ncom = 0;
}
}
/* Get any GAI files. */
if( *status == SAI__OK ) {
kpg1Rgndf( "GAI", size, size, "", &igrpg, &ngai, status );
if( *status == PAR__NULL ) {
errAnnul( status );
ngai = 0;
}
}
/* Get any instrumental polarisation files. */
if( *status == SAI__OK ) {
/* First see if the user wants to use the "INSTQ/INSTU" scheme for
specifying instrumental polarisation. */
ndfAssoc( "INSTQ", "Read", &indfiq, status );
ndfAssoc( "INSTU", "Read", &indfiu, status );
if( *status == PAR__NULL ) {
ndfAnnul( &indfiq, status );
ndfAnnul( &indfiu, status );
errAnnul( status );
} else {
msgOut( " ", "Using user-defined IP model", status );
ndfDim( indfiq, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfiq );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfiq, "DATA", "_DOUBLE", "READ", (void **) &qinst_data,
&nel, status );
}
ndfDim( indfiu, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfiu );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfiu, "DATA", "_DOUBLE", "READ", (void **) &uinst_data,
&nel, status );
}
}
/* If not, see if the user wants to use the Johnstone/Kennedy instrumental
polarisation model. The IPDATA parameter gives the path to an HDS
container file contining NDFs holding the required IP data for all
subarrays. */
if( !qinst_data ) {
parGet0c( "IPDATA", ipdata, sizeof(ipdata), status );
if( *status == PAR__NULL ) {
errAnnul( status );
} else {
msgOutf( " ", "Using Johnstone/Kennedy IP model in %s",
status, ipdata );
hdsOpen( ipdata, "READ", &iploc, status );
}
}
}
/* Loop round all the template time series files. */
for( ifile = 1; ifile <= (int) size && *status == SAI__OK; ifile++ ) {
/* Start a new NDF context. */
ndfBegin();
/* Create the output NDF by propagating everything from the input, except
for quality and variance. */
ndgNdfas( igrp2, ifile, "READ", &indfin, status );
ndfMsg( "FILE", indfin );
msgSeti( "THISFILE", ifile );
msgSeti( "NUMFILES", size );
msgOutif( MSG__NORM, " ", "Simulating ^THISFILE/^NUMFILES ^FILE",
status );
ndgNdfpr( indfin, "DATA,HISTORY,LABEL,TITLE,WCS,UNITS,EXTENSION(*)",
ogrp, ifile, &indfout, status );
ndfAnnul( &indfin, status );
ndfAnnul( &indfout, status );
/* We now re-open the output NDF and then modify its data values. */
smf_open_file( wf, ogrp, ifile, "UPDATE", 0, &odata, status );
/* Issue a suitable message and abort if anything went wrong. */
if( *status != SAI__OK ) {
errRep( FUNC_NAME, "Could not open input template file.", status );
break;
} else {
if( odata->file == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "No smfFile associated with smfData.",
status );
break;
} else if( odata->hdr == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "No smfHead associated with smfData.",
status );
break;
}
}
/* Check the reference time series contains double precision values. */
smf_dtype_check_fatal( odata, NULL, SMF__DOUBLE, status );
/* Get the total number of data elements, and the number of time slices. */
smf_get_dims( odata, NULL, NULL, NULL, &ntslice, &ndata, NULL,
NULL, status );
/* Get the subarray name */
smf_fits_getS( odata->hdr, "SUBARRAY", subarray, sizeof(subarray),
status );
/* If we are using the Johnstone/Kennedy IP model, open and map the
relevant parameter NDFs within the IPDATA container file. */
if( iploc ) {
datFind( iploc, subarray, &cloc, status );
ndfFind( cloc, "C0", &indfc0, status );
ndfDim( indfc0, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfc0 );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfc0, "DATA", "_DOUBLE", "READ", (void **) &c0_data,
&nel, status );
}
ndfFind( cloc, "P0", &indfp0, status );
ndfDim( indfp0, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfp0 );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfp0, "DATA", "_DOUBLE", "READ", (void **) &p0_data,
&nel, status );
}
ndfFind( cloc, "P1", &indfp1, status );
ndfDim( indfp1, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfp1 );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfp1, "DATA", "_DOUBLE", "READ", (void **) &p1_data,
&nel, status );
}
ndfFind( cloc, "ANGC", &indfangc, status );
ndfDim( indfangc, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfangc );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfangc, "DATA", "_DOUBLE", "READ", (void **) &angc_data,
&nel, status );
}
}
/* Open any COM file. */
if( ncom ) {
ndgNdfas( igrpc, ifile, "READ", &indfc, status );
ndfDim( indfc, 3, cdims, &ndimc, status );
/* Check its dimensions. */
if( *status == SAI__OK ) {
if( ndimc == 1 ) {
if( cdims[ 0 ] < (int) ntslice ) {
*status = SAI__ERROR;
ndfMsg( "C", indfc );
ndfMsg( "R", indfin );
msgSeti( "N", cdims[ 0 ] );
msgSeti( "M", ntslice );
errRep( " ", "Supplied COM file (^C) has ^N time-slices, but "
"the reference NDF (^R) has ^M time-slices.", status );
} else {
ndfBound( indfc, 3, lbndc, ubndc, &ndimc, status );
ubndc[ 0 ] = lbndc[ 0 ] + ntslice - 1;
ndfSect( indfc, 1, lbndc, ubndc, &indfcs, status );
}
} else if( ndimc == 3 ) {
if( cdims[ 0 ] != 1 || cdims[ 1 ] != 1 ) {
*status = SAI__ERROR;
ndfMsg( "C", indfc );
errRep( " ", "Supplied 3D COM file (^C) has bad "
"dimensions for axis 1 and/or 2 (should "
"both be 1 pixel long).", status );
} else if( cdims[ 2 ] < (int) ntslice ) {
*status = SAI__ERROR;
ndfMsg( "C", indfc );
ndfMsg( "R", indfin );
msgSeti( "N", cdims[ 2 ] );
msgSeti( "M", ntslice );
errRep( " ", "Supplied COM file (^C) has ^N time-slices, but "
"the reference NDF (^R) has ^M time-slices.", status );
} else {
ndfBound( indfc, 3, lbndc, ubndc, &ndimc, status );
ubndc[ 2 ] = lbndc[ 2 ] + ntslice - 1;
ndfSect( indfc, 3, lbndc, ubndc, &indfcs, status );
}
} else {
*status = SAI__ERROR;
ndfMsg( "C", indfc );
msgSeti( "N", ndimc );
errRep( " ", "Supplied COM file (^C) has ^N dimensions - "
"must be 3.", status );
}
}
ndfMap( indfcs, "DATA", "_DOUBLE", "READ", (void **) &inc_data,
&nelc, status );
} else {
indfcs = NDF__NOID;
inc_data = NULL;
}
/* Open any GAI files. */
if( ngai ) {
ndgNdfas( igrpg, ifile, "READ", &indfg, status );
ndfDim( indfg, 3, gdims, &ndimg, status );
/* Check its dimensions, and map it if OK. */
if( *status == SAI__OK ) {
if( ndimg != 2 ) {
*status = SAI__ERROR;
ndfMsg( "C", indfg );
msgSeti( "N", ndimg );
errRep( " ", "Supplied GAI file (^C) has ^N dimensions - "
"must be 2.", status );
} else if( gdims[ 0 ] != 32 || gdims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "C", indfg );
errRep( " ", "Supplied GAI file (^C) has has bad "
"dimensions - should be 32x40.", status );
}
}
ndfMap( indfg, "DATA", "_DOUBLE", "READ", (void **) &gai_data,
&nelg, status );
} else {
indfg = NDF__NOID;
gai_data = NULL;
}
/* Fill the output with bad values. */
if( *status == SAI__OK ) {
pd = odata->pntr[ 0 ];
for( iel = 0; iel < ndata; iel++ ) *(pd++) = VAL__BADD;
}
/* Resample the sky map data into the output time series. */
smf_resampmap( wf, odata, abskyfrm, skymap, moving, slbnd, subnd,
interp, params, sigma, in_data, odata->pntr[ 0 ],
NULL, &ngood, status );
/* Add on any COM data. */
smf_addcom( wf, odata, inc_data, status );
/* Issue a wrning if there is no good data in the output cube. */
if( ngood == 0 ) msgOutif( MSG__NORM, " ", " Output contains no "
"good data values.", status );
/* If Q and U maps have been given, allocate room to hold resampled Q and
U values, and fill them with bad values. */
if( inq_data && inu_data ) {
pq = outq_data = astMalloc( ndata*sizeof( *outq_data ) );
pu = outu_data = astMalloc( ndata*sizeof( *outu_data ) );
if( *status == SAI__OK ) {
for( iel = 0; iel < ndata; iel++ ) {
*(pu++) = VAL__BADD;
*(pq++) = VAL__BADD;
}
}
/* Determine the harmonic to use. */
parGet0i( "HARMONIC", &harmonic, status );
/* If producing the normal 8 Hz harmonic, get the amplitude and phase of a
other signals to add onto the 8 Hz signal. */
if( harmonic == 4 ) {
parGet0d( "AMP2", &2, status );
parGet0d( "PHASE2", &phase2, status );
parGet0d( "AMP4", &4, status );
parGet0d( "PHASE4", &phase4, status );
parGet0d( "AMP16", &16, status );
parGet0d( "PHASE16", &phase16, status );
} else {
amp2 = 0.0;
phase2 = 0.0;
amp4 = 0.0;
phase4 = 0.0;
amp16 = 0.0;
phase16 = 0.0;
}
/* Allocate room for an array to hold the angle from the Y pixel axis
in the sky map to the focal plane Y axis, in radians, at each time
slice. Positive rotation is in the same sense as rotation from
focal plane X to focal plane Y. */
ang_data = astMalloc( ntslice*sizeof( *ang_data ) );
/* Resample them both into 3D time series. These Q/U values arw with
respect to the sky image Y axis. */
smf_resampmap( wf, odata, abskyfrm, skymap, moving, slbnd, subnd,
interp, params, sigma, inq_data, outq_data,
ang_data, &ngood, status );
smf_resampmap( wf, odata, abskyfrm, skymap, moving, slbnd, subnd,
interp, params, sigma, inu_data, outu_data,
NULL, &ngood, status );
/* Combine these time series with the main output time series so that the
main output is analysed intensity. */
smf_uncalc_iqu( wf, odata, odata->pntr[ 0 ], outq_data, outu_data,
ang_data, pasign, AST__DD2R*paoff, AST__DD2R*angrot,
amp2, AST__DD2R*phase2, amp4, AST__DD2R*phase4,
amp16, AST__DD2R*phase16, qinst_data, uinst_data,
c0_data, p0_data, p1_data, angc_data, harmonic,
status );
/* Release work space. */
outq_data = astFree( outq_data );
outu_data = astFree( outu_data );
ang_data = astFree( ang_data );
}
/* Factor in any GAI data. */
smf_addgai( wf, odata, gai_data, status );
/* Close the output time series file. */
smf_close_file( wf, &odata, status );
/* Close the IP data container for the current subarray, if it is open. */
if( cloc ) datAnnul( &cloc, status );
/* End the NDF context. */
ndfEnd( status );
}
/* Close any input data file that is still open due to an early exit from
the above loop. */
if( odata != NULL ) {
smf_close_file( wf, &odata, status );
odata = NULL;
}
/* Free remaining resources. */
if( igrp1 != NULL) grpDelet( &igrp1, status);
if( igrp2 != NULL) grpDelet( &igrp2, status);
if( igrpq != NULL) grpDelet( &igrpq, status);
if( igrpu != NULL) grpDelet( &igrpu, status);
if( igrpc != NULL) grpDelet( &igrpc, status);
if( igrpg != NULL) grpDelet( &igrpg, status);
if( ogrp != NULL) grpDelet( &ogrp, status);
if( iploc ) datAnnul( &iploc, status );
/* End the NDF context. */
ndfEnd( status );
/* End the tile's AST context. */
astEnd;
/* Issue a status indication.*/
if( *status == SAI__OK ) {
msgOutif(MSG__VERB," ",TASK_NAME " succeeded, time series written.", status);
} else {
msgOutif(MSG__VERB," ",TASK_NAME " failed.", status);
}
}
int smf_fix_metadata_scuba2 ( msglev_t msglev, smfData * data, int have_fixed, int *ncards, int * status ) {
AstFitsChan * fits = NULL; /* FITS header (FitsChan) */
struct FitsHeaderStruct fitsvals; /* Quick access Fits header struct */
smfHead *hdr = NULL; /* Data header struct */
AstKeyMap * obsmap = NULL; /* Info from all observations */
AstKeyMap * objmap = NULL; /* All the object names used */
if (*status != SAI__OK) return have_fixed;
/* Validate arguments - need smfFile and smfHead */
smf_validate_smfData( data, 1, 1, status );
if (*status != SAI__OK) return have_fixed;
hdr = data->hdr;
smf_validate_smfHead( hdr, 1, 1, status );
if (*status != SAI__OK) return have_fixed;
fits = hdr->fitshdr;
if (hdr->instrument != INST__SCUBA2) {
if (*status != SAI__OK) {
*status = SAI__ERROR;
errRep("", " Attempting to fix metadata using SCUBA-2 algorithms but this is not SCUBA-2 data",
status );
}
return have_fixed;
}
/* Update units string to something that is FITS standard compliant
- we used "DAC units" for a while but in FITS land this becomes
"decacoulomb * units" */
if ( strncmp( hdr->units, "DAC", 3) == 0 ) {
one_strlcpy( hdr->units, "adu", SMF__CHARLABEL, status );
}
/* Clock jitter and readout efficiencies mean we need to recalculate STEPTIME from the data.
This is possible because we know that we have a continuous sequence in each file (unlike
ACSIS). */
if (hdr->allState) {
/* it will be odd if it is not there */
size_t nframes = hdr->nframes;
size_t istart = 0;
double start_time = (hdr->allState)[istart].rts_end;
while( start_time == VAL__BADD && ++istart < nframes ) {
start_time = (hdr->allState)[istart].rts_end;
}
size_t iend = nframes - 1;
double end_time = (hdr->allState)[iend].rts_end;
while( end_time == VAL__BADD && iend-- > 0 ) {
end_time = (hdr->allState)[iend].rts_end;
}
double steptime = VAL__BADD;
double newstep;
smf_getfitsd( hdr, "STEPTIME", &steptime, status );
newstep = steptime;
/* it is possible for a file to contain only one step since
the DA just dumps every N-steps. We can not recalculate the
step time in that case. */
nframes = iend - istart + 1;
if (nframes > 1) {
/* duration of file in days */
newstep = end_time - start_time;
/* convert to seconds */
newstep *= SPD;
/* Convert to step time */
newstep /= (nframes - 1);
} else if( nframes > 0 ) {
/* work it out from RTS_END and TCS_TAI */
JCMTState * onlystate = &((hdr->allState)[istart]);
if ( onlystate->tcs_tai != VAL__BADD &&
onlystate->tcs_tai != onlystate->rts_end) {
/* TCS_TAI is in the middle of the step */
newstep = 2.0 * ( onlystate->rts_end - onlystate->tcs_tai ) * SPD;
}
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
if( data->file ) {
smf_smfFile_msg( data->file, "N", 1, "<unknown>" );
errRep("", "No valid RTS_END values found in NDF '^N'.", status );
} else {
errRep("", "No valid RTS_END values found.", status );
}
}
if (steptime != newstep) {
msgOutiff( msglev, "", INDENT "Recalculated step time as %g sec from JCMTSTATE (was %g sec)",
status, newstep, steptime);
smf_fits_updateD( hdr, "STEPTIME", newstep, NULL, status );
have_fixed |= SMF__FIXED_FITSHDR;
}
}
/* Read some FITS headers, intialising the struct first */
fitsvals.utdate = VAL__BADI;
*(fitsvals.instrume) = '\0';
smf_getfitsi( hdr, "UTDATE", &(fitsvals.utdate), status );
smf_getfitss( hdr, "INSTRUME", fitsvals.instrume, sizeof(fitsvals.instrume), status );
/* Print out summary of this observation - this may get repetitive if multiple files come
from the same observation in one invocation but it seems better to describe each fix up
separately and in context. */
obsmap = astKeyMap( " " );
objmap = astKeyMap( " " );
smf_obsmap_fill( data, obsmap, objmap, status );
smf_obsmap_report( msglev, obsmap, objmap, status );
obsmap = astAnnul( obsmap );
objmap = astAnnul( objmap );
/* First we need to look for a BACKEND header which we do not write to the raw
data files but CADC would like to see equal to INSTRUME */
if (!astTestFits( fits, "BACKEND", NULL ) ) {
have_fixed |= SMF__FIXED_FITSHDR;
smf_fits_updateS( hdr, "BACKEND", fitsvals.instrume, "Name of the backend", status );
msgOutif( msglev, "", INDENT "Setting backend for SCUBA-2 observation.", status);
}
/* BASETEMP was reading MUXTEMP for pre-20091101 data */
if ( fitsvals.utdate < 20091101 ) {
double muxtemp = 0.0;
have_fixed |= SMF__FIXED_FITSHDR;
smf_getfitsd( hdr, "BASETEMP", &muxtemp, status );
smf_fits_updateU( hdr, "BASETEMP", "[K] Base temperature", status );
smf_fits_updateD( hdr, "MUXTEMP", muxtemp, "[K] Mux temperature", status );
msgOutif( msglev, "", INDENT "Mux temperature is being read from BASETEMP header.", status );
}
/* Sometime before 20091119 the SHUTTER keyword was written as a string
OPEN or CLOSED. Rewrite those as numbers */
if (fitsvals.utdate < 20091119) {
double shutval = 0.0;
/* Try to read as a double. */
smf_fits_getD( hdr, "SHUTTER", &shutval, status );
/* Old data was a string. Convert to a double */
if (*status == AST__FTCNV) {
char shutter[100];
errAnnul( status );
smf_fits_getS( hdr, "SHUTTER", shutter, sizeof(shutter), status);
if (strcmp(shutter, "CLOSED") == 0) {
shutval = 0.0;
} else {
shutval = 1.0;
}
/* update the value */
have_fixed |= SMF__FIXED_FITSHDR;
smf_fits_updateD( hdr, "SHUTTER", shutval, "shutter position 0-Closed 1-Open", status );
msgOutif( msglev, "", INDENT "Forcing SHUTTER header to be numeric", status );
}
}
/* Engineering data with just SCUBA2 and no RTS left the RTS_NUM field
filled with zeroes. Just assume that a zero in RTS_NUM is always
indicative of a private sequence. */
if (fitsvals.utdate < 20110401) {
size_t nframes = hdr->nframes;
JCMTState * curstate = &((hdr->allState)[0]);
JCMTState * endstate = &((hdr->allState)[nframes-1]);
if (curstate->rts_num == 0 && endstate->rts_num == 0) {
/* have to set the values from the SEQSTART and SEQEND headers
since those were set correctly (although any value would
do of course apart from the sanity check in smf_find_science. */
size_t i;
int seqnum = 1;
smf_fits_getI( hdr, "SEQSTART", &seqnum, status );
for ( i=0; i<nframes; i++) {
curstate = &((hdr->allState)[i]);
curstate->rts_num = seqnum;
seqnum++;
}
have_fixed |= SMF__FIXED_JCMTSTATE;
msgOutif( msglev, "", INDENT "Private RTS sequence. Fixing RTS_NUM.", status );
}
}
/* work out if this is a fast flat observation taken before May 2010 */
if (fitsvals.utdate > 20100218 && fitsvals.utdate < 20100501) {
char buff[100];
/* need to know whether this is a FASTFLAT */
smf_getfitss( hdr, "SEQ_TYPE", buff, sizeof(buff), status );
if (strcmp( buff, "FASTFLAT" ) == 0 ) {
/* Fast flats had incorrect SHUTTER settings for one night */
if (fitsvals.utdate == 20100223) {
have_fixed |= SMF__FIXED_FITSHDR;
smf_fits_updateD( hdr, "SHUTTER", 1.0, "shutter position 0-Closed 1-Open", status );
msgOutif( msglev, "", INDENT "Shutter was open for fast flatfield ramp. Correcting.", status );
}
/* Need to fix up SC2_HEAT ramps */
/* the problem is that the data were assumed to be taken with 3 measurements
in each heater setting. What actually happened was that the first 5 were
done at the reference setting and then the data were grouped in threes
finishing with a single value at the reference setting again.
For example the heater values and the actual values look something like
Stored 1 1 1 2 2 2 3 3 3 4 4 4 5 5 5
Actual 0 0 0 0 0 2 2 2 3 3 3 4 4 4 5
So we can correct for this by starting at the end and copying in the value
two slots further down until we get to position #4. Then replacing that with
the PIXHEAT number.
*/
{
size_t i;
int pixheat = 0;
size_t nframes = hdr->nframes;
smf_getfitsi( hdr, "PIXHEAT", &pixheat, status );
/* shift everything up by 2 */
for (i=nframes-1; i > 4; i--) {
JCMTState * curstate = &((hdr->allState)[i]);
JCMTState * prevstate = &((hdr->allState)[i-2]);
curstate->sc2_heat = prevstate->sc2_heat;
}
/* fill in the first 5 slots with the same value */
for (i=0; i<5; i++) {
JCMTState * curstate = &((hdr->allState)[i]);
curstate->sc2_heat = pixheat;
}
have_fixed |= SMF__FIXED_JCMTSTATE;
}
}
}
/* We always recalculate the WVM start and end tau values so that the header
reflects something approximating the value that was actually used in the
extinction correction.
Note that smf_calc_smoothedwvm can do a better job because it has multiple
subarrays to get all the values from. We just have to try with what we
have from a single subarray. We do step into the time series until we
find something good.
The header values should mostly agree with the recalculated values if the
WVM code at the time matches the code in SMURF for that date. This has not
been true in cases where we have retrospectively realised that there has been
a calibration error in the WVM. So that we do not have to keep track of those
times explicitly we currently recalculate every time. If this recalculation
becomes a problem (smf_calc_wvm has a cache now to minimize this) it should
be possible to disable this recalculation if the file is less than, say,
30 minutes old to indicate we are running in near realtime.
As a special case we do not recalculate the headers for FOCUS observations
as the WVM reading is somewhat irrelevant and simply slows things down.
*/
if( *status == SAI__OK ){
/* Have not parsed header yet to extract type so do it explicitly here */
char obstype[100];
smf_getfitss( hdr, "OBS_TYPE", obstype, sizeof(obstype), status );
if (strcasecmp( obstype, "focus") != 0) {
size_t i;
size_t nframes = hdr->nframes;
double starttau = VAL__BADD;
double starttime = VAL__BADD;
double endtau = VAL__BADD;
double endtime = VAL__BADD;
/* Create a TimeFrame that can be used to format MJD values into ISO
date-time strings, including a "T" separator between time and date. */
AstTimeFrame *tf = astTimeFrame( "Format=iso.0T" );
for (i=0; i < nframes && *status == SAI__OK; i++) {
smf__calc_wvm_index( hdr, "AMSTART", i, &starttau, &starttime, status );
if (starttau != VAL__BADD) break;
if (*status == SAI__ERROR) errAnnul( status );
}
/* if we did not get a start tau we are not going to get an end tau */
if (starttau != VAL__BADD) {
for (i=0; i < nframes && *status == SAI__OK; i++) {
smf__calc_wvm_index( hdr, "AMEND", nframes - 1 - i, &endtau, &endtime, status );
if (endtau != VAL__BADD) break;
if (*status == SAI__ERROR) errAnnul( status );
}
}
/* If we could not find any WVM readings then we have a bit of a problem.
Do we clear the FITS headers or do we leave them untouched? Leave them
alone for now. */
if (starttau != VAL__BADD && starttime != VAL__BADD) {
smf_fits_updateD( hdr, "WVMTAUST", starttau, "186GHz Tau from JCMT WVM at start", status );
/* Convert starttime MJD to ISO format and update the value in the
FITS header. */
smf_fits_updateS( hdr, "WVMDATST", astFormat( tf, 1, starttime ),
"Time of WVMTAUST", status );
have_fixed |= SMF__FIXED_FITSHDR;
}
if (endtau != VAL__BADD && endtime != VAL__BADD) {
smf_fits_updateD( hdr, "WVMTAUEN", endtau, "186GHz Tau from JCMT WVM at end", status );
/* Convert endtime MJD to ISO format and update the value in the
FITS header. */
smf_fits_updateS( hdr, "WVMDATEN", astFormat( tf, 1, endtime ),
"Time of WVMTAUEN", status );
have_fixed |= SMF__FIXED_FITSHDR;
}
/* Free the TimeFrame. */
tf = astAnnul( tf );
}
}
/* SEQ_TYPE header turned up in 20091125. Before that date the SEQ_TYPE only
had two values. If the shutter was open then SEQ_TYPE is just OBS_TYPE. In the
dark only a FLATFIELD sometimes finished with a noise but in that case CALCFLAT
doesn't care so we just call it a flatfield sequence anyhow. We could look at
the OBSEND flag but I'm not sure it makes a difference. */
if ( fitsvals.utdate < 20091125 ) {
char obstype[100];
char seqtype[100];
double shutval = 0.0;
/* need to know what type of observation this is */
smf_getfitss( hdr, "OBS_TYPE", obstype, sizeof(obstype), status );
/* and the shutter status */
smf_fits_getD( hdr, "SHUTTER", &shutval, status );
if (shutval == 0.0 && strcasecmp( obstype, "flatfield" ) != 0 ) {
/* flatfield was the only non-noise observation in the dark */
one_strlcpy( seqtype, "NOISE", sizeof(seqtype), status );
msgOutif( msglev, "", INDENT "Setting sequence type to NOISE", status );
} else {
/* Shutter was open so SEQ_TYPE is just OBS_TYPE */
one_strlcpy( seqtype, obstype, sizeof(seqtype), status );
msgOutif( msglev, "", INDENT "Setting sequence type to obs type", status);
}
smf_fits_updateS( hdr, "SEQ_TYPE", seqtype, "Type of sequence", status );
have_fixed |= SMF__FIXED_FITSHDR;
}
/* The telescope goes crazy at the end of observation 56 on 20110530. Null
the telescope data for subscans 30, 31 and 32 */
if (fitsvals.utdate == 20110530) {
char obsid[81];
smf_getobsidss( hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status);
if (strcmp(obsid, "scuba2_00056_20110530T135530") == 0 ) {
int subscan;
smf_getfitsi( hdr, "NSUBSCAN", &subscan, status );
if (subscan == 30 || subscan == 31 || subscan == 32) {
size_t nframes = hdr->nframes;
JCMTState * curstate;
size_t i;
for ( i=0; i<nframes; i++ ) {
curstate = &((hdr->allState)[i]);
curstate->jos_drcontrol |= DRCNTRL__PTCS_BIT;
}
msgOutif( msglev, "", INDENT "Blanking telescope data due to extreme excursion", status );
have_fixed |= SMF__FIXED_JCMTSTATE;
}
}
}
/* The second half of observation 14 on 20111215 (scuba2_00014_20111215T061536)
has a elevation pointing shift */
if (fitsvals.utdate == 20111215) {
char obsid[81];
const char fitskey[] = "FIXPCORR";
smf_getobsidss( hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status);
if (strcmp(obsid, "scuba2_00014_20111215T061536") == 0 ) {
int seqcount;
smf_getfitsi( hdr, "SEQCOUNT", &seqcount, status );
if (seqcount == 5) {
int have_fixed_pntg = 0;
smf_fits_getL( hdr, fitskey, &have_fixed_pntg, status );
if (*status == SMF__NOKWRD) {
have_fixed = 0;
errAnnul( status );
}
if (!have_fixed_pntg) {
size_t nframes = hdr->nframes;
size_t i;
const double dlon = 0.0;
const double dlat = -16.83; /* From making maps of each half */
/* Correct the pointing */
msgOutif( msglev, "", INDENT "Applying pointing anomaly correction", status );
for (i=0;i<nframes;i++) {
JCMTState * curstate = &((hdr->allState)[i]);
/* This is an AZEL correction */
smf_add_smu_pcorr( curstate, 1, dlon, dlat, status );
}
smf_fits_updateL(hdr, fitskey, 1, "Applied internal pointing correction", status);
have_fixed |= SMF__FIXED_JCMTSTATE;
}
}
}
}
/* For POL-2 data prior to 18-JAN-2013, the POL_CRD header was always
"FPLANE" in reality, even if the POL_CRD value in JCMTSTATE said
something else. */
if ( fitsvals.utdate < 20130118 ) {
char polcrd[80] = "<unset>";
smf_getfitss( hdr, "POL_CRD", polcrd, sizeof(polcrd), status );
if (*status == SMF__NOKWRD) {
errAnnul( status );
} else if( !strcmp( polcrd, "TRACKING" ) || !strcmp( polcrd, "AZEL" ) ) {
msgOutiff( msglev, "", INDENT "Changing POL_CRD from %s to FPLANE", status, polcrd);
smf_fits_updateS( hdr, "POL_CRD", "FPLANE",
"Coordinate system of polarimeter", status );
have_fixed |= SMF__FIXED_FITSHDR;
}
}
return have_fixed;
}
void smf_grp_related( const Grp *igrp, const size_t grpsize,
const int grouping, const int checksubinst,
double maxlen_s, double *srate_maxlen,
AstKeyMap *keymap, dim_t *maxconcatlen,
dim_t *maxfilelen, smfGroup **group,
Grp **basegrp, dim_t *pad, int *status ) {
/* Local variables */
size_t *chunk=NULL; /* Array of flags for continuous chunks */
dim_t * chunklen = NULL; /* Length of continuous chunk */
size_t currentindex = 0; /* Counter */
char cwave[10]; /* String containing wavelength */
smfData *data = NULL; /* Current smfData */
double downsampscale=0; /* Angular scale downsampling size */
double downsampfreq=0; /* Target downsampling frequency */
AstKeyMap * grouped = NULL; /* Primary AstKeyMap for grouping */
size_t i; /* Loop counter for index into Grp */
int isFFT=0; /* Set if data are 4d FFT */
size_t j; /* Loop counter */
int *keepchunk=NULL; /* Flag for chunks that will be kept */
dim_t maxconcat=0; /* Longest continuous chunk length */
dim_t maxflen=0; /* Max file length in time steps */
dim_t maxlen=0; /* Maximum concat length in samples */
int maxlen_scaled=0; /* Set once maxlen has been scaled, if needed */
dim_t maxpad=0; /* Maximum padding neeed for any input file */
size_t maxrelated = 0; /* Keep track of max number of related items */
size_t *new_chunk=NULL; /* keeper chunks associated with subgroups */
dim_t *new_tlen=NULL; /* tlens for new_subgroup */
size_t ngroups = 0; /* Counter for subgroups to be stored */
size_t nkeep = 0; /* Number of chunks to keep */
dim_t * piecelen = NULL; /* Length of single file */
smf_subinst_t refsubinst; /* Subinst of first file */
size_t **subgroups = NULL; /* Array containing index arrays to parent Grp */
smf_subinst_t subinst; /* Subinst of current file */
if ( *status != SAI__OK ) return;
if( maxlen_s < 0 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": maxlen_s cannot be < 0!", status );
return;
}
/* Get downsampling parameters */
if( keymap ) {
smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, &downsampscale, &downsampfreq,
NULL, NULL, NULL, NULL, status );
if( downsampscale && downsampfreq ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": both downsampscale and downsampfreq are set",
status );
return;
}
}
/* Initialize refcwave */
refsubinst = SMF__SUBINST_NONE;
/* Loop over files in input Grp: remember Grps are indexed from 1 */
grouped = astKeyMap( "SortBy=KeyUp" );
for (i=1; i<=grpsize; i++) {
char newkey[128];
char dateobs[81];
char subarray[10];
size_t nrelated = 0;
AstKeyMap * filemap = NULL;
AstKeyMap * indexmap = NULL;
/* First step: open file and harvest metadata */
smf_open_file( NULL, igrp, i, "READ", SMF__NOCREATE_DATA, &data, status );
if (*status != SAI__OK) break;
if( i==1 ) {
isFFT = smf_isfft( data, NULL, NULL, NULL, NULL, NULL, status );
} else if( smf_isfft(data, NULL, NULL, NULL, NULL, NULL, status) != isFFT ){
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": mixture of time-series and FFT data encountered!",
status );
break;
}
/* If maxlen has not been set, do it here */
if( !maxlen && maxlen_s && data->hdr->steptime) {
maxlen = (dim_t) (maxlen_s / data->hdr->steptime );
}
/* Return srate_maxlen if requested: may want to know this number
even if maxlen_s is not set. Only calculate once, although it
gets overwritten once later if down-sampling. */
if( (i==1) && srate_maxlen && data->hdr->steptime ) {
*srate_maxlen = 1. / (double) data->hdr->steptime;
}
/* If requested check to see if we are mixing wavelengths */
if( checksubinst ) {
if( refsubinst == SMF__SUBINST_NONE ) {
refsubinst = smf_calc_subinst( data->hdr, status );
}
subinst = smf_calc_subinst( data->hdr, status );
if( subinst != refsubinst ) {
const char *refsubstr = smf_subinst_str( refsubinst, status );
const char *substr = smf_subinst_str( subinst, status );
*status = SAI__ERROR;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
msgSetc( "REFSUB", refsubstr );
msgSetc( "SUB", substr );
errRep( "", FUNC_NAME
": ^FILE uses sub-instrument ^SUB which doesn't match "
"reference ^REFSUB", status );
}
}
/* Want to form a key that will be unique for a particular subscan
We know that DATE-OBS will be set for SCUBA-2 files and be the same
for a single set. Prefix by wavelength if we are grouping by wavelength.
*/
newkey[0] = '\0';
smf_find_subarray( data->hdr, subarray, sizeof(subarray), NULL, status );
if( grouping == 1 ) {
/* Group different wavelengths separately */
smf_fits_getS( data->hdr, "WAVELEN", cwave, sizeof(cwave), status);
one_strlcat( newkey, cwave, sizeof(newkey), status );
one_strlcat( newkey, "_", sizeof(newkey), status );
}
if( grouping == 2 ) {
/* Group different subarrays separately */
one_strlcat( newkey, subarray, sizeof(newkey), status );
}
smf_fits_getS( data->hdr, "DATE-OBS", dateobs, sizeof(dateobs), status );
one_strlcat( newkey, dateobs, sizeof(newkey), status );
/* Include the dimentionality of the time series in the primary key
so that we do not end up doing something confusing like relating
a truncated file with a full length file */
if (*status == SAI__OK) {
dim_t dims[3];
char formatted[32];
smf_get_dims( data, &dims[0], &dims[1], NULL, &dims[2], NULL, NULL, NULL,
status );
sprintf(formatted, "_%" DIM_T_FMT "_%" DIM_T_FMT "_%" DIM_T_FMT, dims[0], dims[1], dims[2]);
one_strlcat( newkey, formatted, sizeof(newkey), status );
}
/* May want to read the dimensionality of the file outside of loop
so that we can compare values when storing in the keymap */
/* Now we want to create a keymap based on this key */
if (!astMapGet0A( grouped, newkey, &filemap ) ) {
int itemp = 0;
double steptime = data->hdr->steptime;
dim_t ntslice = 0;
dim_t thispad; /* Padding neeed for current input file */
filemap = astKeyMap( " " );
astMapPut0A( grouped, newkey, filemap, NULL );
/* Fill up filemap with general information on this file */
smf_find_seqcount( data->hdr, &itemp, status );
astMapPut0I( filemap, "SEQCOUNT", itemp, NULL );
smf_fits_getI( data->hdr, "NSUBSCAN", &itemp, status );
astMapPut0I( filemap, "NSUBSCAN", itemp, NULL );
/* Number of time slices */
smf_get_dims( data, NULL, NULL, NULL, &ntslice, NULL, NULL, NULL,
status );
/* Find length of down-sampled data, new steptime and maxlen */
if( (downsampscale || downsampfreq) && data->hdr && (*status==SAI__OK) ) {
double scalelen;
if( downsampscale ) {
if( data->hdr->scanvel != VAL__BADD ) {
double oldscale = steptime * data->hdr->scanvel;
scalelen = oldscale / downsampscale;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
scalelen = VAL__BADD;
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep( "", FUNC_NAME ": can't resample ^FILE because it has "
"unknown scan velocity", status );
}
} else {
if( steptime ) {
double oldsampfreq = 1./steptime;
scalelen = downsampfreq / oldsampfreq;
} else {
*status = SAI__ERROR;
scalelen = VAL__BADD;
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep( "", FUNC_NAME ": can't resample ^FILE because it has "
"unknown sample rate", status );
}
}
/* only down-sample if it will be a reasonable factor */
if( (*status==SAI__OK) && (scalelen <= SMF__DOWNSAMPLIMIT) ) {
smf_smfFile_msg(data->file, "FILE", 1, "" );
msgOutiff( MSG__VERB, "", FUNC_NAME
": will down-sample file ^FILE from %5.1lf Hz to "
"%5.1lf Hz", status, (1./steptime), (scalelen/steptime) );
ntslice = round(ntslice * scalelen);
/* If maxlen has been requested, and we have not already worked
out a scaled version (just uses the sample rates for the first
file... should be close enough -- the alternative is a 2-pass
system). */
if( !maxlen_scaled ) {
maxlen = round(maxlen*scalelen);
maxlen_scaled = 1;
msgOutiff( MSG__VERB, "", FUNC_NAME
": requested maxlen %g seconds = %" DIM_T_FMT " down-sampled "
"time-slices", status, maxlen_s, maxlen );
/* Return updated srate_maxlen for down-sampling if requested */
if( srate_maxlen ) {
*srate_maxlen = scalelen/steptime;
}
}
}
}
/* Check that an individual file is too long (we assume related
files are all the same) */
if( maxlen && (ntslice > maxlen) && *status == SAI__OK) {
*status = SAI__ERROR;
msgSeti("NTSLICE",ntslice);
msgSeti("MAXLEN",maxlen);
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep(FUNC_NAME,
"Number of time steps in file ^FILE time exceeds maximum "
"(^NTSLICE>^MAXLEN)", status);
}
/* Scaled values of ntslice and maximum length */
astMapPut0I( filemap, "NTSLICE", ntslice, NULL );
/* Work out the padding needed for this file including downsampling. */
if( keymap ) {
thispad = smf_get_padding( keymap, 0, data->hdr, VAL__BADD, status );
if( thispad > maxpad ) maxpad = thispad;
} else {
thispad = 0;
}
astMapPut0I( filemap, "PADDING", thispad, NULL );
/* Update maxflen */
if( ntslice > maxflen ) {
maxflen = ntslice;
}
/* Store OBSID or OBSIDSS depending on whether we are grouping by wavelength */
if (grouping) {
astMapPut0C( filemap, "OBSID", data->hdr->obsidss, NULL );
} else {
char obsid[81];
smf_getobsidss( data->hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status );
astMapPut0C( filemap, "OBSID", obsid, NULL );
}
}
/* Store the file index in another keymap indexed by subarray */
if ( !astMapGet0A( filemap, "GRPINDICES", &indexmap ) ) {
indexmap = astKeyMap( "SortBy=KeyUp" );
astMapPut0A( filemap, "GRPINDICES", indexmap, NULL );
}
astMapPut0I( indexmap, subarray, i, NULL );
/* Need to track the largest number of related subarrays in a single slot */
nrelated = astMapSize( indexmap );
if (nrelated > maxrelated) maxrelated = nrelated;
/* Free resources */
filemap = astAnnul( filemap );
indexmap = astAnnul( indexmap );
smf_close_file( NULL, &data, status );
}
/* We now know how many groups there are */
ngroups = astMapSize( grouped );
/* Sort out chunking. The items are sorted by date and then by wavelength.
We define a continuous chunk if it has the same OBSID, the same SEQCOUNT
and NSUBSCAN increments by one from the previous entry.
Also count number of related items in each slot.
*/
if (*status == SAI__OK) {
typedef struct { /* somewhere to store the values easily */
char obsid[81];
char related[81]; /* for concatenated subarrays */
int nsubscan;
int seqcount;
} smfCompareSeq;
smfCompareSeq current;
smfCompareSeq previous;
dim_t totlen = 0;
size_t thischunk;
/* Get the chunk flags and also store the size of the chunk */
chunk = astCalloc( ngroups, sizeof(*chunk) );
chunklen = astCalloc( ngroups, sizeof(*chunklen) );
piecelen = astCalloc( ngroups, sizeof(*piecelen) );
thischunk = 0; /* The current chunk */
for (i=0; i<ngroups; i++) {
AstKeyMap * thismap = NULL;
AstKeyMap * grpindices = NULL;
const char * tempstr = NULL;
int thistlen = 0;
size_t nsubarrays = 0;
/* Get the keymap entry for this slot */
astMapGet0A( grouped, astMapKey(grouped, i), &thismap );
/* Get info for length limits */
astMapGet0I( thismap, "NTSLICE", &thistlen );
piecelen[i] = thistlen;
if (isFFT) {
/* Never concatenate FFT data */
thismap = astAnnul(thismap);
chunk[i] = i;
chunklen[i] = thistlen;
continue;
}
/* Get indices information and retrieve the sub-instrument names
in sort order to concatenate for comparison. We only store in
a continuous chunk if we have the same subarrays for the whole
chunk. */
astMapGet0A( thismap, "GRPINDICES", &grpindices );
nsubarrays = astMapSize( grpindices );
(current.related)[0] = '\0';
for (j = 0; j < nsubarrays; j++ ) {
one_strlcat( current.related, astMapKey(grpindices, j), sizeof(current.related), status );
}
grpindices = astAnnul( grpindices );
/* Fill in the current struct */
astMapGet0I( thismap, "SEQCOUNT", &(current.seqcount) );
astMapGet0I( thismap, "NSUBSCAN", &(current.nsubscan) );
astMapGet0C( thismap, "OBSID", &tempstr );
one_strlcpy( current.obsid, tempstr, sizeof(current.obsid), status );
/* First chunk is special, else compare */
if (i == 0) {
totlen = thistlen;
} else {
if ( ( current.seqcount == previous.seqcount ) &&
( current.nsubscan - previous.nsubscan == 1 ) &&
( strcmp( current.obsid, previous.obsid ) == 0 ) &&
( strcmp( current.related, previous.related ) == 0 ) ) {
/* continuous - check length */
totlen += thistlen;
if ( maxlen && totlen > maxlen ) {
thischunk++;
totlen = thistlen; /* reset length */
} else {
/* Continuous */
}
} else {
/* discontinuity */
thischunk++;
totlen = thistlen; /* Update length of current chunk */
}
}
chunklen[thischunk] = totlen;
chunk[i] = thischunk;
memcpy( &previous, ¤t, sizeof(current) );
thismap = astAnnul( thismap );
}
}
/* Decide if we are keeping a chunk by looking at the length. */
maxconcat = 0;
nkeep = 0;
keepchunk = astMalloc( ngroups*sizeof(*keepchunk) );
for (i=0; i<ngroups; i++) {
size_t thischunk;
thischunk = chunk[i];
if ( chunklen[thischunk] < SMF__MINCHUNKSAMP ) {
/* Warning message */
msgSeti("LEN",chunklen[thischunk]);
msgSeti("MIN",SMF__MINCHUNKSAMP);
msgOut( " ", "SMF_GRP_RELATED: ignoring short chunk (^LEN<^MIN)",
status);
keepchunk[i] = 0;
} else {
keepchunk[i] = 1;
if (maxconcat < chunklen[thischunk]) maxconcat = chunklen[thischunk];
nkeep++;
}
}
/* If no useful chunks generate an error */
if( (*status==SAI__OK) && (!nkeep) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No useful chunks.", status );
goto CLEANUP;
}
/* Allocate a subgroup array of the right size and fill it. They keymap
is sorted by date (and wavelength) so we can always index into it by using
indices from the subgroup. */
subgroups = astCalloc( nkeep, sizeof(*subgroups) );
new_chunk = astCalloc( nkeep, sizeof(*new_chunk) );
new_tlen = astCalloc( nkeep, sizeof(*new_tlen) );
currentindex = 0;
for (i=0;i<ngroups;i++) {
AstKeyMap * thismap = NULL;
AstKeyMap * grpindices = NULL;
size_t nsubarrays = 0;
size_t *indices = astCalloc( maxrelated, sizeof(*indices) );
/* skip if we are dropping this chunk */
if (!keepchunk[i]) continue;
/* Get the keymap entry for this slot */
astMapGet0A( grouped, astMapKey(grouped, i), &thismap );
/* Get the indices keymap */
astMapGet0A( thismap, "GRPINDICES", &grpindices );
nsubarrays = astMapSize( grpindices );
for (j=0; j<nsubarrays; j++) {
int myindex;
astMapGet0I( grpindices, astMapKey(grpindices, j), &myindex );
indices[j] = myindex;
}
grpindices = astAnnul( grpindices );
thismap = astAnnul( thismap );
subgroups[currentindex] = indices;
new_chunk[currentindex] = chunk[i];
new_tlen[currentindex] = piecelen[i];
currentindex++;
}
/* Create the smfGroup */
*group = smf_construct_smfGroup( igrp, subgroups, new_chunk, new_tlen,
nkeep, maxrelated, 0, status );
/* Return maxfilelen if requested */
if( maxfilelen ) {
*maxfilelen = maxflen;
}
/* Return maxconcatlen if requested */
if( maxconcatlen ) {
*maxconcatlen = maxconcat;
}
/* Create a base group for output files if required */
/* Create a base group of filenames */
if (*status == SAI__OK && basegrp ) {
*basegrp = smf_grp_new( (*group)->grp, "Base Group", status );
/* Loop over time chunks */
for( i=0; (*status==SAI__OK)&&(i<(*group)->ngroups); i++ ) {
size_t idx;
/* Check for new continuous chunk */
if( i==0 || ( (*group)->chunk[i] != (*group)->chunk[i-1]) ) {
/* Loop over subarray */
for( idx=0; idx<(*group)->nrelated; idx++ ) {
size_t grpindex = (*group)->subgroups[i][idx];
if ( grpindex > 0 ) {
ndgCpsup( (*group)->grp, grpindex, *basegrp, status );
}
}
}
}
}
CLEANUP:
keepchunk = astFree( keepchunk );
chunk = astFree( chunk );
chunklen = astFree( chunklen );
piecelen = astFree( piecelen );
grouped = astAnnul( grouped );
if( *status != SAI__OK ) {
/* free the group */
if (basegrp && *basegrp) grpDelet( basegrp, status );
if (group && *group) {
smf_close_smfGroup( group, status );
} else {
/* have to clean up manually */
new_chunk = astFree( new_chunk );
new_tlen = astFree( new_tlen );
if( subgroups ) {
size_t isub;
for( isub=0; isub<nkeep; isub++ ) {
subgroups[isub] = astFree( subgroups[isub] );
}
subgroups = astFree( subgroups );
}
}
}
/* Return the maximum padding if required. */
if( pad ) *pad = maxpad;
}