void smurf_extinction( int * status ) {
/* Local Variables */
smfArray *bbms = NULL; /* Bad bolometer masks */
smfArray *darks = NULL; /* Dark data */
AstKeyMap *extpars = NULL; /* Tau relation keymap */
Grp *fgrp = NULL; /* Filtered group, no darks */
smfArray *flatramps = NULL;/* Flatfield ramps */
int has_been_sky_removed = 0;/* Data are sky-removed */
AstKeyMap *heateffmap = NULL; /* Heater efficiency data */
size_t i; /* Loop counter */
Grp *igrp = NULL; /* Input group */
AstKeyMap *keymap=NULL; /* Keymap for storing parameters */
smf_tausrc tausrc; /* enum value of optical depth source */
smf_extmeth extmeth; /* Extinction correction method */
char tausource[LEN__METHOD]; /* String for optical depth source */
char method[LEN__METHOD]; /* String for extinction airmass method */
smfData *odata = NULL; /* Output data struct */
Grp *ogrp = NULL; /* Output group */
size_t outsize; /* Total number of NDF names in the output group */
size_t size; /* Number of files in input group */
double tau = 0.0; /* Zenith tau at this wavelength */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
if (*status != SAI__OK) return;
/* Main routine */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Read the input file */
kpg1Rgndf( "IN", 0, 1, "", &igrp, &size, status );
/* Filter out darks */
smf_find_science( igrp, &fgrp, 0, NULL, NULL, 1, 1, SMF__NULL, &darks,
&flatramps, &heateffmap, NULL, status );
/* input group is now the filtered group so we can use that and
free the old input group */
size = grpGrpsz( fgrp, status );
grpDelet( &igrp, status);
igrp = fgrp;
fgrp = NULL;
if (size > 0) {
/* Get output file(s) */
kpg1Wgndf( "OUT", igrp, size, size, "More output files required...",
&ogrp, &outsize, status );
} else {
msgOutif(MSG__NORM, " ","All supplied input frames were DARK,"
" nothing to extinction correct", status );
}
/* Get group of pixel masks and read them into a smfArray */
smf_request_mask( "BBM", &bbms, status );
/* Read the tau relations from config file or group. We do not
allow sub instrument overloading because these are all values
based on filter name. */
keymap = kpg1Config( "TAUREL", "$SMURF_DIR/smurf_extinction.def", NULL,
1, status );
/* and we need to use the EXT entry */
astMapGet0A( keymap, "EXT", &extpars );
keymap = astAnnul( keymap );
/* Get tau source */
parChoic( "TAUSRC", "Auto",
"Auto,CSOtau,CSOFit, Filtertau, WVMraw", 1,
tausource, sizeof(tausource), status);
/* Decide how the correction is to be applied - convert to flag */
parChoic( "METHOD", "ADAPTIVE",
"Adaptive,Quick,Full,", 1, method, sizeof(method), status);
/* Place parameters into a keymap and extract values */
if( *status == SAI__OK ) {
keymap = astKeyMap( " " );
if( astOK ) {
astMapPut0C( keymap, "TAUSRC", tausource, NULL );
astMapPut0C( keymap, "TAUMETHOD", method, NULL );
smf_get_extpar( keymap, &tausrc, &extmeth, NULL, status );
}
}
for (i=1; i<=size && ( *status == SAI__OK ); i++) {
/* Flatfield - if necessary */
smf_open_and_flatfield( igrp, ogrp, i, darks, flatramps, heateffmap,
&odata, status );
if (*status != SAI__OK) {
/* Error flatfielding: tell the user which file it was */
msgSeti("I",i);
errRep(TASK_NAME, "Unable to open the ^I th file", status);
}
/* Mask out bad pixels - mask data array not quality array */
smf_apply_mask( odata, bbms, SMF__BBM_DATA, 0, status );
/* Now check that the data are sky-subtracted */
if ( !smf_history_check( odata, "smf_subtract_plane", status ) ) {
/* Should we override remsky check? */
parGet0l("HASSKYREM", &has_been_sky_removed, status);
if ( !has_been_sky_removed && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti("I",i);
errRep("", "Input data from file ^I are not sky-subtracted", status);
}
}
/* If status is OK, make decisions on source keywords the first
time through. */
if ( *status == SAI__OK && i == 1 ) {
if (tausrc == SMF__TAUSRC_CSOTAU ||
tausrc == SMF__TAUSRC_AUTO ||
tausrc == SMF__TAUSRC_TAU) {
double deftau;
const char * param = NULL;
smfHead *ohdr = odata->hdr;
/* get default CSO tau -- this could be calculated from CSO fits */
deftau = smf_calc_meantau( ohdr, status );
/* Now ask for desired CSO tau */
if ( tausrc == SMF__TAUSRC_CSOTAU || tausrc == SMF__TAUSRC_AUTO) {
param = "CSOTAU";
} else if (tausrc == SMF__TAUSRC_TAU) {
param = "FILTERTAU";
deftau = smf_cso2filt_tau( ohdr, deftau, extpars, status );
}
parGdr0d( param, deftau, 0.0,1.0, 1, &tau, status );
} else if ( tausrc == SMF__TAUSRC_CSOFIT || tausrc == SMF__TAUSRC_WVMRAW ) {
/* Defer a message until after extinction correction */
} else {
*status = SAI__ERROR;
errRep("", "Unsupported opacity source. Possible programming error.",
status);
}
}
/* Apply extinction correction - note that a check is made to
determine whether the data have already been extinction
corrected */
smf_correct_extinction( wf, odata, &tausrc, extmeth, extpars, tau, NULL, NULL, status );
if ( tausrc == SMF__TAUSRC_WVMRAW ) {
msgOutif(MSG__VERB," ", "Used Raw WVM data for extinction correction", status);
} else if ( tausrc == SMF__TAUSRC_CSOFIT ) {
msgOutif(MSG__VERB," ", "Used fit to CSO data for extinction correction", status);
} else if ( tausrc == SMF__TAUSRC_CSOTAU ) {
msgOutif(MSG__VERB," ", "Used an explicit CSO tau value for extinction correction", status);
} else if ( tausrc == SMF__TAUSRC_TAU ) {
msgOutif(MSG__VERB," ", "Used an explicit filter tau value for extinction correction", status);
} else {
if (*status == SAI__OK) {
const char * taustr = smf_tausrc_str( tausrc, status );
*status = SAI__ERROR;
errRepf( "", "Unexpected opacity source used for extinction correction of %s."
" Possible programming error.", status, taustr );
}
}
/* Set character labels */
smf_set_clabels( "Extinction corrected",NULL, NULL, odata->hdr, status);
smf_write_clabels( odata, status );
/* Free resources for output data */
smf_close_file( &odata, status );
}
/* Write out the list of output NDF names, annulling the error if a null
parameter value is supplied. */
if( *status == SAI__OK && ogrp ) {
grpList( "OUTFILES", 0, 0, NULL, ogrp, status );
if( *status == PAR__NULL ) errAnnul( status );
}
/* Tidy up after ourselves: release the resources used by the grp routines */
if (darks) smf_close_related( &darks, status );
if (bbms) smf_close_related( &bbms, status );
if( flatramps ) smf_close_related( &flatramps, status );
if (heateffmap) heateffmap = smf_free_effmap( heateffmap, status );
grpDelet( &igrp, status);
grpDelet( &ogrp, status);
if( keymap ) keymap = astAnnul( keymap );
if (extpars) extpars = astAnnul( extpars );
ndfEnd( status );
}
void smf_calc_stareimage( smfData *data, const int naver, int *status) {
double *avdata = NULL; /* Pointer to averaged data */
int dims[2]; /* Dimensions of the stored image */
smfHead *hdr; /* Header information */
int j; /* Loop counter */
size_t npts; /* Number of points in the averaged data */
int numaver; /* Number of samples to average */
int numimages; /* Number of output STARE images */
int numsamples; /* Number of time slices (samples) */
int remainder; /* Remainder from dividing no of timeslices
by number of frames to average */
HDSLoc *scu2redloc = NULL; /* Locator to SCU2RED extension */
double steptime; /* Step time per sample, sec */
double *zero = NULL; /* Bolometer zero points */
if ( *status != SAI__OK ) return;
/* Check we have time-series data */
if ( data->ndims != 3) {
*status = SAI__ERROR;
errRep(FUNC_NAME,
"File does not contain time series data - unable to process",
status);
return;
}
/* Check we have flatfielded data */
if ( !smf_history_check( data, "smf_flatfield", status) ) {
if ( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep(FUNC_NAME, "Data have not been flatfielded ", status);
return;
}
}
/* Do we have STARE data? */
hdr = data->hdr;
if ( hdr->obsmode == SMF__OBS_STARE ) {
msgOutif(MSG__VERB," ", "Processing STARE data", status);
if (smf_history_check( data, "smf_calc_stareimage", status) ) {
msgOut(" ", "File contains STARE data which has already been processed: proceeding but this WILL OVERWRITE any STARE images already written into the SCU2RED extension",
status);
}
/* Number of output STARE images */
numsamples = (data->dims)[2];
/* If naver < 0 then re-calculate to give 1-second images */
if ( naver < 0 ) {
smf_fits_getD(hdr, "STEPTIME", &steptime, status);
numaver = 1.0 / steptime;
} else if ( naver > numsamples ) {
msgOutif(MSG__NORM, "", "Warning: NAVER exceeds the number of samples - will average entire time stream to create a single image", status);
numaver = numsamples;
} else {
numaver = naver;
}
numimages = numsamples / numaver;
/* Warn user if the number to average is not a factor of the
number of time slices */
remainder = numsamples % numaver;
if ( remainder != 0 ) {
msgSeti("R", remainder);
msgSeti("N", numaver);
msgSeti("T", numsamples);
msgOutif(MSG__NORM, "", "Warning: NAVER (^N) is not a factor of the number of time slices (^T): final ^R samples will not be included in an image", status);
}
/* Helpful info for the user */
msgSeti("N",numimages);
if ( numimages == 1 ) {
msgSetc("IM","image");
} else {
msgSetc("IM","images");
}
msgOutif( MSG__VERB, "", "Calculating ^N STARE ^IM", status );
/* Obtain a locator for the extension where for the images will
be stored */
scu2redloc = smf_get_xloc(data, "SCU2RED", "SCUBA2_MAP_ARR", "WRITE",
0, NULL, status);
/* Set the dimensions of the output images - they are all the same
so they can be safely set outside the loop below */
dims[0] = (data->dims)[0];
dims[1] = (data->dims)[1];
/* Loop over the number of output images */
for ( j=0; j<numimages; j++) {
/* Average the time stream data over the desired interval */
smf_average_dataD( data, j*numaver, numaver, 1, &avdata, &npts, status );
/* Store the averaged data as an image */
smf_store_image( data, scu2redloc, j, 2, dims, numaver, 0, 0, avdata, NULL,
status);
avdata = astFree( avdata );
zero = astFree( zero );
}
/* Add a history entry if everything's OK */
smf_history_add(data, "smf_calc_stareimage", status );
/* Release SCU2RED locator */
datAnnul( &scu2redloc, status );
} else {
msgOutif(MSG__NORM," ",
"Input file is not a STARE observation - ignoring", status);
}
}
void smf_fit_poly( ThrWorkForce *wf, smfData *data, const size_t order,
int remove, double *poly, int *status) {
/* Local variables */
size_t bstride; /* bolo strides */
int i; /* Loop counter */
smfFitPolyData *job_data=NULL;/* Array of job data for each thread */
dim_t nbolo=0; /* Number of bolometers */
int njobs=0; /* Number of jobs to be processed */
dim_t ntslice = 0; /* Number of time slices */
int nw; /* Number of worker threads */
smfFitPolyData *pdata=NULL; /* Pointer to job data */
const smf_qual_t *qual; /* pointer to the quality array */
size_t step; /* step size for dividing up work */
size_t tstride; /* time strides */
/* Check status */
if (*status != SAI__OK) return;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Should check data type for double */
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status)) return;
if ( smf_history_check( data, FUNC_NAME, status) ) {
msgSetc("F", FUNC_NAME);
msgOutif(MSG__VERB," ",
"^F has already been run on these data, returning to caller",
status);
return;
}
/* Get the dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &bstride,
&tstride, status);
/* Return with error if there is no QUALITY component */
qual = smf_select_cqualpntr( data, NULL, status );
if( !qual && (*status == SAI__OK) ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Data doesn't have a QUALITY component.", status );
return;
}
/* Return with error if order is greater than the number of data
points */
if ( order >= ntslice ) {
if ( *status == SAI__OK) {
msgSeti("O",order);
msgSeti("NF",ntslice);
*status = SAI__ERROR;
errRep( FUNC_NAME, "Requested polynomial order, ^O, greater than or "
"equal to the number of points, ^NF. Unable to fit polynomial.",
status );
}
return;
}
/* Set up the job data */
if( nw > (int) nbolo ) {
step = 1;
} else {
step = nbolo/nw;
if( !step ) {
step = 1;
}
}
job_data = astCalloc( nw, sizeof(*job_data) );
for( i=0; (*status==SAI__OK)&&i<nw; i++ ) {
pdata = job_data + i;
pdata->b1 = i*step;
pdata->b2 = (i+1)*step-1;
/* if b1 is greater than the number of bolometers, we've run out of jobs */
if( pdata->b1 >= nbolo ) {
break;
}
/* increase the jobs counter */
njobs++;
/* Ensure that the last thread picks up any left-over bolometers */
if( (i==(nw-1)) && (pdata->b1<(nbolo-1)) ) {
pdata->b2=nbolo-1;
}
pdata->ijob = -1; /* Flag job as ready to start */
pdata->bstride = bstride;
pdata->indata = data->pntr[0];
pdata->isTordered = data->isTordered;
pdata->nbolo = nbolo;
pdata->ntslice = ntslice;
pdata->order = order;
pdata->poly = poly;
pdata->qual = qual;
pdata->remove = remove;
pdata->tstride = tstride;
}
/* Submit jobs to fit polynomial baselines to block of bolos */
thrBeginJobContext( wf, status );
for( i=0; (*status==SAI__OK)&&i<njobs; i++ ) {
pdata = job_data + i;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfFitPolyPar, 0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
thrEndJobContext( wf, status );
/* Free local resources. */
job_data = astFree( job_data );
}
int smf_correct_extinction(ThrWorkForce *wf, smfData *data, smf_tausrc *thetausrc, smf_extmeth method,
AstKeyMap * extpars, double tau, double *allextcorr,
double **wvmtaucache, int *status) {
/* Local variables */
int allquick = 0; /* Is the extinction for all bolometers the same? */
double amstart = VAL__BADD; /* Airmass at start */
double amend = VAL__BADD; /* Airmass at end */
double elstart = VAL__BADD; /* Elevation at start (radians) */
double elend = VAL__BADD;/* Elevation at end (radians) */
smfHead *hdr = NULL; /* Pointer to full header struct */
double *indata = NULL; /* Pointer to data array */
int isTordered; /* data order of input data */
int lbnd[2]; /* Lower bound */
size_t ndims; /* Number of dimensions in input data */
dim_t nframes = 0; /* Number of frames */
dim_t npts = 0; /* Number of data points */
dim_t nx = 0; /* # pixels in x-direction */
dim_t ny = 0; /* # pixels in y-direction */
smf_tausrc tausrc; /* Local copy of tausrc value */
int ubnd[2]; /* Upper bound */
double *vardata = NULL; /* Pointer to variance array */
double * wvmtau = NULL; /* WVM tau (smoothed or not) for these data */
int nw; /* Number of worker threads */
int iw; /* Thread index */
SmfCorrectExtinctionData *job_data = NULL; /* Array of job descriptions */
SmfCorrectExtinctionData *pdata; /* Pointer to next job description */
size_t framestep; /* Number of frames per thread */
/* Check status */
if (*status != SAI__OK) return allquick;
/* If no correction requested, return */
if( method==SMF__EXTMETH_NONE ) {
msgOutif(MSG__VERB, "", FUNC_NAME ": Extinction method=none, returning",
status );
return allquick;
}
if ( ! thetausrc ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Must supply a thetausrc argument. Possible programming error.",
status );
return allquick;
}
/* Use a local value for tausrc as we update it in this routine. In particular,
CSOFIT becomes WVMRAW and this would be confusing to the caller */
tausrc = *thetausrc;
/* If no opacity monitor specified generate bad status */
if( tausrc==SMF__TAUSRC_NULL ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No source of opacity information could be determined",
status );
return allquick;
}
if( smf_history_check( data, FUNC_NAME, status) ) {
/* If caller not requesting allextcorr fail here */
if( !allextcorr ) {
msgSetc("F", FUNC_NAME);
msgOutif(MSG__VERB," ",
"^F has already been run on these data, returning to caller",
status);
return allquick;
}
}
/* Acquire the data order */
isTordered = data->isTordered;
/* make sure we have a header */
hdr = data->hdr;
if( hdr == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Input data has no header", status);
return allquick;
}
/* Do we have 2-D image data? */
ndims = data->ndims;
if (ndims == 2) {
nframes = 1;
nx = (data->dims)[0];
ny = (data->dims)[1];
npts = nx*ny;
} else {
/* this routine will also check for dimensionality */
smf_get_dims( data, &nx, &ny, &npts, &nframes, NULL, NULL, NULL, status );
}
/* Tell user we're correcting for extinction */
msgOutif(MSG__VERB," ",
"Correcting for extinction.", status);
/* Should check data type for double if not allextcorr case */
if( !allextcorr ) {
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status)) return allquick;
}
/* Check that we're not trying to use the WVM for 2-D data */
if ( ndims == 2 && tausrc == SMF__TAUSRC_WVMRAW ) {
if ( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Method WVMRaw can not be used on 2-D image data", status );
return allquick;
}
} else if (ndims == 2 && tausrc == SMF__TAUSRC_CSOFIT ) {
/* This is CSOTAU mode with the value calculated from the fits. We have to either
calculate the value here based on the FITS headers or we have to ensure that
when this mode triggers we've been given the fallback tau derived in this manner.
Revisit this as the code develops (we do not want to be reading fits multiple times).
*/
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Method CSOFIT not yet supported on 2-D image data", status );
return allquick;
}
} else if (ndims < 2 || ndims > 3) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRepf( FUNC_NAME, "Can not extinction correct data with %zd dimension(s)", status,
ndims );
return allquick;
}
}
/* if we are WVMRAW, CSOFIT or AUTO and we have a cache we should always use it since
we assume it was filled in properly the previous time. */
if (wvmtaucache && *wvmtaucache &&
(tausrc == SMF__TAUSRC_WVMRAW ||
tausrc == SMF__TAUSRC_AUTO ||
tausrc == SMF__TAUSRC_CSOFIT)) {
wvmtau = *wvmtaucache;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using cached high resolution data for extinction correction of ^FILE",
status);
tausrc = SMF__TAUSRC_WVMRAW; /* We are now WVMRAW as we have the data */
/* Assume that we only do not know the provenance if in AUTO mode */
if (tausrc == SMF__TAUSRC_AUTO) *thetausrc = SMF__TAUSRC_CACHED;
}
if (!wvmtau && tausrc == SMF__TAUSRC_WVMRAW) {
size_t ntotaltau = 0;
size_t ngoodtau = 0;
smf_calc_smoothedwvm( wf, NULL, data, extpars, &wvmtau, &ntotaltau,
&ngoodtau, status );
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using WVM mode for extinction correction of ^FILE"
" %.0f %% of WVM data are present", status,
(double)(100.0*(double)ngoodtau/(double)ntotaltau) );
}
if (*status == SAI__OK && tausrc == SMF__TAUSRC_CSOFIT) {
/* Calculate the fit but we can use the same cache that WVM uses */
size_t nframes = 0;
smf_calc_csofit( data, extpars, &wvmtau, &nframes, status );
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__QUIET, "", "Using CSO fits for extinction correction of ^FILE",
status );
/* Rebrand as WVM data from this point on */
tausrc = SMF__TAUSRC_WVMRAW;
}
/* AUTO mode logic */
/*
* Default position is to use WVM data but we have two caveats
*
* 1. Was this observation done during a period where the WVM has been flagged as unreliable?
* 2. If the WVM is nominally okay, do we have sufficient good data during this period?
*
* If the WVM should not be used we fallback to seeing if we have a fit available for this
* night from the CSO data.
*
* If we do not have a reliable WVM or CSO fit then we fallback to using a fixed CSO number
* from the header.
*
* This final fallback position is unfortunate as it is highly likely that this is not a reliable
* number if we have fits for every night of observing (we have no information on whether a missing
* fit indicates the CSO was too unstable to use or whether it means we simply haven't got to it
* yet).
*
*/
/* Check auto mode */
if (tausrc == SMF__TAUSRC_AUTO && *status == SAI__OK) {
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
if (ndims == 2) {
/* have to use CSO mode */
tausrc = SMF__TAUSRC_CSOTAU;
*thetausrc = tausrc;
} else if (ndims == 3) {
/* We have already done the cache test so not needed here */
/* Is the WVM nominally stable for this night? */
if (smf_is_wvm_usable( data->hdr, status ) ) {
/* Calculate the WVM tau data and see if we have enough good data */
size_t ngoodtau = 0;
size_t ntotaltau = 0;
double percentgood = 0.0;
smf_calc_smoothedwvm( wf, NULL, data, extpars, &wvmtau, &ntotaltau,
&ngoodtau, status );
percentgood = 100.0 * ((double)ngoodtau / (double)ntotaltau);
if ( percentgood > 80.0) {
tausrc = SMF__TAUSRC_WVMRAW;
msgOutiff( MSG__VERB, "", "Selecting WVM mode for extinction correction of ^FILE."
" %.0f %% of WVM data are present", status, percentgood );
*thetausrc = tausrc;
} else {
tausrc = SMF__TAUSRC_AUTO; /* keep it AUTO (a no-op but make it clear) */
if (wvmtau) wvmtau = astFree( wvmtau );
}
}
/* at this point we either have WVM data handled or we still think we are AUTO.
Do a CSO FIT check */
if (tausrc == SMF__TAUSRC_AUTO && *status == SAI__OK) {
size_t nframes = 0;
smf_calc_csofit( data, extpars, &wvmtau, &nframes, status );
if (*status == SAI__OK) {
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__QUIET, "", "Using CSO fits for extinction correction of ^FILE",
status );
/* Rebrand as WVM data from this point on */
tausrc = SMF__TAUSRC_WVMRAW;
*thetausrc = SMF__TAUSRC_CSOFIT;
} else if (*status == SMF__BADFIT) {
/* No fit, carry on. */
errAnnul( status );
}
}
/* At this point if we are not WVMRAW then we have a serious issue. It means that
WVM was unusable and we did not have a good CSO fit. We should not continue at this
point as to continue implies that we know what we should do. The user should decide
how much they trust the opacity for the night. There has to be a reason why there
is no CSO fit for the night. */
if (*status == SAI__OK && tausrc != SMF__TAUSRC_WVMRAW) {
*status = SAI__ERROR;
errRep("", "Unable to determine opacity data for this observation. Both WVM and CSO fits failed. Please investigate and if necessary use CSO mode explicitly but proceed with caution.", status );
}
}
}
/* If we have a CSO Tau then convert it to the current filter. This will also
convert bad values to a value derived from the header if appropriate. */
if ( tausrc == SMF__TAUSRC_CSOTAU ) {
tau = smf_cso2filt_tau( hdr, tau, extpars, status );
/* The tau source is now a real tau */
tausrc = SMF__TAUSRC_TAU;
}
/* Find the airmass range for this data */
smf_find_airmass_interval( hdr, &amstart, &amend, &elstart, &elend, status );
if (*status == SAI__OK && (amstart == VAL__BADD || amend == VAL__BADD)) {
*status = SAI__ERROR;
errRep( "", "No good airmass values found in JCMTSTATE structure for these data",
status );
}
/* if we are not doing WVM correction but are in adaptive mode we can determine
whether or not we will have to use full or single mode just by looking at the
airmass data. */
if (ndims == 3 && tausrc != SMF__TAUSRC_WVMRAW && method == SMF__EXTMETH_ADAPT) {
/* first and last is a good approximation given that most SCUBA-2 files will only
be a minute duration. */
double refel;
double refam;
/* only need to examine the largest airmass */
if (amstart > amend) {
refam = amstart;
refel = elstart;
} else {
refam = amend;
refel = elend;
}
/* and choose a correction method */
if (is_large_delta_atau( refam, refel, tau, status) ) {
method = SMF__EXTMETH_FULL;
msgOutiff(MSG__DEBUG, " ",
"Adaptive extinction algorithm selected per-bolometer airmass value "
"per time slice (am=%g, tau=%g)", status, refam, tau);
} else {
msgOutiff(MSG__DEBUG, " ",
"Adaptive extinction algorithm selected single airmass value per time slice"
" (am=%g, tau=%g)", status, refam, tau);
method = SMF__EXTMETH_SINGLE;
}
}
/* Assign pointer to input data array if status is good */
if ( *status == SAI__OK ) {
indata = (data->pntr)[0];
vardata = (data->pntr)[1];
}
/* Jump to the cleanup section if status is bad by this point
since we need to free memory */
if (*status != SAI__OK) goto CLEANUP;
/* Array bounds for astTranGrid call */
lbnd[0] = 1;
lbnd[1] = 1;
ubnd[0] = nx;
ubnd[1] = ny;
/* Unlock the AST objects in the smfData so that the worker threads can
lock them. */
smf_lock_data( data, 0, status );
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many frames to process in each worker thread. */
framestep = nframes/nw;
if( framestep == 0 ) {
framestep = 1;
nw = nframes;
}
/* Allocate job data for threads, and store the range of frames to be
processed by each one. Ensure that the last thread picks up any
left-over frames. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->f1 = iw*framestep;
if( iw < nw - 1 ) {
pdata->f2 = pdata->f1 + framestep - 1;
} else {
pdata->f2 = nframes - 1 ;
}
pdata->nframes = nframes;
pdata->npts = npts;
pdata->allextcorr = allextcorr;
pdata->indata = indata;
pdata->tau = tau;
pdata->vardata = vardata;
pdata->wvmtau = wvmtau;
pdata->amstart = amstart;
pdata->amfirst = amstart + ( amend - amstart )*pdata->f1/( nframes - 1 );
pdata->lbnd = lbnd;
pdata->ubnd = ubnd;
pdata->isTordered = isTordered;
pdata->ndims = ndims;
pdata->data = data;
pdata->hdr = hdr;
pdata->method = method;
pdata->tausrc = tausrc;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_correct_extinction, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Record if all time slices used a single air mass. */
allquick = 1;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
if( ! pdata->allquick ) {
allquick = 0;
break;
}
}
/* Free the job data. */
job_data = astFree( job_data );
}
/* Lock the AST objects in the smfData for use by this thread. */
smf_lock_data( data, 1, status );
/* Add history entry if !allextcorr */
if( (*status == SAI__OK) && !allextcorr ) {
smf_history_add( data, FUNC_NAME, status);
}
CLEANUP:
if (wvmtaucache) {
if (!*wvmtaucache) {
*wvmtaucache = wvmtau;
}
} else {
wvmtau = astFree( wvmtau );
}
return allquick;
}
int smf_correct_extinction(ThrWorkForce *wf, smfData *data, smf_tausrc tausrc, smf_extmeth method,
AstKeyMap * extpars, double tau, double *allextcorr,
double **wvmtaucache, int *status) {
/* Local variables */
int allquick = 0; /* Is the extinction for all bolometers the same? */
double amstart = VAL__BADD; /* Airmass at start */
double amend = VAL__BADD; /* Airmass at end */
double elstart = VAL__BADD; /* Elevation at start (radians) */
double elend = VAL__BADD;/* Elevation at end (radians) */
smfHead *hdr = NULL; /* Pointer to full header struct */
double *indata = NULL; /* Pointer to data array */
int isTordered; /* data order of input data */
int lbnd[2]; /* Lower bound */
size_t ndims; /* Number of dimensions in input data */
dim_t nframes = 0; /* Number of frames */
dim_t npts = 0; /* Number of data points */
dim_t nx = 0; /* # pixels in x-direction */
dim_t ny = 0; /* # pixels in y-direction */
int ubnd[2]; /* Upper bound */
double *vardata = NULL; /* Pointer to variance array */
double * wvmtau = NULL; /* WVM tau (smoothed or not) for these data */
int nw; /* Number of worker threads */
int iw; /* Thread index */
SmfCorrectExtinctionData *job_data = NULL; /* Array of job descriptions */
SmfCorrectExtinctionData *pdata; /* Pointer to next job description */
size_t framestep; /* Number of frames per thread */
/* Check status */
if (*status != SAI__OK) return allquick;
/* If no correction requested, return */
if( method==SMF__EXTMETH_NONE ) {
msgOutif(MSG__VERB, "", FUNC_NAME ": Extinction method=none, returning",
status );
return allquick;
}
/* If no opacity monitor specified generate bad status */
if( tausrc==SMF__TAUSRC_NULL ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No extinction monitor specified",
status );
return allquick;
}
if( smf_history_check( data, FUNC_NAME, status) ) {
/* If caller not requesting allextcorr fail here */
if( !allextcorr ) {
msgSetc("F", FUNC_NAME);
msgOutif(MSG__VERB," ",
"^F has already been run on these data, returning to caller",
status);
return allquick;
}
}
/* Acquire the data order */
isTordered = data->isTordered;
/* make sure we have a header */
hdr = data->hdr;
if( hdr == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Input data has no header", status);
return allquick;
}
/* Do we have 2-D image data? */
ndims = data->ndims;
if (ndims == 2) {
nframes = 1;
nx = (data->dims)[0];
ny = (data->dims)[1];
npts = nx*ny;
} else {
/* this routine will also check for dimensionality */
smf_get_dims( data, &nx, &ny, &npts, &nframes, NULL, NULL, NULL, status );
}
/* Tell user we're correcting for extinction */
msgOutif(MSG__VERB," ",
"Correcting for extinction.", status);
/* Should check data type for double if not allextcorr case */
if( !allextcorr ) {
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status)) return allquick;
}
/* Check that we're not trying to use the WVM for 2-D data */
if ( ndims == 2 && tausrc == SMF__TAUSRC_WVMRAW ) {
if ( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Method WVMRaw can not be used on 2-D image data", status );
return allquick;
}
} else if (ndims < 2 || ndims > 3) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRepf( FUNC_NAME, "Can not extinction correct data with %zd dimension(s)", status,
ndims );
return allquick;
}
}
if (tausrc == SMF__TAUSRC_WVMRAW) {
size_t ntotaltau = 0;
size_t ngoodtau = 0;
/* calculate WVM unless we have external values */
if (wvmtaucache && *wvmtaucache) {
wvmtau = *wvmtaucache;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using cached WVM data for extinction correction of ^FILE",
status);
} else {
smf_calc_smoothedwvm( wf, NULL, data, extpars, &wvmtau, &ntotaltau,
&ngoodtau, status );
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using WVM mode for extinction correction of ^FILE"
" %.0f %% of WVM data are present", status,
(double)(100.0*(double)ngoodtau/(double)ntotaltau) );
}
}
/* Check auto mode */
if (tausrc == SMF__TAUSRC_AUTO && *status == SAI__OK) {
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
if (ndims == 2) {
/* have to use CSO mode */
tausrc = SMF__TAUSRC_CSOTAU;
} else if (ndims == 3) {
/* Calculate the WVM tau data and see if we have enough good data */
size_t ngoodtau = 0;
size_t ntotaltau = 0;
double percentgood = 0.0;
if (wvmtaucache && *wvmtaucache) {
wvmtau = *wvmtaucache;
tausrc = SMF__TAUSRC_WVMRAW;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using cached WVM data for extinction correction of ^FILE",
status );
} else {
smf_calc_smoothedwvm( wf, NULL, data, extpars, &wvmtau, &ntotaltau,
&ngoodtau, status );
percentgood = 100.0 * ((double)ngoodtau / (double)ntotaltau);
if ( percentgood > 80.0) {
tausrc = SMF__TAUSRC_WVMRAW;
msgOutiff( MSG__VERB, "", "Selecting WVM mode for extinction correction of ^FILE."
" %.0f %% of WVM data are present", status, percentgood );
} else {
tausrc = SMF__TAUSRC_CSOTAU;
if (wvmtau) wvmtau = astFree( wvmtau );
}
}
}
if (tausrc == SMF__TAUSRC_CSOTAU) {
msgOutiff( MSG__VERB, "", "Selecting CSO mode for extinction correction of ^FILE", status );
} else if (tausrc == SMF__TAUSRC_WVMRAW) {
/* Dealt with this above */
} else {
/* oops. Fall back position */
tausrc = SMF__TAUSRC_CSOTAU;
msgOutiff( MSG__VERB, "", "Selecting CSO mode as unexpected fallback for extinction correction of ^FILE", status );
}
}
/* If we have a CSO Tau then convert it to the current filter. This will also
convert bad values to a value derived from the header if appropriate. */
if ( tausrc == SMF__TAUSRC_CSOTAU ) {
tau = smf_cso2filt_tau( hdr, tau, extpars, status );
/* The tau source is now a real tau */
tausrc = SMF__TAUSRC_TAU;
}
/* Find the airmass range for this data */
smf_find_airmass_interval( hdr, &amstart, &amend, &elstart, &elend, status );
if (*status == SAI__OK && (amstart == VAL__BADD || amend == VAL__BADD)) {
*status = SAI__ERROR;
errRep( "", "No good airmass values found in JCMTSTATE structure for these data",
status );
}
/* if we are not doing WVM correction but are in adaptive mode we can determine
whether or not we will have to use full or single mode just by looking at the
airmass data. */
if (ndims == 3 && tausrc != SMF__TAUSRC_WVMRAW && method == SMF__EXTMETH_ADAPT) {
/* first and last is a good approximation given that most SCUBA-2 files will only
be a minute duration. */
double refel;
double refam;
/* only need to examine the largest airmass */
if (amstart > amend) {
refam = amstart;
refel = elstart;
} else {
refam = amend;
refel = elend;
}
/* and choose a correction method */
if (is_large_delta_atau( refam, refel, tau, status) ) {
method = SMF__EXTMETH_FULL;
msgOutiff(MSG__DEBUG, " ",
"Adaptive extinction algorithm selected per-bolometer airmass value "
"per time slice (am=%g, tau=%g)", status, refam, tau);
} else {
msgOutiff(MSG__DEBUG, " ",
"Adaptive extinction algorithm selected single airmass value per time slice"
" (am=%g, tau=%g)", status, refam, tau);
method = SMF__EXTMETH_SINGLE;
}
}
/* Assign pointer to input data array if status is good */
if ( *status == SAI__OK ) {
indata = (data->pntr)[0];
vardata = (data->pntr)[1];
}
/* Jump to the cleanup section if status is bad by this point
since we need to free memory */
if (*status != SAI__OK) goto CLEANUP;
/* Array bounds for astTranGrid call */
lbnd[0] = 1;
lbnd[1] = 1;
ubnd[0] = nx;
ubnd[1] = ny;
/* Unlock the AST objects in the smfData so that the worker threads can
lock them. */
smf_lock_data( data, 0, status );
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many frames to process in each worker thread. */
framestep = nframes/nw;
if( framestep == 0 ) {
framestep = 1;
nw = nframes;
}
/* Allocate job data for threads, and store the range of frames to be
processed by each one. Ensure that the last thread picks up any
left-over frames. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->f1 = iw*framestep;
if( iw < nw - 1 ) {
pdata->f2 = pdata->f1 + framestep - 1;
} else {
pdata->f2 = nframes - 1 ;
}
pdata->nframes = nframes;
pdata->npts = npts;
pdata->allextcorr = allextcorr;
pdata->indata = indata;
pdata->tau = tau;
pdata->vardata = vardata;
pdata->wvmtau = wvmtau;
pdata->amstart = amstart;
pdata->amfirst = amstart + ( amend - amstart )*pdata->f1/( nframes - 1 );
pdata->lbnd = lbnd;
pdata->ubnd = ubnd;
pdata->isTordered = isTordered;
pdata->ndims = ndims;
pdata->data = data;
pdata->hdr = hdr;
pdata->method = method;
pdata->tausrc = tausrc;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_correct_extinction, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Record if all time slices used a single air mass. */
allquick = 1;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
if( ! pdata->allquick ) {
allquick = 0;
break;
}
}
/* Free the job data. */
job_data = astFree( job_data );
}
/* Lock the AST objects in the smfData for use by this thread. */
smf_lock_data( data, 1, status );
/* Add history entry if !allextcorr */
if( (*status == SAI__OK) && !allextcorr ) {
smf_history_add( data, FUNC_NAME, status);
}
CLEANUP:
if (wvmtaucache) {
if (!*wvmtaucache) {
*wvmtaucache = wvmtau;
}
} else {
wvmtau = astFree( wvmtau );
}
return allquick;
}
