void
smf_flat_standardpow( const smfData * bolvald, double refohms,
const double resistance[],
smfData ** powrefd, smfData ** bolrefd, int * status) {
double current; /* current through heaters */
double *heatframe = NULL; /* Pointer to flatfield heater data */
double *heatframevar = NULL; /* Pointer to flatfield heatframe variance */
double * heatref = NULL; /* local copy of heater settings */
size_t i; /* loop counter */
size_t j; /* loop counter */
size_t k; /* loop counter */
size_t nheat; /* Number of input frames */
size_t numbol; /* Number of bolometers */
double *powbol = NULL; /* pointer to individual bolometer powers */
double s; /* interpolation slope factor */
double * powref = NULL; /* Data array inside powrefd */
double * bolref = NULL; /* Data array inside bolrefd */
double * bolrefvar = NULL; /* Variance inside bolrefd */
*powrefd = NULL;
*bolrefd = NULL;
if (*status != SAI__OK) return;
if ( !smf_dtype_check_fatal( bolvald, NULL, SMF__DOUBLE, status ) ) return;
if (!bolvald->da) {
*status = SAI__ERROR;
errRep( "", "Unable to proceed with flatfield calculation. Heater settings not "
"present in smfData", status);
return;
}
nheat = (bolvald->dims)[2];
numbol = (bolvald->dims)[0] * (bolvald->dims)[1];
heatref = bolvald->da->heatval;
/* Create a smfData for powref and bolref */
smf_flat_malloc( nheat, bolvald, powrefd, bolrefd, status );
if (*status == SAI__OK) {
powref = (*powrefd)->pntr[0];
bolref = (*bolrefd)->pntr[0];
bolrefvar = (*bolrefd)->pntr[1];
}
/* Get some memory -
bolometer power per heater setting */
powbol = astCalloc( nheat, sizeof(*powbol) );
/* pointers to the flatfield data and variance */
heatframe = (bolvald->pntr)[0];
heatframevar = (bolvald->pntr)[1];
/* Choose the reference heater powers to be the actual heater settings acting on
the adopted reference resistance. */
for ( j=0; j<nheat; j++ ) {
current = heatref[j] * SC2FLAT__DTOI;
powref[j] = SIMULT * current * current * refohms;
}
if (*status != SAI__OK) goto CLEANUP;
/* For each bolometer interpolate to find the measurement it would report at the
standard power input values */
for ( i=0; i<numbol; i++ )
{
/* Calculate the actual power seen by each bolometer at each heater setting */
for ( j=0; j<nheat; j++ )
{
current = heatref[j] * SC2FLAT__DTOI;
if ( resistance[i] == VAL__BADD )
{
powbol[j] = VAL__BADD;
}
else
{
powbol[j] = SIMULT * current * current * resistance[i];
}
}
/* Trap bolometers that have any bad values in their measurements */
for ( j=0; j<nheat; j++) {
if ( heatframe[i + j*numbol] == VAL__BADD) {
powbol[0] = VAL__BADD;
break;
}
}
/* Trap bolometers which are unresponsive */
if ( powbol[0] != VAL__BADD &&
(heatframe[i] == VAL__BADD || heatframe[i + (nheat-1)*numbol] == VAL__BADD ||
(fabs ( heatframe[i] - heatframe[i + (nheat-1)*numbol] ) < 1.0) ) )
{
powbol[0] = VAL__BADD;
}
for ( j=0; j<nheat; j++ )
{
if ( powbol[0] == VAL__BADD )
{
bolref[j*numbol+i] = VAL__BADD;
bolrefvar[j*numbol+i] = VAL__BADD;
}
else
{
if ( powref[j] < powbol[0] )
{
double var = VAL__BADD;
/* Standard point is below actual measured range for this bolometer,
extrapolate */
s = ( heatframe[i + 1*numbol] - heatframe[i + 0*numbol] ) /
( powbol[1] - powbol[0] );
/* calculate error in gradient using standard rules */
if (heatframevar && heatframevar[i+ 1*numbol] != VAL__BADD &&
heatframevar[i + 0*numbol] != VAL__BADD) {
var = ( heatframevar[i + 1*numbol] + heatframevar[i + 0*numbol] )
/ pow( powbol[1] - powbol[0], 2 );
}
bolref[j*numbol+i] = heatframe[i+ 0*numbol] +
s * ( powref[j] - powbol[0] );
if (var != VAL__BADD) {
bolrefvar[j*numbol+i] = heatframevar[i + 0*numbol] +
( var * pow( powref[j] - powbol[0], 2 ));
} else {
bolrefvar[j*numbol+i] = VAL__BADD;
}
}
else if ( powref[j] > powbol[nheat-1] )
{
double var = VAL__BADD;
/* Standard point is above actual measured range for this bolometer,
extrapolate */
s = ( heatframe[i + (nheat-1)*numbol] - heatframe[i + (nheat-2)*numbol] ) /
( powbol[nheat-1] - powbol[nheat-2] );
/* calculate error in gradient using standard rules */
if (heatframevar && heatframevar[i + (nheat-1)*numbol] != VAL__BADD &&
heatframevar[i + (nheat-2)*numbol] != VAL__BADD) {
var = ( heatframevar[i + (nheat-1)] + heatframevar[i + (nheat-2)*numbol] )
/ pow( powbol[nheat-1] - powbol[nheat-2], 2 );
}
bolref[j*numbol+i] = heatframe[i + (nheat-2)*numbol] +
s * ( powref[j] - powbol[nheat-2] );
if (var != VAL__BADD) {
bolrefvar[j*numbol+i] = heatframevar[i +(nheat-2)*numbol] +
( var * pow( powref[j] - powbol[nheat-2], 2 ));
} else {
bolrefvar[j*numbol+i] = VAL__BADD;
}
}
else
{
/* Standard point is within actual measured range for this bolometer,
search for the points to interpolate */
for ( k=1; k<nheat; k++ )
{
if ( powref[j] <= powbol[k] )
{
double var = VAL__BADD;
s = ( heatframe[i + k*numbol] -
heatframe[i + (k-1)*numbol] ) /
( powbol[k] - powbol[k-1] );
/* calculate error in gradient using standard rules */
if (heatframevar && heatframevar[i + k*numbol] != VAL__BADD &&
heatframevar[i +(k-1)*numbol] != VAL__BADD) {
var = ( heatframevar[i + k*numbol] + heatframevar[i + (k-1)*numbol] )
/ pow( powbol[k] - powbol[k-1], 2 );
}
bolref[j*numbol+i] = heatframe[i + (k-1)*numbol] +
s * ( powref[j] - powbol[k-1] );
if (var != VAL__BADD) {
bolrefvar[j*numbol+i] = heatframevar[i +(k-1)*numbol] +
( var * pow( powref[j] - powbol[k-1], 2 ));
} else {
bolrefvar[j*numbol+i] = VAL__BADD;
}
break;
}
}
}
}
}
}
CLEANUP:
if (powbol) powbol = astFree( powbol );
if (*status != SAI__OK) {
if (*bolrefd) smf_close_file( NULL, bolrefd, status );
if (*powrefd) smf_close_file( NULL, powrefd, status );
}
}
void
smf_flat_params( const smfData * refdata, const char resistpar[],
const char methpar[], const char orderpar[], const char snrminpar[],
double * refohms, double **resistance, int * outrows,
int * outcols, smf_flatmeth *flatmeth,
int * order, double * snrmin, smfData ** heateff,
int * status ) {
dim_t datarows = 0; /* Number of rows in refdata */
dim_t datacols = 0; /* Number of columns in refdata */
size_t j = 0; /* Counter, index */
char method[SC2STORE_FLATLEN]; /* flatfield method string */
size_t nbols; /* Number of bolometers */
double refohmsval = 0.0; /* Internal version of refohms */
AstKeyMap * resmap = NULL; /* Resistor map */
AstKeyMap * subarrays = NULL; /* Subarray lookup table */
char thissub[32]; /* This sub-instrument string */
if (resistance) *resistance = NULL;
if (*status != SAI__OK) return;
if (!refdata) {
*status = SAI__ERROR;
errRep( "", "Must provide reference data file to calculate flatfield parameters"
" (possible programming error)", status );
return;
}
/* Based on refdata we now need to calculate the default reference
resistance and retrieve the correct heater efficiency file for each array.
We need the unique subarray string so that we can set up a look up keymap.
There is no code in SMURF to return all the known subarrays but
we need to know all the options in order to use kpg1Config. */
subarrays = astKeyMap( " " );
astMapPut0I( subarrays, "CG450MK2_M0907D0501", 0, NULL );
astMapPut0I( subarrays, "CG850MK2_M0904D0503", 0, NULL );
astMapPut0I( subarrays, "SG850_M0906D1005", 0, NULL );
astMapPut0I( subarrays, "SG850_M1002D1006", 0, NULL );
astMapPut0I( subarrays, "SG850_M1005D1007", 0, NULL );
astMapPut0I( subarrays, "SG850_M1003D1004", 0, NULL );
astMapPut0I( subarrays, "SG450_M1004D1000", 0, NULL );
astMapPut0I( subarrays, "SG450_M1007D1002", 0, NULL );
astMapPut0I( subarrays, "SG450_M1006D1003", 0, NULL );
astMapPut0I( subarrays, "SG450_M1009D1008", 0, NULL );
/* and indicate which subarray we are interested in (uppercased) */
smf_fits_getS( refdata->hdr, "ARRAYID", thissub, sizeof(thissub), status );
{ /* need to uppercase */
size_t l = strlen(thissub);
for (j=0;j<l;j++) {
thissub[j] = toupper(thissub[j]);
}
}
astMapPut0I( subarrays, thissub, 1, NULL );
/* Read the config file */
resmap = kpg1Config( resistpar, "$SMURF_DIR/smurf_calcflat.def",
subarrays, 1, status );
subarrays = astAnnul( subarrays );
if (*status != SAI__OK) goto CLEANUP;
/* Read the reference resistance */
astMapGet0D( resmap, "REFRES", &refohmsval );
if (refohms && *status == SAI__OK) {
*refohms = refohmsval;
msgOutiff(MSG__VERB, "",
"Read reference resistance for subarray %s of %g ohms\n",
status, thissub, *refohms );
}
/* We no longer want to read per-bolometer resistor values from the
config file. To retain backwards compatibility with the current
implementation of smf_flat_standardpow we simply fill the
per-bol resistance array with the reference resistance which
effectively disables smf_flat_standardpow */
smf_get_dims( refdata, &datarows, &datacols, NULL, NULL, NULL, NULL, NULL, status );
nbols = datacols * datarows;
if (*status == SAI__OK && resistance ) {
*resistance = astMalloc( nbols*sizeof(**resistance) );
for (j = 0; j < (size_t)nbols; j++) {
(*resistance)[j] = refohmsval;
}
}
/* Get the heater efficiency file */
if (heateff && astMapHasKey( resmap, "HEATEFF" ) ) {
const char * heateffstr = NULL;
if (astMapGet0C( resmap, "HEATEFF", &heateffstr )) {
Grp * heateffgrp = NULL;
smfData * heatefftmp = NULL;
heateffgrp = grpNew( "heateff", status );
grpPut1( heateffgrp, heateffstr, 0, status );
smf_open_file( NULL, heateffgrp, 1, "READ", SMF__NOTTSERIES|SMF__NOFIX_METADATA, &heatefftmp, status );
/* Divorce the smfData from the underlying file. This file stays open for the entire
duration of the data processing and can some times lead to issues when we attempt
to close it an hour after we opened it (it's usually on an NFS disk) */
if (*status == SAI__OK) {
*heateff = smf_deepcopy_smfData( NULL, heatefftmp, 0,
SMF__NOCREATE_FILE | SMF__NOCREATE_FTS |
SMF__NOCREATE_DA,
0, 0, status );
smf_close_file(NULL, &heatefftmp, status);
}
/* Check the dimensions */
if (*status == SAI__OK) {
dim_t heatrows = 0;
dim_t heatcols = 0;
smf_get_dims( *heateff, &heatrows, &heatcols, NULL, NULL, NULL, NULL, NULL, status );
if (*status == SAI__OK) {
if ( datarows != heatrows || datacols != heatcols ) {
*status = SAI__ERROR;
errRepf( "", "Dimensions of heater efficiency file %s are (%zu, %zu)"
" but flatfield has dimensions (%zu, %zu)",
status, heateffstr, (size_t)heatrows, (size_t)heatcols,
(size_t)datarows, (size_t)datacols);
}
}
if (*status == SAI__OK) {
smf_dtype_check_fatal( *heateff, NULL, SMF__DOUBLE, status );
if (*status == SMF__BDTYP) {
errRepf("", "Heater efficiency data in %s should be double precision",
status, heateffstr);
}
}
if (*status == SAI__OK) {
char heateffarrid[32];
smf_fits_getS( refdata->hdr, "ARRAYID", heateffarrid, sizeof(heateffarrid), status );
if (*status != SAI__OK) errAnnul( status );
if (strcasecmp( thissub, heateffarrid ) != 0 ) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRepf("", "Subarray associated with heater efficiency image (%s)"
" does not match that of the data to be flatfielded (%s)",
status, heateffarrid, thissub );
}
}
}
}
if (heateffgrp) grpDelet( &heateffgrp, status );
}
}
if (methpar && flatmeth) {
/* See if we want to use TABLE or POLYNOMIAL mode */
parChoic( methpar, "POLYNOMIAL", "POLYNOMIAL, TABLE", 1,
method, sizeof(method), status );
*flatmeth = smf_flat_methcode( method, status );
if (*flatmeth == SMF__FLATMETH_POLY) {
/* need an order for the polynomial */
if (order && orderpar) {
parGdr0i( orderpar, 1, 1, 3, 1, order, status );
/* and if the order is 1 then we can ask for the snr min */
if (snrminpar && *order == 1) {
parGet0d( snrminpar, snrmin, status );
}
}
} else {
/* need an snr min for table mode responsivities */
if (snrminpar) parGet0d( snrminpar, snrmin, status );
}
}
if (outrows) *outrows = datarows;
if (outcols) *outcols = datacols;
CLEANUP:
resmap = astAnnul( resmap );
if (*status != SAI__OK) {
if (resistance && *resistance) *resistance = astFree( *resistance );
if (heateff && *heateff) smf_close_file( NULL, heateff, status );
}
return;
}
void smf_bolonoise( ThrWorkForce *wf, smfData *data, double gfrac,
size_t window, double f_low,
double f_white1, double f_white2,
int nep, size_t len, double *whitenoise, double *fratio,
smfData **fftpow,int *status ) {
double *base=NULL; /* Pointer to base coordinates of array */
size_t bstride; /* bolometer index stride */
double df=1; /* Frequency step size in Hz */
size_t i; /* Loop counter */
size_t i_low; /* Index in power spectrum to f_low */
size_t i_w1; /* Index in power spectrum to f_white1 */
size_t i_w2; /* Index in power spectrum to f_white2 */
size_t j; /* Loop counter */
size_t mingood; /* Min. required no. of good values in bolometer */
dim_t nbolo; /* Number of bolometers */
dim_t ndata; /* Number of data points */
dim_t nf=0; /* Number of frequencies */
size_t ngood; /* Number of good samples */
dim_t ntslice; /* Number of time slices */
double p_low; /* Power at f_low */
double p_white; /* Average power from f_white1 to f_white2 */
smfData *pow=NULL; /* Pointer to power spectrum data */
smf_qual_t *qua=NULL; /* Pointer to quality component */
double steptime=1; /* Length of a sample in seconds */
size_t tstride; /* time index stride */
if (*status != SAI__OK) return;
/* Check inputs */
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status )) return;
if( !data->hdr ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData has no header", status );
return;
}
/* Obtain dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride, &tstride,
status );
if( *status==SAI__OK ) {
steptime = data->hdr->steptime;
if( steptime < VAL__SMLD ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": FITS header error, STEPTIME must be > 0",
status);
} else {
/* Frequency steps in the FFT */
df = 1. / (steptime * (double) ntslice );
}
}
/* Initialize arrays */
if( whitenoise ) for(i=0; i<nbolo; i++) whitenoise[i] = VAL__BADD;
if( fratio ) for(i=0; i<nbolo; i++) fratio[i] = VAL__BADD;
/* FFT the data and convert to polar power spectral density form */
pow = smf_fft_data( wf, data, NULL, 0, len, status );
smf_convert_bad( wf, pow, status );
smf_fft_cart2pol( wf, pow, 0, 1, status );
{
dim_t fdims[2];
smf_isfft( pow, NULL, NULL, fdims, NULL, NULL, status );
if( *status == SAI__OK ) nf=fdims[0];
}
/* Check for reasonble frequencies, and integer offsets in the array */
i_low = smf_get_findex( f_low, df, nf, status );
i_w1 = smf_get_findex( f_white1, df, nf, status );
i_w2 = smf_get_findex( f_white2, df, nf, status );
/* Get the quality pointer from the smfData so that we can mask known
bad bolometer. */
qua = smf_select_qualpntr( data, NULL, status );
/* The minimum required number of good values in a bolometer. */
mingood = ( gfrac > 0.0 ) ? ntslice*gfrac : 0;
/* Loop over detectors */
for( i=0; (*status==SAI__OK)&&(i<nbolo); i++ )
if( !qua || !(qua[i*bstride]&SMF__Q_BADB) ) {
/* Pointer to start of power spectrum */
base = pow->pntr[0];
base += nf*i;
/* Smooth the power spectrum */
smf_boxcar1D( base, nf, 1, window, NULL, 0, 1, NULL, status );
/* Measure the power */
if( *status == SAI__OK ) {
p_low = base[i_low];
smf_stats1D( base+i_w1, 1, i_w2-i_w1+1, NULL, 0, 0, &p_white, NULL, NULL,
&ngood, status );
/* It's OK if bad status was generated as long as a mean was calculated */
if( *status==SMF__INSMP ) {
errAnnul( status );
/* if we had no good data there was probably a problem with SMF__Q_BADB
so we simply go to the next bolometer */
if (ngood == 0) continue;
}
/* Count the number of initially good values for the current
bolometer. */
if( (*status==SAI__OK) && qua ) {
ngood = 0;
for( j=0; j<ntslice; j++ ) {
if( qua[i*bstride + j*tstride] == 0 ) ngood++;
}
/* Set bolometer to bad if no power detected, or the number of good
values is too low. */
if( (p_low <= 0) || (p_white <= 0) || (ngood < mingood) ) {
for( j=0; j<ntslice; j++ ) {
qua[i*bstride + j*tstride] |= SMF__Q_BADB;
}
}
}
}
if( (*status==SAI__OK) && (!qua || !(qua[i*bstride]&SMF__Q_BADB)) ) {
/* Power ratio requested */
if ( fratio ) {
fratio[i] = p_low/p_white;
}
/* Store values */
if( whitenoise ) {
/* Integrate the PSD by multiplying the average white noise
level by total number of samples and the frequency spacing:
this calculates the time-domain variance (in 200 Hz SCUBA-2
samples for example) assuming this level holds at all
frequencies. */
whitenoise[i] = p_white * ntslice * df;
/* If NEP set, scale this to variance in a 1-second average by
dividing by the sampling frequency (equivalent to
multiplying by sample length). */
if( nep ) {
whitenoise[i] *= steptime;
}
}
}
}
/* Clean up if the caller does not want to take over the power spectrum */
if( pow ) {
if (fftpow) {
*fftpow = pow;
} else {
smf_close_file( &pow, status );
}
}
}
void smf_fillgaps( ThrWorkForce *wf, smfData *data,
smf_qual_t mask, int *status ) {
/* Local Variables */
const gsl_rng_type *type; /* GSL random number generator type */
dim_t bpt; /* Number of bolos per thread */
dim_t i; /* Bolometer index */
dim_t nbolo; /* Number of bolos */
dim_t ntslice; /* Number of time slices */
double *dat=NULL; /* Pointer to bolo data */
int fillpad; /* Fill PAD samples? */
size_t bstride; /* bolo stride */
size_t pend; /* Last non-PAD sample */
size_t pstart; /* First non-PAD sample */
size_t tstride; /* time slice stride */
smfFillGapsData *job_data; /* Structures holding data for worker threads */
smfFillGapsData *pdata; /* Pointer to data for next worker thread */
smf_qual_t *qua=NULL; /* Pointer to quality array */
/* Main routine */
if (*status != SAI__OK) return;
/* Check we have double precision data floating point data. */
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status )) return;
/* Pointers to data and quality */
dat = data->pntr[0];
qua = smf_select_qualpntr( data, NULL, status );
if( !qua ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No valid QUALITY array was provided", status );
return;
}
if( !dat ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData does not contain a DATA component",status);
return;
}
/* obtain data dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &bstride, &tstride,
status );
/* Determine how many bolometers to process in each thread, and create
the structures used to pass data to the threads. */
if( wf ) {
bpt = nbolo/wf->nworker;
if( wf->nworker*bpt < nbolo ) bpt++;
job_data = astMalloc( sizeof( smfFillGapsData )*wf->nworker );
} else {
bpt = nbolo;
job_data = astMalloc( sizeof( smfFillGapsData ) );
}
/* Find the indices of the first and last non-PAD sample. */
smf_get_goodrange( qua, ntslice, tstride, SMF__Q_PAD, &pstart, &pend,
status );
/* Report an error if it is too short. */
if( pend - pstart <= 2*BOX && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": length of data (%d samples) is too small "
"to fill gaps. Must have at least %d samples per bolometer.",
status, (int) ( pend - pstart ), 2*BOX + 1 );
}
/* If the supplied "mask" value includes SMF__Q_PAD, then we will be
replacing the zero-padded region at the start and end of each time series
with artificial noisey data that connects the first and last data values
smoothly. Remove SMF__Q_PAD from the mask. */
if( mask & SMF__Q_PAD ) {
mask &= ~SMF__Q_PAD;
fillpad = 1;
} else {
fillpad = 0;
}
/* Get the default GSL randim number generator type. A separate random
number generator is used for each worker thread so that the gap filling
process does not depend on the the order in which threads are
executed. */
type = gsl_rng_default;
/* Begin a job context. */
thrBeginJobContext( wf, status );
/* Loop over bolometer in groups of "bpt". */
pdata = job_data;
for( i = 0; i < nbolo; i += bpt, pdata++ ) {
/* Store information for this group in the next smfFillGapsData
structure. */
pdata->ntslice = ntslice;
pdata->dat = dat;
pdata->r = gsl_rng_alloc( type );
pdata->b1 = i;
pdata->b2 = i + bpt - 1;
pdata->pend = pend;
pdata->fillpad = fillpad;
pdata->pstart = pstart;
if( pdata->b2 >= nbolo ) pdata->b2 = nbolo - 1;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->qua = qua;
pdata->mask = mask;
/* Submit a job to the workforce to process this group of bolometers. */
(void) thrAddJob( wf, 0, pdata, smfFillGapsParallel, 0, NULL, status );
}
/* Wait until all jobs in the current job context have completed, and
then end the job context. */
thrWait( wf, status );
thrEndJobContext( wf, status );
/* Free resources. */
if( job_data ) {
pdata = job_data;
for( i = 0; i < nbolo; i += bpt, pdata++ ) {
if( pdata->r ) gsl_rng_free( pdata->r );
}
job_data = astFree( job_data );
}
}
void smf_flag_spikes( ThrWorkForce *wf, smfData *data, smf_qual_t mask,
double thresh, size_t box, size_t *nflagged,
int *status ){
/* Local Variables */
int i; /* Loop counter */
smfFlagSpikesData *job_data=NULL;/* Array of job data for each thread */
dim_t nbolo; /* Number of bolometers */
dim_t ntime; /* Number of time-slices */
double *dat = NULL; /* Pointer to bolo data */
smfFlagSpikesData *pdata=NULL;/* Pointer to job data */
int nw; /* Number of worker threads */
size_t bstride; /* Vector stride between bolometer samples */
size_t nflag; /* Number of samples flagged */
size_t tstride; /* Vector stride between time samples */
smf_qual_t *qua = NULL; /* Pointer to quality flags */
size_t step; /* step size for dividing up work */
int njobs=0; /* Number of jobs to be processed */
/* Check inherited status. Also return immediately if no spike flagging
is to be performed. */
if( *status != SAI__OK || thresh == 0.0 ) return;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Check we have double precision data. */
smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status );
/* Get a pointer to the quality array to use. */
qua = smf_select_qualpntr( data, NULL, status );
/* Report an error if we have no quality array. */
if( !qua && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": No valid QUALITY array was provided", status );
}
/* Get a pointer to the data array to use. Report an error if we have
no data array. */
dat = data->pntr[0];
if( !dat && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": smfData does not contain a DATA component",
status);
}
/* Check the supplied thresh value is valid. */
if( thresh <= 0 && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSetd( "THRESH", thresh );
errRep( " ", FUNC_NAME ": Can't find spikes: thresh=^THRESH, must be > 0",
status);
}
/* Check the supplied box value is valid. */
if( box <= 2 && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "BOX", box );
errRep( " ", FUNC_NAME ": Can't find spikes: box=^BOX, must be > 2",
status);
}
/* Obtain data dimensions, and the stride between adjacent elements on
each axis (bolometer and time). Use the existing data order to avoid
the cost of re-ordering. */
smf_get_dims( data, NULL, NULL, &nbolo, &ntime, NULL, &bstride, &tstride,
status );
/* Check we have room for at least 3 boxes along the time axis. */
if( 3*box > ntime && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "BOX", box );
msgSeti( "MAX", ntime/3 - 1 );
errRep( " ", FUNC_NAME ": Can't find spikes: box=^BOX is too large, "
" must be < ^MAX.", status);
}
/* Set up the job data */
if( nw > (int) nbolo ) {
step = 1;
} else {
step = nbolo/nw;
if( !step ) {
step = 1;
}
}
job_data = astCalloc( nw, sizeof(*job_data) );
for( i=0; (*status==SAI__OK)&&i<nw; i++ ) {
pdata = job_data + i;
pdata->b1 = i*step;
pdata->b2 = (i+1)*step-1;
/* if b1 is greater than the number of bolometers, we've run out of jobs... */
if( pdata->b1 >= nbolo ) {
break;
}
/* increase the jobs counter */
njobs++;
/* Ensure that the last thread picks up any left-over bolometers */
if( (i==(nw-1)) && (pdata->b1<(nbolo-1)) ) {
pdata->b2=nbolo-1;
}
pdata->ijob = -1; /* Flag job as ready to start */
pdata->box = box;
pdata->bstride = bstride;
pdata->dat = dat;
pdata->mask = mask;
pdata->thresh = thresh;
pdata->nbolo = nbolo;
pdata->ntime = ntime;
pdata->qua = qua;
pdata->tstride = tstride;
}
/* Submit jobs to find spikes in each block of bolos */
thrBeginJobContext( wf, status );
for( i=0; (*status==SAI__OK)&&i<njobs; i++ ) {
pdata = job_data + i;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfFlagSpikesPar, 0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
thrEndJobContext( wf, status );
/* Count flagged samples from all of the jobs and free resources */
nflag=0;
if( job_data ) {
for( i=0; i<njobs; i++ ) {
pdata = job_data + i;
nflag += pdata->nflag;
}
job_data = astFree( job_data );
}
/* Return the number of flagged samples, if requested */
if( nflagged ) *nflagged = nflag;
}
void smurf_unmakemap( int *status ) {
/* Local Variables */
AstFrameSet *wcsin = NULL; /* WCS Frameset for input cube */
AstMapping *skymap; /* GRID->SkyFrame Mapping from input WCS */
AstSkyFrame *abskyfrm; /* Input SkyFrame (always absolute) */
AstSkyFrame *skyfrm = NULL;/* SkyFrame from the input WCS Frameset */
Grp *igrp1 = NULL; /* Group of input sky files */
Grp *igrp2 = NULL; /* Group of input template files */
Grp *igrpc = NULL; /* Group of input COM files */
Grp *igrpg = NULL; /* Group of input GAI files */
Grp *igrpq = NULL; /* Group of input Q sky files */
Grp *igrpu = NULL; /* Group of input U sky files */
Grp *ogrp = NULL; /* Group containing output file */
HDSLoc *cloc = NULL; /* HDS locator for component ipdata structure */
HDSLoc *iploc = NULL; /* HDS locator for top level ipdata structure */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
char ipdata[ 200 ]; /* Text buffer for IPDATA value */
char pabuf[ 10 ]; /* Text buffer for parameter value */
char subarray[ 5 ]; /* Name of SCUBA-2 subarray (s8a,s8b,etc) */
dim_t iel; /* Index of next element */
dim_t ndata; /* Number of elements in array */
dim_t ntslice; /* Number of time slices in array */
double *ang_data = NULL; /* Pointer to the FP orientation angles */
double *angc_data = NULL; /* Pointer to the instrumental ANGC data */
double *c0_data = NULL; /* Pointer to the instrumental C0 data */
double *gai_data = NULL; /* Pointer to the input GAI map */
double *in_data = NULL; /* Pointer to the input I sky map */
double *inc_data = NULL; /* Pointer to the input COM data */
double *inq_data = NULL; /* Pointer to the input Q sky map */
double *inu_data = NULL; /* Pointer to the input U sky map */
double *outq_data = NULL; /* Pointer to the Q time series data */
double *outu_data = NULL; /* Pointer to the U time series data */
double *p0_data = NULL; /* Pointer to the instrumental P0 data */
double *p1_data = NULL; /* Pointer to the instrumental P1 data */
double *pd; /* Pointer to next element */
double *pq = NULL; /* Pointer to next Q time series value */
double *pu = NULL; /* Pointer to next U time series value */
double *qinst_data = NULL; /* Pointer to the instrumental Q data */
double *uinst_data = NULL; /* Pointer to the instrumental U data */
double amp16; /* Amplitude of 16 Hz signal */
double amp2; /* Amplitude of 2 Hz signal */
double amp4; /* Amplitude of 4 Hz signal */
double angrot; /* Angle from focal plane X axis to fixed analyser */
double paoff; /* WPLATE value corresponding to POL_ANG=0.0 */
double params[ 4 ]; /* astResample parameters */
double phase16; /* Phase of 16 Hz signal */
double phase2; /* Phase of 2 Hz signal */
double phase4; /* Phase of 4 Hz signal */
double sigma; /* Standard deviation of noise to add to output */
int alignsys; /* Align data in the map's system? */
int cdims[ 3 ]; /* Common-mode NDF dimensions */
int dims[ NDF__MXDIM ]; /* NDF dimensions */
int flag; /* Was the group expression flagged? */
int gdims[ 3 ]; /* GAI model NDF dimensions */
int harmonic; /* The requested harmonic */
int ifile; /* Input file index */
int indf; /* Input sky map NDF identifier */
int indfangc; /* IP ANGC values NDF identifier */
int indfc0; /* IP C0 values NDF identifier */
int indfc; /* Input COM NDF identifier */
int indfcs; /* NDF identifier for matching section of COM */
int indfg; /* Input GAI NDF identifier */
int indfin; /* Input template cube NDF identifier */
int indfiq; /* Input instrumental Q NDF */
int indfiu; /* Input instrumental U NDF */
int indfout; /* Output cube NDF identifier */
int indfp0; /* IP P0 values NDF identifier */
int indfp1; /* IP P1 values NDF identifier */
int indfq; /* Input Q map NDF identifier */
int indfu; /* Input U map NDF identifier */
int interp = 0; /* Pixel interpolation method */
int lbndc[ 3 ]; /* Array of lower bounds of COM NDF */
int moving; /* Is the telescope base position changing? */
int ndim; /* Number of pixel axes in NDF */
int ndimc; /* Number of pixel axes in common-mode NDF */
int ndimg; /* Number of pixel axes in GAI NDF */
int nel; /* Number of elements in array */
int nelc; /* Number of elements in COM array */
int nelg; /* Number of elements in GAI array */
int nelqu; /* Number of elements in Q or U array */
int ngood; /* No. of good values in putput cube */
int nparam = 0; /* No. of parameters required for interpolation scheme */
int pasign; /* Indicates sense of POL_ANG value */
int sdim[ 2 ]; /* Array of significant pixel axes */
int slbnd[ 2 ]; /* Array of lower bounds of input map */
int subnd[ 2 ]; /* Array of upper bounds of input map */
int ubndc[ 3 ]; /* Array of upper bounds of COM NDF */
size_t ncom; /* Number of com files */
size_t ngai; /* Number of gai files */
size_t nskymap; /* Number of supplied sky cubes */
size_t outsize; /* Number of files in output group */
size_t size; /* Number of files in input group */
smfData *odata = NULL; /* Pointer to output data struct */
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Begin an AST context */
astBegin;
/* Begin an NDF context. */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get an identifier for the input NDF. We use NDG (via kpg1Rgndf)
instead of calling ndfAssoc directly since NDF/HDS has problems with
file names containing spaces, which NDG does not have. */
kpg1Rgndf( "IN", 1, 1, "", &igrp1, &nskymap, status );
ndgNdfas( igrp1, 1, "READ", &indf, status );
/* Map the data array in the input sky map. */
ndfMap( indf, "DATA", "_DOUBLE", "READ", (void **) &in_data, &nel,
status );
/* Get the WCS FrameSet from the sky map, together with its pixel index
bounds. */
kpg1Asget( indf, 2, 0, 1, 1, sdim, slbnd, subnd, &wcsin, status );
/* Check the current Frame is a SKY frame. */
skyfrm = astGetFrame( wcsin, AST__CURRENT );
if( !astIsASkyFrame( skyfrm ) && *status == SAI__OK ) {
ndfMsg( "N", indf );
*status = SAI__ERROR;
errRep( " ", " Current Frame in ^N is not a SKY Frame.", status );
}
/* Get a copy of the current frame that represents absolute coords rather
than offsets. We assume the target is moving if the map represents
offsets. */
moving = ( *status == SAI__OK &&
!strcmp( astGetC( skyfrm, "SkyRefIs" ), "Origin" ) ) ? 1 : 0;
abskyfrm = astCopy( skyfrm );
astClear( abskyfrm, "SkyRefIs" );
/* If the ALIGNSYS parameter is TRUE then we align the raw data with the
map in the current system of the map, rather than the default ICRS. */
parGet0l( "ALIGNSYS", &alignsys, status );
if( alignsys ) astSetC( abskyfrm, "AlignSystem", astGetC( abskyfrm,
"System" ) );
/* Get the Mapping from the Sky Frame to grid axis in the iput map. */
skymap = astGetMapping( wcsin, AST__CURRENT, AST__BASE );
/* Get the pixel interpolation scheme to use. */
parChoic( "INTERP", "NEAREST", "NEAREST,LINEAR,SINC,"
"SINCSINC,SINCCOS,SINCGAUSS,SOMB,SOMBCOS",
1, pabuf, 10, status );
if( !strcmp( pabuf, "NEAREST" ) ) {
interp = AST__NEAREST;
nparam = 0;
} else if( !strcmp( pabuf, "LINEAR" ) ) {
interp = AST__LINEAR;
nparam = 0;
} else if( !strcmp( pabuf, "SINC" ) ) {
interp = AST__SINC;
nparam = 1;
} else if( !strcmp( pabuf, "SINCSINC" ) ) {
interp = AST__SINCSINC;
nparam = 2;
} else if( !strcmp( pabuf, "SINCCOS" ) ) {
interp = AST__SINCCOS;
nparam = 2;
} else if( !strcmp( pabuf, "SINCGAUSS" ) ) {
interp = AST__SINCGAUSS;
nparam = 2;
} else if( !strcmp( pabuf, "SOMB" ) ) {
interp = AST__SOMB;
nparam = 1;
} else if( !strcmp( pabuf, "SOMBCOS" ) ) {
interp = AST__SOMBCOS;
nparam = 2;
} else if( *status == SAI__OK ) {
nparam = 0;
*status = SAI__ERROR;
msgSetc( "V", pabuf );
errRep( "", "Support not available for INTERP = ^V (programming "
"error)", status );
}
/* Get an additional parameter vector if required. */
if( nparam > 0 ) parExacd( "PARAMS", nparam, params, status );
/* Get a group of reference time series files to use as templates for
the output time series files.*/
ndgAssoc( "REF", 1, &igrp2, &size, &flag, status );
/* Get output file(s) */
kpg1Wgndf( "OUT", igrp2, size, size, "More output files required...",
&ogrp, &outsize, status );
/* Get he noise level to add to the output data. */
parGet0d( "SIGMA", &sigma, status );
/* Get any Q and U input maps. */
if( *status == SAI__OK ) {
kpg1Rgndf( "QIN", 1, 1, "", &igrpq, &nskymap, status );
ndgNdfas( igrpq, 1, "READ", &indfq, status );
ndfMap( indfq, "DATA", "_DOUBLE", "READ", (void **) &inq_data, &nelqu,
status );
if( nelqu != nel && *status == SAI__OK ) {
ndfMsg( "Q", indfq );
*status = SAI__ERROR;
errRep( "", "Q image '^Q' is not the same size as the I image.",
status );
}
kpg1Rgndf( "UIN", 1, 1, "", &igrpu, &nskymap, status );
ndgNdfas( igrpu, 1, "READ", &indfu, status );
ndfMap( indfu, "DATA", "_DOUBLE", "READ", (void **) &inu_data, &nelqu,
status );
if( nelqu != nel && *status == SAI__OK ) {
ndfMsg( "U", indfu );
*status = SAI__ERROR;
errRep( "", "U image '^U' is not the same size as the I image.",
status );
}
if( *status == PAR__NULL ) {
ndfAnnul( &indfq, status );
ndfAnnul( &indfu, status );
inq_data = NULL;
inu_data = NULL;
errAnnul( status );
} else {
parGet0d( "ANGROT", &angrot, status );
parGet0d( "PAOFF", &paoff, status );
parGet0l( "PASIGN", &pasign, status );
}
}
/* Get any common-mode files. */
if( *status == SAI__OK ) {
kpg1Rgndf( "COM", size, size, "", &igrpc, &ncom, status );
if( *status == PAR__NULL ) {
errAnnul( status );
ncom = 0;
}
}
/* Get any GAI files. */
if( *status == SAI__OK ) {
kpg1Rgndf( "GAI", size, size, "", &igrpg, &ngai, status );
if( *status == PAR__NULL ) {
errAnnul( status );
ngai = 0;
}
}
/* Get any instrumental polarisation files. */
if( *status == SAI__OK ) {
/* First see if the user wants to use the "INSTQ/INSTU" scheme for
specifying instrumental polarisation. */
ndfAssoc( "INSTQ", "Read", &indfiq, status );
ndfAssoc( "INSTU", "Read", &indfiu, status );
if( *status == PAR__NULL ) {
ndfAnnul( &indfiq, status );
ndfAnnul( &indfiu, status );
errAnnul( status );
} else {
msgOut( " ", "Using user-defined IP model", status );
ndfDim( indfiq, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfiq );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfiq, "DATA", "_DOUBLE", "READ", (void **) &qinst_data,
&nel, status );
}
ndfDim( indfiu, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfiu );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfiu, "DATA", "_DOUBLE", "READ", (void **) &uinst_data,
&nel, status );
}
}
/* If not, see if the user wants to use the Johnstone/Kennedy instrumental
polarisation model. The IPDATA parameter gives the path to an HDS
container file contining NDFs holding the required IP data for all
subarrays. */
if( !qinst_data ) {
parGet0c( "IPDATA", ipdata, sizeof(ipdata), status );
if( *status == PAR__NULL ) {
errAnnul( status );
} else {
msgOutf( " ", "Using Johnstone/Kennedy IP model in %s",
status, ipdata );
hdsOpen( ipdata, "READ", &iploc, status );
}
}
}
/* Loop round all the template time series files. */
for( ifile = 1; ifile <= (int) size && *status == SAI__OK; ifile++ ) {
/* Start a new NDF context. */
ndfBegin();
/* Create the output NDF by propagating everything from the input, except
for quality and variance. */
ndgNdfas( igrp2, ifile, "READ", &indfin, status );
ndfMsg( "FILE", indfin );
msgSeti( "THISFILE", ifile );
msgSeti( "NUMFILES", size );
msgOutif( MSG__NORM, " ", "Simulating ^THISFILE/^NUMFILES ^FILE",
status );
ndgNdfpr( indfin, "DATA,HISTORY,LABEL,TITLE,WCS,UNITS,EXTENSION(*)",
ogrp, ifile, &indfout, status );
ndfAnnul( &indfin, status );
ndfAnnul( &indfout, status );
/* We now re-open the output NDF and then modify its data values. */
smf_open_file( wf, ogrp, ifile, "UPDATE", 0, &odata, status );
/* Issue a suitable message and abort if anything went wrong. */
if( *status != SAI__OK ) {
errRep( FUNC_NAME, "Could not open input template file.", status );
break;
} else {
if( odata->file == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "No smfFile associated with smfData.",
status );
break;
} else if( odata->hdr == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "No smfHead associated with smfData.",
status );
break;
}
}
/* Check the reference time series contains double precision values. */
smf_dtype_check_fatal( odata, NULL, SMF__DOUBLE, status );
/* Get the total number of data elements, and the number of time slices. */
smf_get_dims( odata, NULL, NULL, NULL, &ntslice, &ndata, NULL,
NULL, status );
/* Get the subarray name */
smf_fits_getS( odata->hdr, "SUBARRAY", subarray, sizeof(subarray),
status );
/* If we are using the Johnstone/Kennedy IP model, open and map the
relevant parameter NDFs within the IPDATA container file. */
if( iploc ) {
datFind( iploc, subarray, &cloc, status );
ndfFind( cloc, "C0", &indfc0, status );
ndfDim( indfc0, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfc0 );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfc0, "DATA", "_DOUBLE", "READ", (void **) &c0_data,
&nel, status );
}
ndfFind( cloc, "P0", &indfp0, status );
ndfDim( indfp0, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfp0 );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfp0, "DATA", "_DOUBLE", "READ", (void **) &p0_data,
&nel, status );
}
ndfFind( cloc, "P1", &indfp1, status );
ndfDim( indfp1, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfp1 );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfp1, "DATA", "_DOUBLE", "READ", (void **) &p1_data,
&nel, status );
}
ndfFind( cloc, "ANGC", &indfangc, status );
ndfDim( indfangc, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfangc );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfangc, "DATA", "_DOUBLE", "READ", (void **) &angc_data,
&nel, status );
}
}
/* Open any COM file. */
if( ncom ) {
ndgNdfas( igrpc, ifile, "READ", &indfc, status );
ndfDim( indfc, 3, cdims, &ndimc, status );
/* Check its dimensions. */
if( *status == SAI__OK ) {
if( ndimc == 1 ) {
if( cdims[ 0 ] < (int) ntslice ) {
*status = SAI__ERROR;
ndfMsg( "C", indfc );
ndfMsg( "R", indfin );
msgSeti( "N", cdims[ 0 ] );
msgSeti( "M", ntslice );
errRep( " ", "Supplied COM file (^C) has ^N time-slices, but "
"the reference NDF (^R) has ^M time-slices.", status );
} else {
ndfBound( indfc, 3, lbndc, ubndc, &ndimc, status );
ubndc[ 0 ] = lbndc[ 0 ] + ntslice - 1;
ndfSect( indfc, 1, lbndc, ubndc, &indfcs, status );
}
} else if( ndimc == 3 ) {
if( cdims[ 0 ] != 1 || cdims[ 1 ] != 1 ) {
*status = SAI__ERROR;
ndfMsg( "C", indfc );
errRep( " ", "Supplied 3D COM file (^C) has bad "
"dimensions for axis 1 and/or 2 (should "
"both be 1 pixel long).", status );
} else if( cdims[ 2 ] < (int) ntslice ) {
*status = SAI__ERROR;
ndfMsg( "C", indfc );
ndfMsg( "R", indfin );
msgSeti( "N", cdims[ 2 ] );
msgSeti( "M", ntslice );
errRep( " ", "Supplied COM file (^C) has ^N time-slices, but "
"the reference NDF (^R) has ^M time-slices.", status );
} else {
ndfBound( indfc, 3, lbndc, ubndc, &ndimc, status );
ubndc[ 2 ] = lbndc[ 2 ] + ntslice - 1;
ndfSect( indfc, 3, lbndc, ubndc, &indfcs, status );
}
} else {
*status = SAI__ERROR;
ndfMsg( "C", indfc );
msgSeti( "N", ndimc );
errRep( " ", "Supplied COM file (^C) has ^N dimensions - "
"must be 3.", status );
}
}
ndfMap( indfcs, "DATA", "_DOUBLE", "READ", (void **) &inc_data,
&nelc, status );
} else {
indfcs = NDF__NOID;
inc_data = NULL;
}
/* Open any GAI files. */
if( ngai ) {
ndgNdfas( igrpg, ifile, "READ", &indfg, status );
ndfDim( indfg, 3, gdims, &ndimg, status );
/* Check its dimensions, and map it if OK. */
if( *status == SAI__OK ) {
if( ndimg != 2 ) {
*status = SAI__ERROR;
ndfMsg( "C", indfg );
msgSeti( "N", ndimg );
errRep( " ", "Supplied GAI file (^C) has ^N dimensions - "
"must be 2.", status );
} else if( gdims[ 0 ] != 32 || gdims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "C", indfg );
errRep( " ", "Supplied GAI file (^C) has has bad "
"dimensions - should be 32x40.", status );
}
}
ndfMap( indfg, "DATA", "_DOUBLE", "READ", (void **) &gai_data,
&nelg, status );
} else {
indfg = NDF__NOID;
gai_data = NULL;
}
/* Fill the output with bad values. */
if( *status == SAI__OK ) {
pd = odata->pntr[ 0 ];
for( iel = 0; iel < ndata; iel++ ) *(pd++) = VAL__BADD;
}
/* Resample the sky map data into the output time series. */
smf_resampmap( wf, odata, abskyfrm, skymap, moving, slbnd, subnd,
interp, params, sigma, in_data, odata->pntr[ 0 ],
NULL, &ngood, status );
/* Add on any COM data. */
smf_addcom( wf, odata, inc_data, status );
/* Issue a wrning if there is no good data in the output cube. */
if( ngood == 0 ) msgOutif( MSG__NORM, " ", " Output contains no "
"good data values.", status );
/* If Q and U maps have been given, allocate room to hold resampled Q and
U values, and fill them with bad values. */
if( inq_data && inu_data ) {
pq = outq_data = astMalloc( ndata*sizeof( *outq_data ) );
pu = outu_data = astMalloc( ndata*sizeof( *outu_data ) );
if( *status == SAI__OK ) {
for( iel = 0; iel < ndata; iel++ ) {
*(pu++) = VAL__BADD;
*(pq++) = VAL__BADD;
}
}
/* Determine the harmonic to use. */
parGet0i( "HARMONIC", &harmonic, status );
/* If producing the normal 8 Hz harmonic, get the amplitude and phase of a
other signals to add onto the 8 Hz signal. */
if( harmonic == 4 ) {
parGet0d( "AMP2", &2, status );
parGet0d( "PHASE2", &phase2, status );
parGet0d( "AMP4", &4, status );
parGet0d( "PHASE4", &phase4, status );
parGet0d( "AMP16", &16, status );
parGet0d( "PHASE16", &phase16, status );
} else {
amp2 = 0.0;
phase2 = 0.0;
amp4 = 0.0;
phase4 = 0.0;
amp16 = 0.0;
phase16 = 0.0;
}
/* Allocate room for an array to hold the angle from the Y pixel axis
in the sky map to the focal plane Y axis, in radians, at each time
slice. Positive rotation is in the same sense as rotation from
focal plane X to focal plane Y. */
ang_data = astMalloc( ntslice*sizeof( *ang_data ) );
/* Resample them both into 3D time series. These Q/U values arw with
respect to the sky image Y axis. */
smf_resampmap( wf, odata, abskyfrm, skymap, moving, slbnd, subnd,
interp, params, sigma, inq_data, outq_data,
ang_data, &ngood, status );
smf_resampmap( wf, odata, abskyfrm, skymap, moving, slbnd, subnd,
interp, params, sigma, inu_data, outu_data,
NULL, &ngood, status );
/* Combine these time series with the main output time series so that the
main output is analysed intensity. */
smf_uncalc_iqu( wf, odata, odata->pntr[ 0 ], outq_data, outu_data,
ang_data, pasign, AST__DD2R*paoff, AST__DD2R*angrot,
amp2, AST__DD2R*phase2, amp4, AST__DD2R*phase4,
amp16, AST__DD2R*phase16, qinst_data, uinst_data,
c0_data, p0_data, p1_data, angc_data, harmonic,
status );
/* Release work space. */
outq_data = astFree( outq_data );
outu_data = astFree( outu_data );
ang_data = astFree( ang_data );
}
/* Factor in any GAI data. */
smf_addgai( wf, odata, gai_data, status );
/* Close the output time series file. */
smf_close_file( wf, &odata, status );
/* Close the IP data container for the current subarray, if it is open. */
if( cloc ) datAnnul( &cloc, status );
/* End the NDF context. */
ndfEnd( status );
}
/* Close any input data file that is still open due to an early exit from
the above loop. */
if( odata != NULL ) {
smf_close_file( wf, &odata, status );
odata = NULL;
}
/* Free remaining resources. */
if( igrp1 != NULL) grpDelet( &igrp1, status);
if( igrp2 != NULL) grpDelet( &igrp2, status);
if( igrpq != NULL) grpDelet( &igrpq, status);
if( igrpu != NULL) grpDelet( &igrpu, status);
if( igrpc != NULL) grpDelet( &igrpc, status);
if( igrpg != NULL) grpDelet( &igrpg, status);
if( ogrp != NULL) grpDelet( &ogrp, status);
if( iploc ) datAnnul( &iploc, status );
/* End the NDF context. */
ndfEnd( status );
/* End the tile's AST context. */
astEnd;
/* Issue a status indication.*/
if( *status == SAI__OK ) {
msgOutif(MSG__VERB," ",TASK_NAME " succeeded, time series written.", status);
} else {
msgOutif(MSG__VERB," ",TASK_NAME " failed.", status);
}
}
void smurf_unmakemap( int *status ) {
/* Local Variables */
AstFrameSet *wcsin = NULL; /* WCS Frameset for input cube */
AstMapping *skymap; /* GRID->SkyFrame Mapping from input WCS */
AstSkyFrame *abskyfrm; /* Input SkyFrame (always absolute) */
AstSkyFrame *skyfrm = NULL;/* SkyFrame from the input WCS Frameset */
Grp *igrp1 = NULL; /* Group of input sky files */
Grp *igrp2 = NULL; /* Group of input template files */
Grp *igrpq = NULL; /* Group of input Q sky files */
Grp *igrpu = NULL; /* Group of input U sky files */
Grp *ogrp = NULL; /* Group containing output file */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
char pabuf[ 10 ]; /* Text buffer for parameter value */
dim_t iel; /* Index of next element */
dim_t ndata; /* Number of elements in array */
dim_t ntslice; /* Number of time slices in array */
double *ang_data = NULL; /* Pointer to the FP orientation angles */
double *in_data = NULL; /* Pointer to the input I sky map */
double *inq_data = NULL; /* Pointer to the input Q sky map */
double *inu_data = NULL; /* Pointer to the input U sky map */
double *outq_data = NULL; /* Pointer to the Q time series data */
double *outu_data = NULL; /* Pointer to the U time series data */
double *pd; /* Pointer to next element */
double *pq = NULL; /* Pointer to next Q time series value */
double *pu = NULL; /* Pointer to next U time series value */
double angrot; /* Angle from focal plane X axis to fixed analyser */
double paoff; /* WPLATE value corresponding to POL_ANG=0.0 */
double params[ 4 ]; /* astResample parameters */
double sigma; /* Standard deviation of noise to add to output */
int alignsys; /* Align data in the map's system? */
int flag; /* Was the group expression flagged? */
int harmonic; /* The requested harmonic */
int ifile; /* Input file index */
int indf; /* Input sky map NDF identifier */
int indfin; /* Input template cube NDF identifier */
int indfout; /* Output cube NDF identifier */
int indfq; /* Input Q map NDF identifier */
int indfu; /* Input U map NDF identifier */
int interp = 0; /* Pixel interpolation method */
int moving; /* Is the telescope base position changing? */
int nel; /* Number of elements in array */
int nelqu; /* Number of elements in Q or U array */
int ngood; /* No. of good values in putput cube */
int nparam = 0; /* No. of parameters required for interpolation scheme */
int pasign; /* Indicates sense of POL_ANG value */
int sdim[ 2 ]; /* Array of significant pixel axes */
int slbnd[ 2 ]; /* Array of lower bounds of input map */
int subnd[ 2 ]; /* Array of upper bounds of input map */
size_t nskymap; /* Number of supplied sky cubes */
size_t outsize; /* Number of files in output group */
size_t size; /* Number of files in input group */
smfData *odata = NULL; /* Pointer to output data struct */
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Begin an AST context */
astBegin;
/* Begin an NDF context. */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get an identifier for the input NDF. We use NDG (via kpg1Rgndf)
instead of calling ndfAssoc directly since NDF/HDS has problems with
file names containing spaces, which NDG does not have. */
kpg1Rgndf( "IN", 1, 1, "", &igrp1, &nskymap, status );
ndgNdfas( igrp1, 1, "READ", &indf, status );
/* Map the data array in the input sky map. */
ndfMap( indf, "DATA", "_DOUBLE", "READ", (void **) &in_data, &nel,
status );
/* Get the WCS FrameSet from the sky map, together with its pixel index
bounds. */
kpg1Asget( indf, 2, 0, 1, 1, sdim, slbnd, subnd, &wcsin, status );
/* Check the current Frame is a SKY frame. */
skyfrm = astGetFrame( wcsin, AST__CURRENT );
if( !astIsASkyFrame( skyfrm ) && *status == SAI__OK ) {
ndfMsg( "N", indf );
*status = SAI__ERROR;
errRep( " ", " Current Frame in ^N is not a SKY Frame.", status );
}
/* Get a copy of the current frame that represents absolute coords rather
than offsets. We assume the target is moving if the map represents
offsets. */
moving = ( *status == SAI__OK &&
!strcmp( astGetC( skyfrm, "SkyRefIs" ), "Origin" ) ) ? 1 : 0;
abskyfrm = astCopy( skyfrm );
astClear( abskyfrm, "SkyRefIs" );
/* If the ALIGNSYS parameter is TRUE then we align the raw data with the
map in the current system of the map, rather than the default ICRS. */
parGet0l( "ALIGNSYS", &alignsys, status );
if( alignsys ) astSetC( abskyfrm, "AlignSystem", astGetC( abskyfrm,
"System" ) );
/* Get the Mapping from the Sky Frame to grid axis in the iput map. */
skymap = astGetMapping( wcsin, AST__CURRENT, AST__BASE );
/* Get the pixel interpolation scheme to use. */
parChoic( "INTERP", "NEAREST", "NEAREST,LINEAR,SINC,"
"SINCSINC,SINCCOS,SINCGAUSS,SOMB,SOMBCOS",
1, pabuf, 10, status );
if( !strcmp( pabuf, "NEAREST" ) ) {
interp = AST__NEAREST;
nparam = 0;
} else if( !strcmp( pabuf, "LINEAR" ) ) {
interp = AST__LINEAR;
nparam = 0;
} else if( !strcmp( pabuf, "SINC" ) ) {
interp = AST__SINC;
nparam = 1;
} else if( !strcmp( pabuf, "SINCSINC" ) ) {
interp = AST__SINCSINC;
nparam = 2;
} else if( !strcmp( pabuf, "SINCCOS" ) ) {
interp = AST__SINCCOS;
nparam = 2;
} else if( !strcmp( pabuf, "SINCGAUSS" ) ) {
interp = AST__SINCGAUSS;
nparam = 2;
} else if( !strcmp( pabuf, "SOMB" ) ) {
interp = AST__SOMB;
nparam = 1;
} else if( !strcmp( pabuf, "SOMBCOS" ) ) {
interp = AST__SOMBCOS;
nparam = 2;
} else if( *status == SAI__OK ) {
nparam = 0;
*status = SAI__ERROR;
msgSetc( "V", pabuf );
errRep( "", "Support not available for INTERP = ^V (programming "
"error)", status );
}
/* Get an additional parameter vector if required. */
if( nparam > 0 ) parExacd( "PARAMS", nparam, params, status );
/* Get a group of reference time series files to use as templates for
the output time series files.*/
ndgAssoc( "REF", 1, &igrp2, &size, &flag, status );
/* Get output file(s) */
kpg1Wgndf( "OUT", igrp2, size, size, "More output files required...",
&ogrp, &outsize, status );
/* Get he noise level to add to the output data. */
parGet0d( "SIGMA", &sigma, status );
/* Get any Q and U input maps. */
if( *status == SAI__OK ) {
kpg1Rgndf( "QIN", 1, 1, "", &igrpq, &nskymap, status );
ndgNdfas( igrpq, 1, "READ", &indfq, status );
ndfMap( indfq, "DATA", "_DOUBLE", "READ", (void **) &inq_data, &nelqu,
status );
if( nelqu != nel && *status == SAI__OK ) {
ndfMsg( "Q", indfq );
*status = SAI__ERROR;
errRep( "", "Q image '^Q' is not the same size as the I image.",
status );
}
kpg1Rgndf( "UIN", 1, 1, "", &igrpu, &nskymap, status );
ndgNdfas( igrpu, 1, "READ", &indfu, status );
ndfMap( indfu, "DATA", "_DOUBLE", "READ", (void **) &inu_data, &nelqu,
status );
if( nelqu != nel && *status == SAI__OK ) {
ndfMsg( "U", indfu );
*status = SAI__ERROR;
errRep( "", "U image '^U' is not the same size as the I image.",
status );
}
if( *status == PAR__NULL ) {
ndfAnnul( &indfq, status );
ndfAnnul( &indfu, status );
inq_data = NULL;
inu_data = NULL;
errAnnul( status );
} else {
parGet0d( "ANGROT", &angrot, status );
parGet0d( "PAOFF", &paoff, status );
parGet0l( "PASIGN", &pasign, status );
}
}
/* Loop round all the template time series files. */
for( ifile = 1; ifile <= (int) size && *status == SAI__OK; ifile++ ) {
/* Start a new NDF context. */
ndfBegin();
/* Create the output NDF by propagating everything from the input, except
for quality and variance. */
ndgNdfas( igrp2, ifile, "READ", &indfin, status );
ndfMsg( "FILE", indfin );
msgSeti( "THISFILE", ifile );
msgSeti( "NUMFILES", size );
msgOutif( MSG__NORM, " ", "Simulating ^THISFILE/^NUMFILES ^FILE",
status );
ndgNdfpr( indfin, "DATA,HISTORY,LABEL,TITLE,WCS,UNITS,EXTENSION(*)",
ogrp, ifile, &indfout, status );
ndfAnnul( &indfin, status );
ndfAnnul( &indfout, status );
/* We now re-open the output NDF and then modify its data values. */
smf_open_file( wf, ogrp, ifile, "UPDATE", 0, &odata, status );
/* Issue a suitable message and abort if anything went wrong. */
if( *status != SAI__OK ) {
errRep( FUNC_NAME, "Could not open input template file.", status );
break;
} else {
if( odata->file == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "No smfFile associated with smfData.",
status );
break;
} else if( odata->hdr == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "No smfHead associated with smfData.",
status );
break;
}
}
/* Check the reference time series contains double precision values. */
smf_dtype_check_fatal( odata, NULL, SMF__DOUBLE, status );
/* Get the total number of data elements, and the number of time slices. */
smf_get_dims( odata, NULL, NULL, NULL, &ntslice, &ndata, NULL,
NULL, status );
/* Fill the output with bad values. */
if( *status == SAI__OK ) {
pd = odata->pntr[ 0 ];
for( iel = 0; iel < ndata; iel++ ) *(pd++) = VAL__BADD;
}
/* Resample the sky map data into the output time series. */
smf_resampmap( wf, odata, abskyfrm, skymap, moving, slbnd, subnd,
interp, params, sigma, in_data, odata->pntr[ 0 ],
NULL, &ngood, status );
/* Issue a wrning if there is no good data in the output cube. */
if( ngood == 0 ) msgOutif( MSG__NORM, " ", " Output contains no "
"good data values.", status );
/* If Q and U maps have been given, allocate room to hold resampled Q and
U values, and fill them with bad values. */
if( inq_data && inu_data ) {
pq = outq_data = astMalloc( ndata*sizeof( *outq_data ) );
pu = outu_data = astMalloc( ndata*sizeof( *outu_data ) );
if( *status == SAI__OK ) {
for( iel = 0; iel < ndata; iel++ ) {
*(pu++) = VAL__BADD;
*(pq++) = VAL__BADD;
}
}
/* Determine the harmonic to use. */
parGet0i( "HARMONIC", &harmonic, status );
/* Allocate room for an array to hold the anti-clockwise angle from the
focal plane Y axis to the Y pixel axis in the reference map, at each
time slice. */
ang_data = astMalloc( ntslice*sizeof( *ang_data ) );
/* Resample them both into 3D time series. */
smf_resampmap( wf, odata, abskyfrm, skymap, moving, slbnd, subnd,
interp, params, sigma, inq_data, outq_data,
ang_data, &ngood, status );
smf_resampmap( wf, odata, abskyfrm, skymap, moving, slbnd, subnd,
interp, params, sigma, inu_data, outu_data,
NULL, &ngood, status );
/* Combine these time series with the main output time series so that the
main output is analysed intensity. */
smf_uncalc_iqu( wf, odata, odata->pntr[ 0 ], outq_data, outu_data,
ang_data, pasign, AST__DD2R*paoff, AST__DD2R*angrot,
harmonic, status );
/* Release work space. */
outq_data = astFree( outq_data );
outu_data = astFree( outu_data );
ang_data = astFree( ang_data );
}
/* Close the output time series file. */
smf_close_file( wf, &odata, status );
/* End the NDF context. */
ndfEnd( status );
}
/* Close any input data file that is still open due to an early exit from
the above loop. */
if( odata != NULL ) {
smf_close_file( wf, &odata, status );
odata = NULL;
}
/* Free remaining resources. */
if( igrp1 != NULL) grpDelet( &igrp1, status);
if( igrp2 != NULL) grpDelet( &igrp2, status);
if( igrpq != NULL) grpDelet( &igrpq, status);
if( igrpu != NULL) grpDelet( &igrpu, status);
if( ogrp != NULL) grpDelet( &ogrp, status);
/* End the NDF context. */
ndfEnd( status );
/* End the tile's AST context. */
astEnd;
/* Issue a status indication.*/
if( *status == SAI__OK ) {
msgOutif(MSG__VERB," ",TASK_NAME " succeeded, time series written.", status);
} else {
msgOutif(MSG__VERB," ",TASK_NAME " failed.", status);
}
}
int smf_correct_extinction(ThrWorkForce *wf, smfData *data, smf_tausrc *thetausrc, smf_extmeth method,
AstKeyMap * extpars, double tau, double *allextcorr,
double **wvmtaucache, int *status) {
/* Local variables */
int allquick = 0; /* Is the extinction for all bolometers the same? */
double amstart = VAL__BADD; /* Airmass at start */
double amend = VAL__BADD; /* Airmass at end */
double elstart = VAL__BADD; /* Elevation at start (radians) */
double elend = VAL__BADD;/* Elevation at end (radians) */
smfHead *hdr = NULL; /* Pointer to full header struct */
double *indata = NULL; /* Pointer to data array */
int isTordered; /* data order of input data */
int lbnd[2]; /* Lower bound */
size_t ndims; /* Number of dimensions in input data */
dim_t nframes = 0; /* Number of frames */
dim_t npts = 0; /* Number of data points */
dim_t nx = 0; /* # pixels in x-direction */
dim_t ny = 0; /* # pixels in y-direction */
smf_tausrc tausrc; /* Local copy of tausrc value */
int ubnd[2]; /* Upper bound */
double *vardata = NULL; /* Pointer to variance array */
double * wvmtau = NULL; /* WVM tau (smoothed or not) for these data */
int nw; /* Number of worker threads */
int iw; /* Thread index */
SmfCorrectExtinctionData *job_data = NULL; /* Array of job descriptions */
SmfCorrectExtinctionData *pdata; /* Pointer to next job description */
size_t framestep; /* Number of frames per thread */
/* Check status */
if (*status != SAI__OK) return allquick;
/* If no correction requested, return */
if( method==SMF__EXTMETH_NONE ) {
msgOutif(MSG__VERB, "", FUNC_NAME ": Extinction method=none, returning",
status );
return allquick;
}
if ( ! thetausrc ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Must supply a thetausrc argument. Possible programming error.",
status );
return allquick;
}
/* Use a local value for tausrc as we update it in this routine. In particular,
CSOFIT becomes WVMRAW and this would be confusing to the caller */
tausrc = *thetausrc;
/* If no opacity monitor specified generate bad status */
if( tausrc==SMF__TAUSRC_NULL ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No source of opacity information could be determined",
status );
return allquick;
}
if( smf_history_check( data, FUNC_NAME, status) ) {
/* If caller not requesting allextcorr fail here */
if( !allextcorr ) {
msgSetc("F", FUNC_NAME);
msgOutif(MSG__VERB," ",
"^F has already been run on these data, returning to caller",
status);
return allquick;
}
}
/* Acquire the data order */
isTordered = data->isTordered;
/* make sure we have a header */
hdr = data->hdr;
if( hdr == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Input data has no header", status);
return allquick;
}
/* Do we have 2-D image data? */
ndims = data->ndims;
if (ndims == 2) {
nframes = 1;
nx = (data->dims)[0];
ny = (data->dims)[1];
npts = nx*ny;
} else {
/* this routine will also check for dimensionality */
smf_get_dims( data, &nx, &ny, &npts, &nframes, NULL, NULL, NULL, status );
}
/* Tell user we're correcting for extinction */
msgOutif(MSG__VERB," ",
"Correcting for extinction.", status);
/* Should check data type for double if not allextcorr case */
if( !allextcorr ) {
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status)) return allquick;
}
/* Check that we're not trying to use the WVM for 2-D data */
if ( ndims == 2 && tausrc == SMF__TAUSRC_WVMRAW ) {
if ( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Method WVMRaw can not be used on 2-D image data", status );
return allquick;
}
} else if (ndims == 2 && tausrc == SMF__TAUSRC_CSOFIT ) {
/* This is CSOTAU mode with the value calculated from the fits. We have to either
calculate the value here based on the FITS headers or we have to ensure that
when this mode triggers we've been given the fallback tau derived in this manner.
Revisit this as the code develops (we do not want to be reading fits multiple times).
*/
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Method CSOFIT not yet supported on 2-D image data", status );
return allquick;
}
} else if (ndims < 2 || ndims > 3) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRepf( FUNC_NAME, "Can not extinction correct data with %zd dimension(s)", status,
ndims );
return allquick;
}
}
/* if we are WVMRAW, CSOFIT or AUTO and we have a cache we should always use it since
we assume it was filled in properly the previous time. */
if (wvmtaucache && *wvmtaucache &&
(tausrc == SMF__TAUSRC_WVMRAW ||
tausrc == SMF__TAUSRC_AUTO ||
tausrc == SMF__TAUSRC_CSOFIT)) {
wvmtau = *wvmtaucache;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using cached high resolution data for extinction correction of ^FILE",
status);
tausrc = SMF__TAUSRC_WVMRAW; /* We are now WVMRAW as we have the data */
/* Assume that we only do not know the provenance if in AUTO mode */
if (tausrc == SMF__TAUSRC_AUTO) *thetausrc = SMF__TAUSRC_CACHED;
}
if (!wvmtau && tausrc == SMF__TAUSRC_WVMRAW) {
size_t ntotaltau = 0;
size_t ngoodtau = 0;
smf_calc_smoothedwvm( wf, NULL, data, extpars, &wvmtau, &ntotaltau,
&ngoodtau, status );
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using WVM mode for extinction correction of ^FILE"
" %.0f %% of WVM data are present", status,
(double)(100.0*(double)ngoodtau/(double)ntotaltau) );
}
if (*status == SAI__OK && tausrc == SMF__TAUSRC_CSOFIT) {
/* Calculate the fit but we can use the same cache that WVM uses */
size_t nframes = 0;
smf_calc_csofit( data, extpars, &wvmtau, &nframes, status );
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__QUIET, "", "Using CSO fits for extinction correction of ^FILE",
status );
/* Rebrand as WVM data from this point on */
tausrc = SMF__TAUSRC_WVMRAW;
}
/* AUTO mode logic */
/*
* Default position is to use WVM data but we have two caveats
*
* 1. Was this observation done during a period where the WVM has been flagged as unreliable?
* 2. If the WVM is nominally okay, do we have sufficient good data during this period?
*
* If the WVM should not be used we fallback to seeing if we have a fit available for this
* night from the CSO data.
*
* If we do not have a reliable WVM or CSO fit then we fallback to using a fixed CSO number
* from the header.
*
* This final fallback position is unfortunate as it is highly likely that this is not a reliable
* number if we have fits for every night of observing (we have no information on whether a missing
* fit indicates the CSO was too unstable to use or whether it means we simply haven't got to it
* yet).
*
*/
/* Check auto mode */
if (tausrc == SMF__TAUSRC_AUTO && *status == SAI__OK) {
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
if (ndims == 2) {
/* have to use CSO mode */
tausrc = SMF__TAUSRC_CSOTAU;
*thetausrc = tausrc;
} else if (ndims == 3) {
/* We have already done the cache test so not needed here */
/* Is the WVM nominally stable for this night? */
if (smf_is_wvm_usable( data->hdr, status ) ) {
/* Calculate the WVM tau data and see if we have enough good data */
size_t ngoodtau = 0;
size_t ntotaltau = 0;
double percentgood = 0.0;
smf_calc_smoothedwvm( wf, NULL, data, extpars, &wvmtau, &ntotaltau,
&ngoodtau, status );
percentgood = 100.0 * ((double)ngoodtau / (double)ntotaltau);
if ( percentgood > 80.0) {
tausrc = SMF__TAUSRC_WVMRAW;
msgOutiff( MSG__VERB, "", "Selecting WVM mode for extinction correction of ^FILE."
" %.0f %% of WVM data are present", status, percentgood );
*thetausrc = tausrc;
} else {
tausrc = SMF__TAUSRC_AUTO; /* keep it AUTO (a no-op but make it clear) */
if (wvmtau) wvmtau = astFree( wvmtau );
}
}
/* at this point we either have WVM data handled or we still think we are AUTO.
Do a CSO FIT check */
if (tausrc == SMF__TAUSRC_AUTO && *status == SAI__OK) {
size_t nframes = 0;
smf_calc_csofit( data, extpars, &wvmtau, &nframes, status );
if (*status == SAI__OK) {
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__QUIET, "", "Using CSO fits for extinction correction of ^FILE",
status );
/* Rebrand as WVM data from this point on */
tausrc = SMF__TAUSRC_WVMRAW;
*thetausrc = SMF__TAUSRC_CSOFIT;
} else if (*status == SMF__BADFIT) {
/* No fit, carry on. */
errAnnul( status );
}
}
/* At this point if we are not WVMRAW then we have a serious issue. It means that
WVM was unusable and we did not have a good CSO fit. We should not continue at this
point as to continue implies that we know what we should do. The user should decide
how much they trust the opacity for the night. There has to be a reason why there
is no CSO fit for the night. */
if (*status == SAI__OK && tausrc != SMF__TAUSRC_WVMRAW) {
*status = SAI__ERROR;
errRep("", "Unable to determine opacity data for this observation. Both WVM and CSO fits failed. Please investigate and if necessary use CSO mode explicitly but proceed with caution.", status );
}
}
}
/* If we have a CSO Tau then convert it to the current filter. This will also
convert bad values to a value derived from the header if appropriate. */
if ( tausrc == SMF__TAUSRC_CSOTAU ) {
tau = smf_cso2filt_tau( hdr, tau, extpars, status );
/* The tau source is now a real tau */
tausrc = SMF__TAUSRC_TAU;
}
/* Find the airmass range for this data */
smf_find_airmass_interval( hdr, &amstart, &amend, &elstart, &elend, status );
if (*status == SAI__OK && (amstart == VAL__BADD || amend == VAL__BADD)) {
*status = SAI__ERROR;
errRep( "", "No good airmass values found in JCMTSTATE structure for these data",
status );
}
/* if we are not doing WVM correction but are in adaptive mode we can determine
whether or not we will have to use full or single mode just by looking at the
airmass data. */
if (ndims == 3 && tausrc != SMF__TAUSRC_WVMRAW && method == SMF__EXTMETH_ADAPT) {
/* first and last is a good approximation given that most SCUBA-2 files will only
be a minute duration. */
double refel;
double refam;
/* only need to examine the largest airmass */
if (amstart > amend) {
refam = amstart;
refel = elstart;
} else {
refam = amend;
refel = elend;
}
/* and choose a correction method */
if (is_large_delta_atau( refam, refel, tau, status) ) {
method = SMF__EXTMETH_FULL;
msgOutiff(MSG__DEBUG, " ",
"Adaptive extinction algorithm selected per-bolometer airmass value "
"per time slice (am=%g, tau=%g)", status, refam, tau);
} else {
msgOutiff(MSG__DEBUG, " ",
"Adaptive extinction algorithm selected single airmass value per time slice"
" (am=%g, tau=%g)", status, refam, tau);
method = SMF__EXTMETH_SINGLE;
}
}
/* Assign pointer to input data array if status is good */
if ( *status == SAI__OK ) {
indata = (data->pntr)[0];
vardata = (data->pntr)[1];
}
/* Jump to the cleanup section if status is bad by this point
since we need to free memory */
if (*status != SAI__OK) goto CLEANUP;
/* Array bounds for astTranGrid call */
lbnd[0] = 1;
lbnd[1] = 1;
ubnd[0] = nx;
ubnd[1] = ny;
/* Unlock the AST objects in the smfData so that the worker threads can
lock them. */
smf_lock_data( data, 0, status );
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many frames to process in each worker thread. */
framestep = nframes/nw;
if( framestep == 0 ) {
framestep = 1;
nw = nframes;
}
/* Allocate job data for threads, and store the range of frames to be
processed by each one. Ensure that the last thread picks up any
left-over frames. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->f1 = iw*framestep;
if( iw < nw - 1 ) {
pdata->f2 = pdata->f1 + framestep - 1;
} else {
pdata->f2 = nframes - 1 ;
}
pdata->nframes = nframes;
pdata->npts = npts;
pdata->allextcorr = allextcorr;
pdata->indata = indata;
pdata->tau = tau;
pdata->vardata = vardata;
pdata->wvmtau = wvmtau;
pdata->amstart = amstart;
pdata->amfirst = amstart + ( amend - amstart )*pdata->f1/( nframes - 1 );
pdata->lbnd = lbnd;
pdata->ubnd = ubnd;
pdata->isTordered = isTordered;
pdata->ndims = ndims;
pdata->data = data;
pdata->hdr = hdr;
pdata->method = method;
pdata->tausrc = tausrc;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_correct_extinction, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Record if all time slices used a single air mass. */
allquick = 1;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
if( ! pdata->allquick ) {
allquick = 0;
break;
}
}
/* Free the job data. */
job_data = astFree( job_data );
}
/* Lock the AST objects in the smfData for use by this thread. */
smf_lock_data( data, 1, status );
/* Add history entry if !allextcorr */
if( (*status == SAI__OK) && !allextcorr ) {
smf_history_add( data, FUNC_NAME, status);
}
CLEANUP:
if (wvmtaucache) {
if (!*wvmtaucache) {
*wvmtaucache = wvmtau;
}
} else {
wvmtau = astFree( wvmtau );
}
return allquick;
}
void smf_fit_poly( ThrWorkForce *wf, smfData *data, const size_t order,
int remove, double *poly, int *status) {
/* Local variables */
size_t bstride; /* bolo strides */
int i; /* Loop counter */
smfFitPolyData *job_data=NULL;/* Array of job data for each thread */
dim_t nbolo=0; /* Number of bolometers */
int njobs=0; /* Number of jobs to be processed */
dim_t ntslice = 0; /* Number of time slices */
int nw; /* Number of worker threads */
smfFitPolyData *pdata=NULL; /* Pointer to job data */
const smf_qual_t *qual; /* pointer to the quality array */
size_t step; /* step size for dividing up work */
size_t tstride; /* time strides */
/* Check status */
if (*status != SAI__OK) return;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Should check data type for double */
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status)) return;
if ( smf_history_check( data, FUNC_NAME, status) ) {
msgSetc("F", FUNC_NAME);
msgOutif(MSG__VERB," ",
"^F has already been run on these data, returning to caller",
status);
return;
}
/* Get the dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &bstride,
&tstride, status);
/* Return with error if there is no QUALITY component */
qual = smf_select_cqualpntr( data, NULL, status );
if( !qual && (*status == SAI__OK) ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Data doesn't have a QUALITY component.", status );
return;
}
/* Return with error if order is greater than the number of data
points */
if ( order >= ntslice ) {
if ( *status == SAI__OK) {
msgSeti("O",order);
msgSeti("NF",ntslice);
*status = SAI__ERROR;
errRep( FUNC_NAME, "Requested polynomial order, ^O, greater than or "
"equal to the number of points, ^NF. Unable to fit polynomial.",
status );
}
return;
}
/* Set up the job data */
if( nw > (int) nbolo ) {
step = 1;
} else {
step = nbolo/nw;
if( !step ) {
step = 1;
}
}
job_data = astCalloc( nw, sizeof(*job_data) );
for( i=0; (*status==SAI__OK)&&i<nw; i++ ) {
pdata = job_data + i;
pdata->b1 = i*step;
pdata->b2 = (i+1)*step-1;
/* if b1 is greater than the number of bolometers, we've run out of jobs */
if( pdata->b1 >= nbolo ) {
break;
}
/* increase the jobs counter */
njobs++;
/* Ensure that the last thread picks up any left-over bolometers */
if( (i==(nw-1)) && (pdata->b1<(nbolo-1)) ) {
pdata->b2=nbolo-1;
}
pdata->ijob = -1; /* Flag job as ready to start */
pdata->bstride = bstride;
pdata->indata = data->pntr[0];
pdata->isTordered = data->isTordered;
pdata->nbolo = nbolo;
pdata->ntslice = ntslice;
pdata->order = order;
pdata->poly = poly;
pdata->qual = qual;
pdata->remove = remove;
pdata->tstride = tstride;
}
/* Submit jobs to fit polynomial baselines to block of bolos */
thrBeginJobContext( wf, status );
for( i=0; (*status==SAI__OK)&&i<njobs; i++ ) {
pdata = job_data + i;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfFitPolyPar, 0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
thrEndJobContext( wf, status );
/* Free local resources. */
job_data = astFree( job_data );
}
int smf_correct_extinction(ThrWorkForce *wf, smfData *data, smf_tausrc tausrc, smf_extmeth method,
AstKeyMap * extpars, double tau, double *allextcorr,
double **wvmtaucache, int *status) {
/* Local variables */
int allquick = 0; /* Is the extinction for all bolometers the same? */
double amstart = VAL__BADD; /* Airmass at start */
double amend = VAL__BADD; /* Airmass at end */
double elstart = VAL__BADD; /* Elevation at start (radians) */
double elend = VAL__BADD;/* Elevation at end (radians) */
smfHead *hdr = NULL; /* Pointer to full header struct */
double *indata = NULL; /* Pointer to data array */
int isTordered; /* data order of input data */
int lbnd[2]; /* Lower bound */
size_t ndims; /* Number of dimensions in input data */
dim_t nframes = 0; /* Number of frames */
dim_t npts = 0; /* Number of data points */
dim_t nx = 0; /* # pixels in x-direction */
dim_t ny = 0; /* # pixels in y-direction */
int ubnd[2]; /* Upper bound */
double *vardata = NULL; /* Pointer to variance array */
double * wvmtau = NULL; /* WVM tau (smoothed or not) for these data */
int nw; /* Number of worker threads */
int iw; /* Thread index */
SmfCorrectExtinctionData *job_data = NULL; /* Array of job descriptions */
SmfCorrectExtinctionData *pdata; /* Pointer to next job description */
size_t framestep; /* Number of frames per thread */
/* Check status */
if (*status != SAI__OK) return allquick;
/* If no correction requested, return */
if( method==SMF__EXTMETH_NONE ) {
msgOutif(MSG__VERB, "", FUNC_NAME ": Extinction method=none, returning",
status );
return allquick;
}
/* If no opacity monitor specified generate bad status */
if( tausrc==SMF__TAUSRC_NULL ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No extinction monitor specified",
status );
return allquick;
}
if( smf_history_check( data, FUNC_NAME, status) ) {
/* If caller not requesting allextcorr fail here */
if( !allextcorr ) {
msgSetc("F", FUNC_NAME);
msgOutif(MSG__VERB," ",
"^F has already been run on these data, returning to caller",
status);
return allquick;
}
}
/* Acquire the data order */
isTordered = data->isTordered;
/* make sure we have a header */
hdr = data->hdr;
if( hdr == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Input data has no header", status);
return allquick;
}
/* Do we have 2-D image data? */
ndims = data->ndims;
if (ndims == 2) {
nframes = 1;
nx = (data->dims)[0];
ny = (data->dims)[1];
npts = nx*ny;
} else {
/* this routine will also check for dimensionality */
smf_get_dims( data, &nx, &ny, &npts, &nframes, NULL, NULL, NULL, status );
}
/* Tell user we're correcting for extinction */
msgOutif(MSG__VERB," ",
"Correcting for extinction.", status);
/* Should check data type for double if not allextcorr case */
if( !allextcorr ) {
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status)) return allquick;
}
/* Check that we're not trying to use the WVM for 2-D data */
if ( ndims == 2 && tausrc == SMF__TAUSRC_WVMRAW ) {
if ( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Method WVMRaw can not be used on 2-D image data", status );
return allquick;
}
} else if (ndims < 2 || ndims > 3) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRepf( FUNC_NAME, "Can not extinction correct data with %zd dimension(s)", status,
ndims );
return allquick;
}
}
if (tausrc == SMF__TAUSRC_WVMRAW) {
size_t ntotaltau = 0;
size_t ngoodtau = 0;
/* calculate WVM unless we have external values */
if (wvmtaucache && *wvmtaucache) {
wvmtau = *wvmtaucache;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using cached WVM data for extinction correction of ^FILE",
status);
} else {
smf_calc_smoothedwvm( wf, NULL, data, extpars, &wvmtau, &ntotaltau,
&ngoodtau, status );
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using WVM mode for extinction correction of ^FILE"
" %.0f %% of WVM data are present", status,
(double)(100.0*(double)ngoodtau/(double)ntotaltau) );
}
}
/* Check auto mode */
if (tausrc == SMF__TAUSRC_AUTO && *status == SAI__OK) {
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
if (ndims == 2) {
/* have to use CSO mode */
tausrc = SMF__TAUSRC_CSOTAU;
} else if (ndims == 3) {
/* Calculate the WVM tau data and see if we have enough good data */
size_t ngoodtau = 0;
size_t ntotaltau = 0;
double percentgood = 0.0;
if (wvmtaucache && *wvmtaucache) {
wvmtau = *wvmtaucache;
tausrc = SMF__TAUSRC_WVMRAW;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using cached WVM data for extinction correction of ^FILE",
status );
} else {
smf_calc_smoothedwvm( wf, NULL, data, extpars, &wvmtau, &ntotaltau,
&ngoodtau, status );
percentgood = 100.0 * ((double)ngoodtau / (double)ntotaltau);
if ( percentgood > 80.0) {
tausrc = SMF__TAUSRC_WVMRAW;
msgOutiff( MSG__VERB, "", "Selecting WVM mode for extinction correction of ^FILE."
" %.0f %% of WVM data are present", status, percentgood );
} else {
tausrc = SMF__TAUSRC_CSOTAU;
if (wvmtau) wvmtau = astFree( wvmtau );
}
}
}
if (tausrc == SMF__TAUSRC_CSOTAU) {
msgOutiff( MSG__VERB, "", "Selecting CSO mode for extinction correction of ^FILE", status );
} else if (tausrc == SMF__TAUSRC_WVMRAW) {
/* Dealt with this above */
} else {
/* oops. Fall back position */
tausrc = SMF__TAUSRC_CSOTAU;
msgOutiff( MSG__VERB, "", "Selecting CSO mode as unexpected fallback for extinction correction of ^FILE", status );
}
}
/* If we have a CSO Tau then convert it to the current filter. This will also
convert bad values to a value derived from the header if appropriate. */
if ( tausrc == SMF__TAUSRC_CSOTAU ) {
tau = smf_cso2filt_tau( hdr, tau, extpars, status );
/* The tau source is now a real tau */
tausrc = SMF__TAUSRC_TAU;
}
/* Find the airmass range for this data */
smf_find_airmass_interval( hdr, &amstart, &amend, &elstart, &elend, status );
if (*status == SAI__OK && (amstart == VAL__BADD || amend == VAL__BADD)) {
*status = SAI__ERROR;
errRep( "", "No good airmass values found in JCMTSTATE structure for these data",
status );
}
/* if we are not doing WVM correction but are in adaptive mode we can determine
whether or not we will have to use full or single mode just by looking at the
airmass data. */
if (ndims == 3 && tausrc != SMF__TAUSRC_WVMRAW && method == SMF__EXTMETH_ADAPT) {
/* first and last is a good approximation given that most SCUBA-2 files will only
be a minute duration. */
double refel;
double refam;
/* only need to examine the largest airmass */
if (amstart > amend) {
refam = amstart;
refel = elstart;
} else {
refam = amend;
refel = elend;
}
/* and choose a correction method */
if (is_large_delta_atau( refam, refel, tau, status) ) {
method = SMF__EXTMETH_FULL;
msgOutiff(MSG__DEBUG, " ",
"Adaptive extinction algorithm selected per-bolometer airmass value "
"per time slice (am=%g, tau=%g)", status, refam, tau);
} else {
msgOutiff(MSG__DEBUG, " ",
"Adaptive extinction algorithm selected single airmass value per time slice"
" (am=%g, tau=%g)", status, refam, tau);
method = SMF__EXTMETH_SINGLE;
}
}
/* Assign pointer to input data array if status is good */
if ( *status == SAI__OK ) {
indata = (data->pntr)[0];
vardata = (data->pntr)[1];
}
/* Jump to the cleanup section if status is bad by this point
since we need to free memory */
if (*status != SAI__OK) goto CLEANUP;
/* Array bounds for astTranGrid call */
lbnd[0] = 1;
lbnd[1] = 1;
ubnd[0] = nx;
ubnd[1] = ny;
/* Unlock the AST objects in the smfData so that the worker threads can
lock them. */
smf_lock_data( data, 0, status );
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many frames to process in each worker thread. */
framestep = nframes/nw;
if( framestep == 0 ) {
framestep = 1;
nw = nframes;
}
/* Allocate job data for threads, and store the range of frames to be
processed by each one. Ensure that the last thread picks up any
left-over frames. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->f1 = iw*framestep;
if( iw < nw - 1 ) {
pdata->f2 = pdata->f1 + framestep - 1;
} else {
pdata->f2 = nframes - 1 ;
}
pdata->nframes = nframes;
pdata->npts = npts;
pdata->allextcorr = allextcorr;
pdata->indata = indata;
pdata->tau = tau;
pdata->vardata = vardata;
pdata->wvmtau = wvmtau;
pdata->amstart = amstart;
pdata->amfirst = amstart + ( amend - amstart )*pdata->f1/( nframes - 1 );
pdata->lbnd = lbnd;
pdata->ubnd = ubnd;
pdata->isTordered = isTordered;
pdata->ndims = ndims;
pdata->data = data;
pdata->hdr = hdr;
pdata->method = method;
pdata->tausrc = tausrc;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_correct_extinction, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Record if all time slices used a single air mass. */
allquick = 1;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
if( ! pdata->allquick ) {
allquick = 0;
break;
}
}
/* Free the job data. */
job_data = astFree( job_data );
}
/* Lock the AST objects in the smfData for use by this thread. */
smf_lock_data( data, 1, status );
/* Add history entry if !allextcorr */
if( (*status == SAI__OK) && !allextcorr ) {
smf_history_add( data, FUNC_NAME, status);
}
CLEANUP:
if (wvmtaucache) {
if (!*wvmtaucache) {
*wvmtaucache = wvmtau;
}
} else {
wvmtau = astFree( wvmtau );
}
return allquick;
}
