void smf_reportprogress( int max, int *status ){
/* Local Variables */
static int count = 0.0;
static int maxcount = 1;
static int perc_last = 0;
int perc;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* If a non-zero max is supplied, record it and reset the counter to
zero. Also print a blank line. */
if( max > 0 ) {
maxcount = max;
count = 0.0;
msgOutif( MSG__VERB, "", " ", status );
perc_last = -1;
}
/* Do nothing if we have already displayed "100 %" */
if( perc_last < 100 ) {
/* Calculate the current percentage of the job. This also increments the
count of invocations. */
perc = (int) ( 100.0*( ((float) count++)/((float) maxcount ) ) + 0.5 );
if( perc > 100 ) perc = 100;
/* If the percentage has changed, display the current progress, including
vt100 escape sequences that result in each displayed percentage
over-printing the previous displayed percentage. Set the MSG "STREAM"
tuning parameter non-zero so that the escape characters are left in place. */
if( perc != perc_last ) {
if( perc < 100 ) {
msgTune( "STREAM", 1, status );
msgOutiff( MSG__VERB, "", "%3d %% done[9D[A",
status, perc );
msgTune( "STREAM", 0, status );
} else {
msgOutiff( MSG__VERB, "", "%3d %% done", status, perc );
msgBlankif( MSG__VERB, status );
}
}
/* Record the previous percentage for next time. */
perc_last = perc;
}
}
double smf_raw2current( smfHead *hdr, int *status ) {
/* Local Variables */
double raw2current=VAL__BADD; /* Conversion factor */
double dateobs=VAL__BADD; /* UTC MJD at observation start */
double dateconst=VAL__BADD; /* UTC MJD when constant changes */
AstTimeFrame *tf = NULL; /* time frame for date conversion */
/* Main routine */
if( *status != SAI__OK ) return VAL__BADD;
/* Get the MJD for start of the observation */
smf_find_dateobs( hdr, &dateobs, NULL, status );
/* Convert ISO dates for when the values changed to MJDs for comparison */
tf = astTimeFrame( " " );
astSet( tf, "TimeScale=UTC" );
astSet( tf, "TimeOrigin=%s", "2011-06-04T00:00:00" );
dateconst = astGetD( tf, "TimeOrigin" );
if( (*status==SAI__OK) && (dateobs!=VAL__BADD) && (dateconst!=VAL__BADD) ) {
if( dateobs < dateconst) {
/* Prior to 2011-06-04 we use the following value. According to
Dan 1.52e-13 was the measured factor for mce mode 1
(unfiltered output) on some old prototype array. The 3.3
multiplier corrected the conversion for mce mode 2 (filtered
output). Note that SIMULT is defined in smurf_par.h */
raw2current = SIMULT * 3.3 * 1.52e-13;
} else {
/* Later the MCE low-pass filter was modified to accomodate a
faster sample rate. With the new firmware on s8a the measured
factor for mce mode 1 (unfiltered output) is 1.56e-13 and to
correct for mce mode 2 (filtered output) requires a 1/1.15
multiplier. */
raw2current = SIMULT * 1.56e-13 / 1.15;
}
}
if( (*status==SAI__OK) && (raw2current==VAL__BADD) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": was unable to determine conversion factor",
status );
}
msgOutiff( MSG__DEBUG, "", FUNC_NAME ": calculated a value of %lf", status,
raw2current );
/* Clean up */
tf = astAnnul( tf );
return raw2current;
}
double smf_map_getpixsize( const smfData *data, int *status ) {
double at[3]={0,0,0}; /* Grid coords. where we check the scale */
int naxes; /* Number of axes */
double pixsize=VAL__BADD; /* The pixel size */
double pixscl[3];
if( *status != SAI__OK ) return pixsize;
if( !data ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfData supplied", status );
return pixsize;
}
if( !data->hdr || !data->hdr->wcs ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": no header, or missing WCS", status );
return pixsize;
}
/* Check number of axes in the frameset. It will usually be 3 because
we have a frequency axis of length 1 for normal SMURF maps */
naxes = astGetI( data->hdr->wcs, "naxes" );
if( (naxes < 2) || (naxes > 3) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": Frameset does not appear to corresond to a 2-d map", status );
return pixsize;
}
/* Take the average of the x- and y-pixel spacings in radians at the
centre of the map, and then convert to arcsec */
at[0] = -(data->lbnd[0]-1);
at[1] = -(data->lbnd[1]-1);
kpgPixsc( data->hdr->wcs, at, pixscl, NULL, NULL, 0, status );
if( *status == SAI__OK ) {
pixsize = (pixscl[0] + pixscl[1])/2.;
pixsize *= DR2AS;
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": determined pixel size from WCS at map coordinates (%g,%g) "
"to be %g arcsec", status, at[0], at[1], pixsize );
} else {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": could not determine pixel size from WCS", status );
}
return pixsize;
}
static size_t
smf__addto_sortinfo ( const smfData * indata, smfSortInfo allinfo[], size_t this_index,
size_t counter, const char * type, int *status ) {
smfSortInfo * sortinfo = NULL;
if (*status != SAI__OK) return counter;
/* store the name and time in the struct */
sortinfo = &(allinfo[counter]);
one_strlcpy( sortinfo->name, indata->file->name, sizeof(sortinfo->name),
status );
smf_find_dateobs( indata->hdr, &(sortinfo->sortval), NULL, status );
msgOutiff(MSG__DEBUG, " ", "%s file: %s",status,
type, indata->file->name);
sortinfo->index = this_index;
counter++;
return counter;
}
int smf_history_check( const smfData* data, const char * appl, int *status) {
const char *key; /* History item to check */
int i = 0; /* Loop counter */
int nrec = 0; /* Number of history records */
int retval = 0; /* Return value */
size_t nc; /* Number of characters to compare */
/* Check entry status */
if (*status != SAI__OK) return retval;
/* check that we have a smfData */
if ( data == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME,
"Supplied smfData is a NULL pointer. Possible programming error.",
status);
return retval;
}
/* Check that history even exists */
if( !data->history ) {
return retval;
}
/* Get the number of characters to compare. */
nc = strlen( appl );
nrec = astMapSize( data->history );
if ( nrec != 0 ) {
for ( i=0; i<nrec; i++ ) {
msgOutiff(MSG__DEBUG2, " ", "Checking history item '%s' for '%s'",
status, astMapKey( data->history, i ), appl);
key = astMapKey( data->history, i );
if ( key && strncmp( key, appl, nc ) == 0 ) {
retval = 1;
break;
}
}
}
return retval;
}
void smf_history_add( smfData* data, const char * appl, int * status ) {
smfFile *file = NULL; /* data->file */
int done; /* Flag to denote whether history entry has been added */
AstKeyMap *history; /* AstKeyMap storing history information */
/* Check entry status */
if (*status != SAI__OK) return;
/* Check that we have a smfData */
if ( data == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME,
"Supplied smfData is a NULL pointer. Possible programming error.",
status);
return;
}
/* Now add entry to the history AstKeyMap if not already present */
if ( data->history == NULL ) {
history = astKeyMap(" " );
data->history = history;
astMapPut0I( data->history, appl, 1, " " );
} else if ( !astMapGet0I( data->history, appl, &done ) ) {
astMapPut0I( data->history, appl, 1, " " );
}
/* If we have a file, write the history entry into the file */
file = data->file;
if ( (file !=NULL) && (file->ndfid != NDF__NOID) ) {
smf_history_write( data, status);
}
msgOutiff( MSG__DEBUG2, " ", "Adding history item '%s'", status, appl);
}
/* Private function for carrying out the noisy bolometer masking */
void smf__noisymask( ThrWorkForce *wf, smfData *data, smfArray **noisemaps,
double noisecliphigh, double noisecliplow,
int zeropad, struct timeval *tv1,
struct timeval *tv2, int *status ) {
smfData *noisemap=NULL; /* Individual noisemap */
if( *status != SAI__OK ) return;
msgOutif( MSG__VERB, "", FUNC_NAME ": masking noisy bolometers", status );
smf_mask_noisy( wf, data,
(noisemaps && *noisemaps) ? (&noisemap) : NULL,
noisecliphigh, noisecliplow, 1, zeropad, status );
if( noisemaps && *noisemaps && noisemap ) {
smf_addto_smfArray( *noisemaps, noisemap, status );
}
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s masking noisy",
status, smf_timerupdate(tv1,tv2,status) );
}
/* Main entry */
void smf_jsadicer( int indf, const char *base, int trim, smf_inst_t instrument,
smf_jsaproj_t proj, size_t *ntile, Grp *grp, int *status ){
/* Local Variables: */
AstBox *box;
AstFitsChan *fc;
AstFrame *specfrm = NULL;
AstFrame *tile_frm = NULL;
AstFrameSet *iwcs;
AstFrameSet *tfs = NULL;
AstFrameSet *tile_wcs;
AstMapping *ndf_map = NULL;
AstMapping *p2pmap = NULL;
AstMapping *specmap = NULL;
AstMapping *tile_map = NULL;
AstRegion *region;
Grp *grpt = NULL;
char *path;
char dtype[ NDF__SZFTP + 1 ];
char jsatile_comment[45];
char type[ NDF__SZTYP + 1 ];
const char *dom = NULL;
const char *keyword;
const char *latsys = NULL;
const char *lonsys = NULL;
double *pd;
double dlbnd[3];
double dubnd[3];
double gcen[3];
double lbnd_in[3];
double lbnd_out[3];
double ubnd_in[3];
double ubnd_out[3];
float *pf;
int *created_tiles = NULL;
int *tiles;
int axlat;
int axlon;
int axspec;
int bbox[ 6 ];
int i;
int ifrm;
int igrid;
int indfo;
int indfs;
int indfx;
int inperm[3];
int ipixel;
int ishpx;
int isxph;
int itile;
int ix;
int iy;
int iz;
int junk;
int latax = -1;
int lbnd[3];
int lbnd_tile[ 3 ];
int lbndx[ NDF__MXDIM ];
int lonax = -1;
int nbase;
int ndim;
int ndimx;
int nfrm;
int nsig;
int ntiles;
int olbnd[ 3 ];
int oubnd[ 3 ];
int outperm[ 3 ];
int place;
int qual;
int tile_index;
int tile_lbnd[2];
int tile_ubnd[2];
int ubnd[3];
int ubnd_tile[ 3 ];
int ubndx[ NDF__MXDIM ];
int var;
size_t iext;
size_t size;
smfJSATiling tiling;
unsigned char *ipq = NULL;
void *ipd = NULL;
void *ipv = NULL;
/* Initialise */
*ntile = 0;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Begin an AST context. */
astBegin;
/* Begin an NDF context. */
ndfBegin();
/* Note the used length of the supplied base string. If it ends with
".sdf", reduce it by 4. */
nbase = astChrLen( base );
if( !strcmp( base + nbase - 4, ".sdf" ) ) nbase -= 4;
/* Allocate a buffer large enough to hold the full path for an output NDF. */
path = astMalloc( nbase + 25 );
/* Get the WCS from the NDF. */
kpg1Gtwcs( indf, &iwcs, status );
/* Note if the NDF projection is HPX or XPH. */
ishpx = astChrMatch( astGetC( iwcs, "Projection" ), "HEALPix" );
isxph = astChrMatch( astGetC( iwcs, "Projection" ), "polar HEALPix" );
/* Report an error if the NDFs projection is neither of these. */
if( !ishpx && !isxph && *status == SAI__OK ) {
ndfMsg( "N", indf );
*status = SAI__ERROR;
errRep( "", "The input NDF (^N) does not appear to be gridded "
"on the JSA all-sky pixel grid.", status );
}
/* Get the bounds of the NDF in pixel indices and the the corresponding
double precision GRID bounds (reduce the size of the grid by a small
amount to avoid problems with tiles that are on the edge of the valid sky
regions - astMapRegion can report an error for such tiles). Also store
the GRID coords of the centre. Also count the number of significant
pixel axes. */
ndfBound( indf, 3, lbnd, ubnd, &ndim, status );
nsig = 0;
for( i = 0; i < ndim; i++ ) {
dlbnd[ i ] = 0.5 + 0.1;
dubnd[ i ] = ubnd[ i ] - lbnd[ i ] + 1.5 - 0.1;
gcen[ i ] = 0.5*( dlbnd[ i ] + dubnd[ i ] );
if( ubnd[ i ] > lbnd[ i ] ) nsig++;
}
/* Find the one-based indices of the RA, Dec and spectral axes in the
current Frame of the NDF. */
axlon = 0;
if( astGetI( iwcs, "IsLonAxis(1)" ) ) {
axlon = 1;
lonsys = astGetC( iwcs, "System(1)" );
} else if( astGetI( iwcs, "IsLonAxis(2)" ) ) {
axlon = 2;
lonsys = astGetC( iwcs, "System(2)" );
} else if( ndim == 3 && astGetI( iwcs, "IsLonAxis(3)" ) ) {
axlon = 3;
lonsys = astGetC( iwcs, "System(3)" );
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "smf_jsadicer: Cannot find the longitude axis in the "
"input NDF.", status );
}
axlat = 0;
if( astGetI( iwcs, "IsLatAxis(1)" ) ) {
axlat = 1;
latsys = astGetC( iwcs, "System(1)" );
} else if( astGetI( iwcs, "IsLatAxis(2)" ) ) {
axlat = 2;
latsys = astGetC( iwcs, "System(2)" );
} else if( ndim == 3 && astGetI( iwcs, "IsLatAxis(3)" ) ) {
axlat = 3;
latsys = astGetC( iwcs, "System(3)" );
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "smf_jsadicer: Cannot find the latitude axis in the "
"input NDF.", status );
}
axspec = 6 - axlon - axlat;
/* Report an error if the spatial axes are not ICRS RA and Dec. */
if( ( lonsys && strcmp( lonsys, "ICRS" ) ) ||
( latsys && strcmp( latsys, "ICRS" ) ) ) {
if( *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", indf );
errRep( "", "smf_jsadicer: The spatial axes in '^N' are not "
"ICRS RA and Dec.", status );
}
}
/* Create a Box describing the region covered by the NDF pixel grid in
GRID coords. */
box = astBox( astGetFrame( iwcs, AST__BASE ), 1, dlbnd, dubnd,
AST__NULL, " " );
/* Map this Box into the current WCS Frame of the NDF. */
region = astMapRegion( box, iwcs, iwcs );
/* If no instrument was specified, we will determine the instrument from
the contexts of the FITS extension. Copy the NDF FITS extension to a
FitsChan. Annul the error if the NDF no FITS extension. */
if( instrument == SMF__INST_NONE && *status == SAI__OK ) {
kpgGtfts( indf, &fc, status );
if( *status == KPG__NOFTS ) {
errAnnul( status );
fc = NULL;
}
} else {
fc = NULL;
}
/* Get the parameters of the required tiling scheme. */
smf_jsainstrument( NULL, fc, instrument, &tiling, status );
/* Get a list of the JSA tiles touched by the supplied NDF. */
tiles = smf_jsatiles_region( region, &tiling, &ntiles, status );
if( ntiles == 0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "smf_jsadicer: No JSA tiles found touching supplied NDF "
"(programming error).", status );
}
/* Does the input NDF have a Variance component? */
ndfState( indf, "Variance", &var, status );
/* Does the input NDF have a Quality component? */
ndfState( indf, "Quality", &qual, status );
/* Decide on the data type to use: _REAL or _DOUBLE. */
ndfMtype( "_REAL,_DOUBLE", indf, indf, "Data", type, sizeof(type), dtype,
sizeof(dtype), status );
/* Tell the user what is happening. */
msgBlank( status );
msgOutf( "", "Dicing %s into JSA tiles:", status,
( nsig == 2 ) ? "map" : "cube" );
/* Loop round all tiles that overlap the supplied NDF. */
for( itile = 0; itile < ntiles && *status == SAI__OK; itile++ ) {
tile_index = tiles[ itile ];
/* Get the spatial pixel bounds of the current tile within the requested
JSA all-sky projection. Also get the (2D) WCS FrameSet for the tile. */
smf_jsatile( tile_index, &tiling, 0, proj, NULL, &tile_wcs, NULL,
tile_lbnd, tile_ubnd, status );
/* Extract the tile pixel->WCS mapping and WCS Frame. We know the indices
of the required Frames because they are hard-wired in smf_jsatile. */
tile_map = astGetMapping( tile_wcs, 3, 2 );
tile_frm = astGetFrame( tile_wcs, 2 );
/* Find the indices of the grid and pixel frames in the input NDF. */
ipixel = -1;
igrid = astGetI( iwcs, "Base" );
nfrm = astGetI( iwcs, "NFrame" );
for( ifrm = 0; ifrm < nfrm; ifrm++ ) {
dom = astGetC( astGetFrame( iwcs, ifrm + 1 ), "Domain" );
if( astChrMatch( dom, "PIXEL" ) ) ipixel = ifrm + 1;
}
/* If required, extract the pixel->spectral mapping and spectral frame in
the input NDF, and add it in parallel with the above tile mapping. */
if( ndim == 3 ) {
astSetI( iwcs, "Base", ipixel );
tfs = atlFrameSetSplit( iwcs, "DSBSPECTRUM SPECTRUM", NULL,
NULL, status );
astSetI( iwcs, "Base", igrid );
if( tfs ) {
specmap = astGetMapping( tfs, AST__BASE, AST__CURRENT );
specfrm = astGetFrame( tfs, AST__CURRENT );
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", indf );
errRep( "", "smf_jsadicer: Cannot find the spectral axis "
"in '^N'.", status );
}
tile_map = (AstMapping *) astCmpMap( tile_map, specmap, 0, " " );
tile_frm = (AstFrame *) astCmpFrame( tile_frm, specfrm, " " );
}
/* Ensure the Epoch is inherited form the input NDF. */
astSetD( tile_frm, "Epoch", astGetD( iwcs, "Epoch" ) );
/* Currently tile axis 1 is RA, axis 2 is Dec and axis 3 (if present) is
spectral. Append a PermMap that re-orders these tile WCS axes to match
those of the NDF. */
outperm[ axlon - 1 ] = 1;
outperm[ axlat - 1 ] = 2;
outperm[ axspec - 1 ] = 3;
inperm[ 0 ] = axlon;
inperm[ 1 ] = axlat;
inperm[ 2 ] = axspec;
tile_map = (AstMapping *) astCmpMap( tile_map, astPermMap( ndim, inperm,
ndim, outperm,
NULL, " " ),
1, " " );
tile_map = astSimplify( tile_map );
/* Also re-order the WCS axes in the tile frame. */
astPermAxes( tile_frm, outperm );
/* We want the zero-based indicies of the input pixel axes corresponding
to ra, dec and spectral. So find the indicies of the pixel axes in the
supplied NDF that are most closely aligned with each WCS axis. */
atlPairAxes( iwcs, NULL, gcen, NULL, inperm, status );
if( inperm[ 0 ] == axlon ) {
lonax = 0;
} else if( inperm[ 1 ] == axlon ) {
lonax = 1;
} else {
lonax = 2;
}
if( inperm[ 0 ] == axlat ) {
latax = 0;
} else if( inperm[ 1 ] == axlat ) {
latax = 1;
} else {
latax = 2;
}
/* To get the mapping from pixel coords in the input NDF to pixel coords
in the output NDF, we invert the above mapping so that it goes from WCS
to pixel, and append it to the end of the NDF pixel->WCS mapping. */
ndf_map = astGetMapping( iwcs, ipixel, AST__CURRENT );
astInvert( tile_map );
p2pmap = (AstMapping *) astCmpMap( ndf_map, tile_map, 1, " " );
p2pmap = astSimplify( p2pmap );
astInvert( tile_map );
/* Show the bounds of the tile within the input NDF. */
msgOutiff( MSG__DEBUG, "", " tile %d has bounds (%d:%d,%d:%d) "
"within the output NDF.", status, tile_index,
tile_lbnd[ 0 ], tile_ubnd[ 0 ], tile_lbnd[ 1 ],
tile_ubnd[ 1 ] );
/* Next job is to find the pixel bounds of the output NDF to create
which will hold data for the current tile. First map the pixel bounds
of the whole tile from output to input. */
lbnd_in[ 0 ] = tile_lbnd[ 0 ] - 0.5;
lbnd_in[ 1 ] = tile_lbnd[ 1 ] - 0.5;
lbnd_in[ 2 ] = lbnd[ 2 ] - 0.5;
ubnd_in[ 0 ] = tile_ubnd[ 0 ] - 0.5;
ubnd_in[ 1 ] = tile_ubnd[ 1 ] - 0.5;
ubnd_in[ 2 ] = ubnd[ 2 ] - 0.5;
astMapBox( p2pmap, lbnd_in, ubnd_in, 0, 1, lbnd_out + 0,
ubnd_out + 0, NULL, NULL );
astMapBox( p2pmap, lbnd_in, ubnd_in, 0, 2, lbnd_out + 1,
ubnd_out + 1, NULL, NULL );
if( ndim == 3 ) astMapBox( p2pmap, lbnd_in, ubnd_in, 0, 3,
lbnd_out + 2, ubnd_out + 2, NULL,
NULL );
lbnd_tile[ 0 ] = floor( lbnd_out[ 0 ] ) + 1;
lbnd_tile[ 1 ] = floor( lbnd_out[ 1 ] ) + 1;
lbnd_tile[ 2 ] = floor( lbnd_out[ 2 ] ) + 1;
ubnd_tile[ 0 ] = floor( ubnd_out[ 0 ] ) + 1;
ubnd_tile[ 1 ] = floor( ubnd_out[ 1 ] ) + 1;
ubnd_tile[ 2 ] = floor( ubnd_out[ 2 ] ) + 1;
/* Show the bounds of the tile within the input NDF. */
msgOutiff( MSG__DEBUG, "", " tile %d has bounds (%d:%d,%d:%d) "
"within the input NDF.", status, tile_index,
lbnd_tile[ 0 ], ubnd_tile[ 0 ], lbnd_tile[ 1 ],
ubnd_tile[ 1 ] );
/* If required, trim the bounds to the extent of the input NDF. */
if( trim ) {
if( lbnd_tile[ 0 ] < lbnd[ 0 ] ) lbnd_tile[ 0 ] = lbnd[ 0 ];
if( lbnd_tile[ 1 ] < lbnd[ 1 ] ) lbnd_tile[ 1 ] = lbnd[ 1 ];
if( lbnd_tile[ 2 ] < lbnd[ 2 ] ) lbnd_tile[ 2 ] = lbnd[ 2 ];
if( ubnd_tile[ 0 ] > ubnd[ 0 ] ) ubnd_tile[ 0 ] = ubnd[ 0 ];
if( ubnd_tile[ 1 ] > ubnd[ 1 ] ) ubnd_tile[ 1 ] = ubnd[ 1 ];
if( ubnd_tile[ 2 ] > ubnd[ 2 ] ) ubnd_tile[ 2 ] = ubnd[ 2 ];
}
/* Check there is some overlap. */
if( lbnd_tile[ 0 ] <= ubnd_tile[ 0 ] &&
lbnd_tile[ 1 ] <= ubnd_tile[ 1 ] &&
lbnd_tile[ 2 ] <= ubnd_tile[ 2 ] ){
/* Now need to check if this section of the input NDF contains any good
values. We also find the bounding box of the good values (within the
input pixel coordinate system). So first obtain and map the required
section of the input NDF. */
ndfSect( indf, ndim, lbnd_tile, ubnd_tile, &indfs, status );
ndfMap( indfs, "Data", type, "Read", &ipd, &junk, status );
if( var ) ndfMap( indfs, "Variance", type, "Read", &ipv, &junk, status );
if( qual ) ndfMap( indfs, "Quality", "_UBYTE", "Read", (void **) &ipq,
&junk, status );
/* Initialise the pixel bounds (within the input NDF) of the box holding
good data values for the current tile. */
bbox[ 0 ] = INT_MAX;
bbox[ 1 ] = INT_MAX;
bbox[ 2 ] = INT_MAX;
bbox[ 3 ] = -INT_MAX;
bbox[ 4 ] = -INT_MAX;
bbox[ 5 ] = -INT_MAX;
/* Loop round all pixels in the section. */
if( *status == SAI__OK ) {
if( !strcmp( type, "_REAL" ) ) {
pf = (float *) ipd;
for( iz = lbnd_tile[ 2 ]; iz <= ubnd_tile[ 2 ]; iz++ ) {
for( iy = lbnd_tile[ 1 ]; iy <= ubnd_tile[ 1 ]; iy++ ) {
for( ix = lbnd_tile[ 0 ]; ix <= ubnd_tile[ 0 ]; ix++ ) {
if( *(pf++) != VAL__BADR ) {
if( ix < bbox[ 0 ] ) bbox[ 0 ] = ix;
if( iy < bbox[ 1 ] ) bbox[ 1 ] = iy;
if( iz < bbox[ 2 ] ) bbox[ 2 ] = iz;
if( ix > bbox[ 3 ] ) bbox[ 3 ] = ix;
if( iy > bbox[ 4 ] ) bbox[ 4 ] = iy;
if( iz > bbox[ 5 ] ) bbox[ 5 ] = iz;
}
}
}
}
} else {
pd = (double *) ipd;
for( iz = lbnd_tile[ 2 ]; iz <= ubnd_tile[ 2 ]; iz++ ) {
for( iy = lbnd_tile[ 1 ]; iy <= ubnd_tile[ 1 ]; iy++ ) {
for( ix = lbnd_tile[ 0 ]; ix <= ubnd_tile[ 0 ]; ix++ ) {
if( *(pd++) != VAL__BADD ) {
if( ix < bbox[ 0 ] ) bbox[ 0 ] = ix;
if( iy < bbox[ 1 ] ) bbox[ 1 ] = iy;
if( iz < bbox[ 2 ] ) bbox[ 2 ] = iz;
if( ix > bbox[ 3 ] ) bbox[ 3 ] = ix;
if( iy > bbox[ 4 ] ) bbox[ 4 ] = iy;
if( iz > bbox[ 5 ] ) bbox[ 5 ] = iz;
}
}
}
}
}
/* Skip empty tiles. */
if( bbox[ 0 ] != INT_MAX ) {
msgOutf( "", " tile %d", status, tile_index );
/* If required, trim the bounds to the edges of the bounding box. */
if( trim >= 2 ) {
olbnd[ 0 ] = bbox[ 0 ];
olbnd[ 1 ] = bbox[ 1 ];
olbnd[ 2 ] = bbox[ 2 ];
oubnd[ 0 ] = bbox[ 3 ];
oubnd[ 1 ] = bbox[ 4 ];
oubnd[ 2 ] = bbox[ 5 ];
} else {
olbnd[ 0 ] = lbnd_tile[ 0 ];
olbnd[ 1 ] = lbnd_tile[ 1 ];
olbnd[ 2 ] = lbnd_tile[ 2 ];
oubnd[ 0 ] = ubnd_tile[ 0 ];
oubnd[ 1 ] = ubnd_tile[ 1 ];
oubnd[ 2 ] = ubnd_tile[ 2 ];
}
/* Modify these pixel bounds so that they refer to the output NDF. */
lbnd_in[ 0 ] = olbnd[ 0 ] - 0.5;
lbnd_in[ 1 ] = olbnd[ 1 ] - 0.5;
lbnd_in[ 2 ] = olbnd[ 2 ] - 0.5;
ubnd_in[ 0 ] = oubnd[ 0 ] - 0.5;
ubnd_in[ 1 ] = oubnd[ 1 ] - 0.5;
ubnd_in[ 2 ] = oubnd[ 2 ] - 0.5;
astMapBox( p2pmap, lbnd_in, ubnd_in, 1, 1, lbnd_out + 0,
ubnd_out + 0, NULL, NULL );
astMapBox( p2pmap, lbnd_in, ubnd_in, 1, 2, lbnd_out + 1,
ubnd_out + 1, NULL, NULL );
if( ndim == 3 ) astMapBox( p2pmap, lbnd_in, ubnd_in, 1, 3,
lbnd_out + 2, ubnd_out + 2, NULL,
NULL );
olbnd[ 0 ] = floor( lbnd_out[ 0 ] ) + 1;
olbnd[ 1 ] = floor( lbnd_out[ 1 ] ) + 1;
olbnd[ 2 ] = floor( lbnd_out[ 2 ] ) + 1;
oubnd[ 0 ] = floor( ubnd_out[ 0 ] ) + 1;
oubnd[ 1 ] = floor( ubnd_out[ 1 ] ) + 1;
oubnd[ 2 ] = floor( ubnd_out[ 2 ] ) + 1;
/* Get the full path to the output NDF for the current tile, and create an
NDF placeholder for it. */
sprintf( path, "%.*s_%d", nbase, base, tile_index );
ndfPlace( NULL, path, &place, status );
/* Create a new output NDF by copying the meta-data from the input NDF
section. */
ndfScopy( indfs, "Units", &place, &indfo, status );
/* Set the pixel bounds of the output NDF to the values found above and copy
the input data for the current tile into it. */
smf1_jsadicer( indfo, olbnd, oubnd, tile_map, tile_frm, p2pmap,
ipd, ipv, ipq, status );
/* Add the name of this output NDF to the group holding the names of the
output NDFs that have actually been created. */
if( grp ) grpPut1( grp, path, 0, status );
/* Add a TILENUM header to the output FITS extension. */
kpgGtfts( indfo, &fc, status );
if( *status == KPG__NOFTS ) {
errAnnul( status );
fc = astFitsChan( NULL, NULL, " " );
/* If the last card is "END", remove it. */
} else {
astSetI( fc, "Card", astGetI( fc, "NCARD" ) );
keyword = astGetC( fc, "CardName" );
if( keyword && !strcmp( keyword, "END" ) ) astDelFits( fc );
}
one_snprintf(jsatile_comment, 45, "JSA all-sky tile index (Nside=%i)",
status, tiling.ntpf);
atlPtfti( fc, "TILENUM", tile_index, jsatile_comment, status );
kpgPtfts( indfo, fc, status );
fc = astAnnul( fc );
/* Now store an STC-S polygon that describes the shortest boundary
enclosing the good data in the output NDF, and store it as an NDF extension. */
kpgPutOutline( indfo, 0.5, 1, status );
/* We now reshape any extension NDFs contained within the output NDF to
have the same spatial bounds as the main NDF (but only for extension
NDFs that originally have the same spatial bounds as the supplied NDF).
Get a group containing paths to all extension NDFs in the output NDF. */
ndgMoreg( indfo, &grpt, &size, status );
/* Loop round each output extension NDF. */
for( iext = 1; iext <= size && *status == SAI__OK; iext++ ) {
ndgNdfas( grpt, iext, "Update", &indfx, status );
/* Get its bounds. */
ndfBound( indfx, NDF__MXDIM, lbndx, ubndx, &ndimx, status );
/* See if this extension NDF has the same bounds on the spatial axes as
the supplied NDF. */
if( ndimx > 1 && lbndx[ lonax ] == lbnd[ lonax ] &&
lbndx[ latax ] == lbnd[ latax ] &&
ubndx[ lonax ] == ubnd[ lonax ] &&
ubndx[ latax ] == ubnd[ latax ] ) {
/* If so, change the bounds of the output extension NDF so that they are
the same as the main NDF on the spatial axes, and map the original
contents of the NDF onto the new pixel grid. */
smf1_jsadicer( indfx, olbnd, oubnd, tile_map, tile_frm, p2pmap,
NULL, NULL, NULL, status );
}
/* Annul the extension NDF identifier. */
ndfAnnul( &indfx, status );
}
/* Free resources associated with the current tile. */
grpDelet( &grpt, status );
ndfAnnul( &indfo, status );
/* Issue warnings about empty tiles. */
} else {
msgOutiff( MSG__VERB, "", " tile %d is empty and so will not be "
"created", status, tile_index );
}
}
/* Free the section of the input NDF. */
ndfAnnul( &indfs, status );
/* Append the index of this tile in the list of tiles to be created. */
created_tiles = astGrow( created_tiles, ++(*ntile),
sizeof( *created_tiles ) );
if( *status == SAI__OK ) created_tiles[ *ntile - 1 ] = tile_index;
} else {
msgOutiff( MSG__DEBUG, "", " Tile %d does not overlap the input "
"NDF after trimming.", status, tile_index );
}
}
msgBlank( status );
/* Write the indicies of the created tiles out to a parameter. */
if( *ntile ) parPut1i( "JSATILELIST", *ntile, created_tiles, status );
/* Free resources. */
created_tiles = astFree( created_tiles );
tiles = astFree( tiles );
path = astFree( path );
/* End the NDF context. */
ndfEnd( status );
/* End the AST context. */
astEnd;
}
void smf_calcmodel_noi( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel, int flags,
int *status) {
/* Local Variables */
dim_t bolostep; /* Number of bolos per thread */
dim_t boxsize; /* No. of time slices in each noise box */
smfData *box = NULL; /* SmfData holding one box of input data */
size_t bstride; /* bolometer stride */
int calcfirst=0; /* Were bolo noises already measured? */
int dclimcorr; /* Min number of correlated steps */
int dcmaxsteps; /* Maximum allowed number of dc jumps */
dim_t dcfitbox; /* Width of box for DC step detection */
double dcthresh; /* Threshold for DC step detection */
dim_t dcsmooth; /* Width of median filter in DC step detection*/
double *din; /* Pointer to next input value */
double *dout; /* Pointer to next output value */
int fillgaps; /* If set perform gap filling */
dim_t i; /* Loop counter */
dim_t ibolo; /* Bolometer index */
int ibox; /* Index of current noise box */
dim_t itime; /* Time slice index */
dim_t idx=0; /* Index within subgroup */
JCMTState *instate=NULL; /* Pointer to input JCMTState */
int iw; /* Thread index */
dim_t j; /* Loop counter */
AstKeyMap *kmap=NULL; /* Local keymap */
size_t mbstride; /* model bolometer stride */
dim_t mntslice; /* Number of model time slices */
size_t mtstride; /* model time slice stride */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
dim_t nbolo; /* Number of bolometers */
int nbox = 0; /* Number of noise boxes */
size_t nchisq; /* Number of data points in chisq calc */
dim_t nelbox; /* Number of data points in a noise box */
dim_t ndata; /* Total number of data points */
size_t nflag; /* Number of new flags */
int nleft; /* Number of samples not in a noise box */
dim_t ntslice; /* Number of time slices */
int nw; /* Number of worker threads */
size_t pend; /* Last non-PAD sample */
size_t pstart; /* First non-PAD sample */
smf_qual_t *qin; /* Pointer to next input quality value */
smf_qual_t *qout; /* Pointer to next output quality value */
smfArray *qua=NULL; /* Pointer to RES at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to RES at chunk */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
dim_t spikebox=0; /* Box size for spike detection */
double spikethresh=0; /* Threshold for spike detection */
size_t tend; /* Last input sample to copy */
size_t tstart; /* First input sample to copy */
size_t tstride; /* time slice stride */
double *var=NULL; /* Sample variance */
size_t xbstride; /* Box bolometer stride */
int zeropad; /* Pad with zeros? */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointer to sub-keymap containing NOI parameters */
astMapGet0A( keymap, "NOI", &kmap );
/* Assert bolo-ordered data */
smf_model_dataOrder( dat, allmodel, chunk, SMF__RES|SMF__QUA, 0, status );
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
model = allmodel[chunk];
/* Obtain parameters for NOI */
/* Data-cleaning parameters */
smf_get_cleanpar( kmap, res->sdata[0], NULL, &dcfitbox, &dcmaxsteps, &dcthresh,
&dcsmooth, &dclimcorr, NULL, &fillgaps, &zeropad, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
&spikethresh, &spikebox, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, status );
/* Did we already calculate the noise on each detector? */
astMapGet0I( kmap, "CALCFIRST", &calcfirst );
/* Initialize chisquared */
dat->chisquared[chunk] = 0;
nchisq = 0;
/* Loop over index in subgrp (subarray) */
for( idx=0; idx<res->ndat; idx++ ) {
/* Get pointers to DATA components */
res_data = (res->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Get the raw data dimensions */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice, &ndata,
&bstride, &tstride, status );
/* NOI model dimensions */
smf_get_dims( model->sdata[idx], NULL, NULL, NULL, &mntslice, NULL,
&mbstride, &mtstride, status );
/* Only estimate the white noise level once at the beginning - the
reason for this is to make measurements of the convergence
easier. We either do it prior to the start of iterations (in which
case the relative weights will be influeced by low-frequency noise,
this is initialized in smf_model_create), or else we calculate
the noise after the first iteration. */
if( (flags & SMF__DIMM_FIRSTITER) && (!calcfirst) ) {
/* There are two forms for the NOI model: one constant noise value
for each bolometer, or "ntslice" noise values for each bolometer.
Handle the first case now. */
if( mntslice == 1 ) {
var = astMalloc( nbolo*sizeof(*var) );
if (var) {
/* Measure the noise from power spectra */
smf_bolonoise( wf, res->sdata[idx], 0, 0.5, SMF__F_WHITELO,
SMF__F_WHITEHI, 0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT,
var, NULL, NULL, status );
for( i=0; i<nbolo; i++ ) if( !(qua_data[i*bstride]&SMF__Q_BADB) ) {
/* Loop over time and store the variance for each sample */
for( j=0; j<mntslice; j++ ) {
model_data[i*mbstride+(j%mntslice)*mtstride] = var[i];
}
}
var = astFree( var );
}
/* If the NOI model is of the second form, the noise is estimated
in boxes of samples lasting "NOI.BOX_SIZE" seconds, and then the
noise level in the box is assigned to all samples in the box. */
} else if( mntslice == ntslice ) {
/* If not already done, get NOI.BOX_SIZE and convert from seconds to
samples. */
if( idx == 0 ) {
boxsize = 0;
smf_get_nsamp( kmap, "BOX_SIZE", res->sdata[0], &boxsize, status );
msgOutf( "", FUNC_NAME ": Calculating a NOI variance for each "
"box of %d samples.", status, (int) boxsize );
/* Find the indices of the first and last non-PAD sample. */
smf_get_goodrange( qua_data, ntslice, tstride, SMF__Q_PAD,
&pstart, &pend, status );
/* How many whole boxes fit into this range? */
nbox = ( pend - pstart + 1 ) / boxsize;
if( nbox == 0 ) nbox = 1;
/* How many samples would be left over at the end if we used this
many boxes? */
nleft = ( pend - pstart + 1 ) - nbox*boxsize;
/* Increase "boxsize" to reduce this number as far as possible.
Any samples that are left over after this increase of boxsize
will not be used when calculating the noise levels in each
bolometer. */
boxsize += nleft/nbox;
/* Create a smfData to hold one box-worth of input data. We
do not need to copy jcmtstate information. */
if( res->sdata[idx]->hdr ) {
instate = res->sdata[idx]->hdr->allState;
res->sdata[idx]->hdr->allState = NULL;
}
box = smf_deepcopy_smfData( res->sdata[idx], 0,
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY,
0, 0, status );
if( instate ) res->sdata[idx]->hdr->allState = instate;
/* Set the length of the time axis to the box size plus padding,
and create empty data and quality arrays for it. */
if( *status == SAI__OK ) {
box->dims[ box->isTordered?2:0 ] = boxsize + pstart + (ntslice - pend - 1);
smf_get_dims( box, NULL, NULL, NULL, NULL, &nelbox,
&xbstride, NULL, status );
box->pntr[0] = astMalloc( sizeof( double )*nelbox );
box->qual = astMalloc( sizeof( smf_qual_t )*nelbox );
/* For every bolometer, flag the start and end of the quality
array as padding, and store zeros in the data array. */
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
dout = ((double *) box->pntr[0]) + xbstride*ibolo;
qout = box->qual + xbstride*ibolo;
for( itime = 0; itime < pstart; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
dout = ((double *) box->pntr[0]) + xbstride*ibolo + pstart + boxsize;;
qout = box->qual + xbstride*ibolo + pstart + boxsize;
for( itime = pend + 1; itime < ntslice; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
}
}
}
/* Work space to hold the variance for each bolometer in a box */
var = astMalloc( nbolo*sizeof(*var) );
if( *status == SAI__OK ) {
/* Index of the first time slice within the input smfData
that is included in the first box. */
tstart = pstart;
/* Loop round each noise box */
for( ibox = 0; ibox < nbox; ibox++ ) {
/* Copy the data and quality values for this box from the
input smfData into "box", leaving room for padding at
both ends of box. Note, data is bolo-ordered so we
can assume that "tstride" is 1. */
din = ((double *)(res->sdata[idx]->pntr[0])) + tstart;
dout = ((double *)(box->pntr[0])) + pstart;
qin = qua_data + tstart;
qout = box->qual + pstart;
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
memcpy( dout, din, boxsize*sizeof( *din ) );
memcpy( qout, qin, boxsize*sizeof( *qin ) );
din += bstride;
dout += xbstride;
qin += bstride;
qout += xbstride;
}
/* Measure the noise from power spectra in the box. */
smf_bolonoise( wf, box, 0, 0.5, SMF__F_WHITELO, SMF__F_WHITEHI,
0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT, var,
NULL, NULL, status );
/* Loop over time and store the variance for each sample in
the NOI model. On the last box, pick up any left over time
slices. */
if( ibox < nbox - 1 ) {
tend = tstart + boxsize - 1;
} else {
tend = pend;
}
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
if( !( qua_data[ ibolo*bstride ] & SMF__Q_BADB ) ) {
dout = model_data + ibolo*bstride + tstart;
for( itime = tstart; itime <= tend; itime++ ) {
*(dout++) = var[ ibolo ];
}
}
}
/* Update the index of the first time slice within the input
smfData that is included in the next box. */
tstart += boxsize;
}
var = astFree( var );
}
/* Report an error if the number of samples for each bolometer in
the NOI model is not 1 or "ntslice". */
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": NOI model has %d samples - should be "
"%d or 1.", status, (int) mntslice, (int) ntslice);
}
}
if( kmap ) {
/* Flag spikes in the residual after first iteration */
if( spikethresh && !(flags&SMF__DIMM_FIRSTITER) ) {
/* Now re-flag */
smf_flag_spikes( wf, res->sdata[idx], SMF__Q_MOD,
spikethresh, spikebox, &nflag, status );
msgOutiff(MSG__VERB," ", " flagged %zu new %lf-sig spikes",
status, nflag, spikethresh );
}
if( dcthresh && dcfitbox ) {
smf_fix_steps( wf, res->sdata[idx], dcthresh, dcsmooth,
dcfitbox, dcmaxsteps, dclimcorr, 1, &nflag, NULL,
NULL, status );
msgOutiff(MSG__VERB, ""," detected %zu bolos with DC steps\n",
status, nflag);
}
}
/* Now calculate contribution to chi^2. This bit takes along time
if there is a lot of data so share the work out amongst the available
worker threads. How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many bolometers to process in each worker thread. */
bolostep = nbolo/nw;
if( bolostep == 0 ) bolostep = 1;
/* Allocate job data for threads, and store the range of bolos to be
processed by each one. Ensure that the last thread picks up any
left-over bolos. */
SmfCalcModelNoiData *job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
SmfCalcModelNoiData *pdata;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->b1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->b2 = pdata->b1 + bolostep - 1;
} else {
pdata->b2 = nbolo - 1 ;
}
/* Store other values common to all jobs. */
pdata->ntslice = ntslice;
pdata->mntslice = mntslice;
pdata->qua_data = qua_data;
pdata->model_data = model_data;
pdata->res_data = res_data;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->mbstride = mbstride;
pdata->mtstride = mtstride;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_calcmodel_noi, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Accumulate the results from all the worker threads. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
dat->chisquared[chunk] += pdata->chisquared;
nchisq += pdata->nchisq;
}
/* Free the job data. */
job_data = astFree( job_data );
}
}
}
/* Free resources */
if( box ) {
box->pntr[0] = astFree( box->pntr[0] );
box->qual = astFree( box->qual );
smf_close_file( &box, status );
}
/* Normalize chisquared for this chunk */
if( (*status == SAI__OK) && (nchisq >0) ) {
dat->chisquared[chunk] /= (double) nchisq;
}
/* Clean Up */
if( kmap ) kmap = astAnnul( kmap );
}
HDSLoc *cupidGaussClumps( int type, int ndim, int *slbnd, int *subnd, void *ipd,
double *ipv, double rms, AstKeyMap *config, int velax,
double beamcorr[ 3 ], int *status ){
/*
*+
* Name:
* cupidGaussClumps
* Purpose:
* Identify clumps of emission within a 1, 2 or 3 dimensional NDF using
* the GAUSSCLUMPS algorithm.
* Language:
* Starlink C
* Synopsis:
* HDSLoc *cupidGaussClumps( int type, int ndim, int *slbnd, int *subnd,
* void *ipd, double *ipv, double rms,
* AstKeyMap *config, int velax,
* double beamcorr[ 3 ], int *status )
* Description:
* This function identifies clumps within a 1, 2 or 3 dimensional data
* array using the GAUSSCLUMPS algorithm, described by Stutski & Gusten
* (1990, ApJ 356, 513). This algorithm proceeds by fitting a Gaussian
* profile to the brightest peak in the data. It then subtracts the fit
* from the data and iterates, fitting a new ellipse to the brightest peak
* in the residuals. This continues until a termination criterion is
* reached. The main termination criterion in this implementation is
* not quite the same as in the Stutski & Gusten paper. They had two main
* termination criteria; 1) the total data sum of the fitted gaussians
* is close to the total data sum of the original data, and 2) the peak
* residual is less than a given multiple of the RMS noise in the data.
* However, 1) is very sensitive to errors in the estimation of the
* background level in the data, and 2) may never be achieved because
* the expected residuals depend not only on the RMS noise in the data
* but also on how accurately gaussian the clumps are, which is not
* known. Therefore, this implementation instead terminates when the
* peak amplitude of the fitted clumps falls below a given fraction of
* the first (i.e. largest) fitted peak.
*
* Two additional termination criteria are used; 1) If there are many
* failed attempts to fit a clump to the peak residual or if 2) a
* specified maximum number of clumps are found, then the process
* terminates early.
* Parameters:
* type
* An integer identifying the data type of the array values pointed to
* by "ipd". Must be either CUPID__DOUBLE or CUPID__FLOAT (defined in
* cupid.h).
* ndim
* The number of dimensions in the data array. Must be 1, 2 or 3.
* slbnd
* Pointer to an array holding the lower pixel index bound of the
* data array on each axis.
* subnd
* Pointer to an array holding the upper pixel index bound of the
* data array on each axis.
* ipd
* Pointer to the data array. The elements should be stored in
* Fortran order. The data type of this array is given by "itype".
* ipv
* Pointer to the input Variance array, or NULL if there is no Variance
* array. The elements should be stored in Fortran order. The data
* type of this array is "double".
* rms
* The default value for the global RMS error in the data array.
* config
* An AST KeyMap holding tuning parameters for the algorithm.
* velax
* The index of the velocity axis in the data array (if any). Only
* used if "ndim" is 3.
* beamcorr
* An array in which is returned the FWHM (in pixels) describing the
* instrumental smoothing along each pixel axis. The clump widths
* stored in the output catalogue are reduced to correct for this
* smoothing.
* status
* Pointer to the inherited status value.
* Notes:
* - The specific form of algorithm used here is informed by a Fortran
* implementation of GaussClumps obtained on 27/9/05 from
* ftp.astro.uni-bonn.de/pub/heith/gaussclumps.
* - Most of the "cupid..." functions used in this file which start
* with a "type" parameter (e.g. cupidFindMax, cupidUpdateArrays, etc) are
* actually not functions at all, but macros defined in cupid.h. These
* macros are wrappers which invoke a type-specific function (e.g.
* cupidFindMaxD, cupidFindMaxF) appropriate to the specific data type
* being used (as indicated by the "type" parameter). Macros are used in
* order to simplify the code here, and thus make the flow of the
* algorithm clearer. The source code for the type-specific functions
* are generated automatically at build time from equivalent files which
* have file type ".cupid". For instance, the files cupidfindmaxD.c and
* cupidfindmaxF.c are generated automatically from cupidfindmax.cupid.
* Also, the rlevant macros definitions and prototypes within cupid.h
* are generated automatically at build time from these ".cupid" files.
* Returned Value:
* A locator for a new HDS object which is an array of NDF structures.
* Each NDF will hold the data values associated with a single clump and
* will be the smallest possible NDF that completely contains the
* corresponding clump. Pixels not in the clump will be set bad. The
* pixel origin is set to the same value as the supplied NDF.
* Copyright:
* Copyright (C) 2009 Science & Technology Facilities Council.
* Copyright (C) 2005 Particle Physics & Astronomy Research Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* TIMJ: Tim Jenness (JAC, Hawaii)
* {enter_new_authors_here}
* History:
* 29-SEP-2005 (DSB):
* Original version.
* 7-MAR-2007 (DSB):
* Use VELORES instead of FWHMBEAM if the data is 1D.
* 7-MAR-2007 (DSB):
* Guard against segvio caused by use of null pointers that are
* returned by astMalloc if an error has occurred.
* 14-JAN-2009 (TIMJ):
* Use MERS for message filtering.
* {enter_further_changes_here}
* Bugs:
* {note_any_bugs_here}
*-
*/
/* Local Variables: */
AstKeyMap *gcconfig; /* Configuration parameters for this algorithm */
char buf[30]; /* File name buffer */
HDSLoc *ret; /* Locator for the returned array of NDFs */
double *peaks; /* Holds the "npeak" most recently fitted peak values */
double chisq; /* Chi-squared value of most recently fitted Gaussian */
double mean_peak; /* The mean of the values within "peaks" */
double mlim; /* Truncation level for Gaussians */
double new_peak; /* The value most recently added to "peaks" */
double nsig; /* No.of standard deviations at which to reject peaks */
double old_peak; /* The oldest value within "peaks" */
double peak_thresh; /* The lower threshold for clump peak values */
double sigma_peak; /* The standard deviation of the values within "peaks" */
double sumdata; /* Sum of the supplied data values */
double sum_peak2; /* Sum of the squares of the values in "peaks" */
double sum_peak; /* Sum of the values in "peaks" */
double sumclumps; /* Sum of the values in all the used clumps so far */
double x[ CUPID__GCNP3 ]; /* Parameters describing new Gaussian clump */
int *dims; /* Pointer to array of array dimensions */
int allbad; /* Are all the residuals bad? */
int area; /* Number of pixels contributing to the clump */
int area_thresh; /* The lower threshold for clump areas */
int excols; /* Are extra output columns required? */
int el; /* Number of elements in array */
size_t i; /* Loop count */
size_t iclump; /* Number of clumps found so far */
int imax; /* Index of element with largest residual */
size_t ipeak; /* Index within "peaks" at which to store the new peak */
int iter; /* Continue finding more clumps? */
double maxbad; /* Max fraction of bad pixels allowed in a clump */
size_t maxclump; /* Max no. of clumps */
int maxskip; /* Max no. of failed fits between good fits */
size_t nclump; /* Number of usable clumps */
int niter; /* Iterations performed so far */
int npad; /* No. of peaks below threshold for temination */
size_t npeak; /* The number of elements in the "peaks" array. */
int nskip; /* No. of failed fits since last good fit */
int area_below; /* Count of consecutive clump areas below the threshold */
int peaks_below; /* Count of consecutive peaks below the threshold */
void *res; /* Pointer to residuals array */
/* Initialise */
ret = NULL;
/* Abort if an error has already occurred. */
if( *status != SAI__OK ) return ret;
/* Initialise things to avoid compiler warnings. */
mean_peak = 0.0;
sigma_peak = 0.0;
new_peak = 0.0;
/* Say which method is being used. */
msgBlankif( MSG__NORM, status );
msgOutif( MSG__NORM, "", "GaussClumps:", status );
msgBlankif( MSG__VERB, status );
/* Get the AST KeyMap holding the configuration parameters for this
algorithm. */
if( !astMapGet0A( config, "GAUSSCLUMPS", (AstObject *) &gcconfig ) ) {
gcconfig = astKeyMap( " " );
astMapPut0A( config, "GAUSSCLUMPS", gcconfig, " " );
}
/* The configuration file can optionally omit the algorithm name. In this
case the "config" KeyMap may contain values which should really be in
the "gcconfig" KeyMap. Add a copy of the "config" KeyMap into "gcconfig"
so that it can be searched for any value which cannot be found in the
"gcconfig" KeyMap. */
astMapPut0A( gcconfig, CUPID__CONFIG, astCopy( config ), NULL );
/* Return the instrumental smoothing FWHMs. For 1D data, we assume the
axis is spectral and so use VELORES instead of FWHMBEAM. */
if( ndim == 1 ) {
beamcorr[ 0 ] = cupidConfigD( gcconfig, "VELORES", 2.0, status );
} else {
beamcorr[ 0 ]= cupidConfigD( gcconfig, "FWHMBEAM", 2.0, status );
beamcorr[ 1 ] = beamcorr[ 0 ];
if( ndim == 3 ) {
beamcorr[ 2 ] = beamcorr[ 0 ];
beamcorr[ velax ]= cupidConfigD( gcconfig, "VELORES", 2.0, status );
}
}
/* See if extra diagnostic info is required. */
excols = cupidConfigI( gcconfig, "EXTRACOLS", 0, status );
/* Get the maximum allowed number of failed fits between succesful fits. */
maxskip = cupidConfigI( gcconfig, "MAXSKIP", 10, status );
/* Get the maximum allowed number of failed fits between succesful fits. */
maxclump = cupidConfigI( gcconfig, "MAXCLUMPS", VAL__MAXI, status );
/* The iterative process ends when "npad" consecutive clumps all had peak
values below "peak_thresh" or all had areas below "area_thresh". */
npad = cupidConfigI( gcconfig, "NPAD", 10, status );
/* Get the RMS noise level to use. */
rms = cupidConfigD( gcconfig, "RMS", rms, status );
/* Find the size of each dimension of the data array, and the total number
of elements in the array. We use the memory management functions of the
AST library since they provide greater security and functionality than
direct use of malloc, etc. */
dims = astMalloc( sizeof( *dims )*(size_t) ndim );
el = 1;
if( dims ) {
for( i = 0; i < (size_t)ndim; i++ ) {
dims[ i ] = subnd[ i ] - slbnd[ i ] + 1;
el *= dims[ i ];
}
}
/* Copy the supplied data array into a work array which will hold the
residuals remaining after subtraction of the fitted Gaussians. The
cupidStore macro is a wrapper around the astStore function. */
res = cupidStore( NULL, ipd, el, type, "cupidGaussClumps" );
if( res ) {
/* Set the lower threshold for clump peaks to a user-specified multiple
of the RMS noise. */
peak_thresh = cupidConfigD( gcconfig, "THRESH", 2.0, status );
/* Set the lower threshold for clump area to a user-specified number of
pixels. */
area_thresh = cupidConfigI( gcconfig, "MINPIX", 3, status );
/* Get the lowest value (normalised to the RMS noise level) at which
model Gaussians should be evaluated. */
mlim = cupidConfigD( gcconfig, "MODELLIM", 0.5, status );
/* Get the max allowed fraction of bad pixels in a clump. */
maxbad = cupidConfigD( gcconfig, "MAXBAD", 0.05, status );
/* Initialise the number of clumps found so far. */
iclump = 0;
/* Indicate that no peaks have been found below the lower threshold for clump
peak values, or below the lower area threshold. */
peaks_below = 0;
area_below = 0;
/* Initialise the variables used to keep track of the mean and standard
deviation of the most recent "npeak" fitted peak values. */
nsig = cupidConfigD( gcconfig, "NSIGMA", 3.0, status );
npeak = cupidConfigI( gcconfig, "NPEAK", 9, status );
ipeak = 0;
sum_peak = 0.0;
sum_peak2 = 0.0;
iter = 1;
niter = 0;
nskip = 0;
sumclumps = 0.0;
sumdata = VAL__BADD;
peaks = astMalloc( sizeof( *peaks )*npeak );
if( peaks ) {
for( i = 0; i < npeak; i++ ) peaks[ i ] = 0.0;
/* Use the setjmp function to define here to be the place to which the
signal handling function will jump when a signal is detected. Zero is
returned on the first invocation of setjmp. If a signal is detected,
a jump is made into setjmp which then returns a positive signal
identifier. */
if( setjmp( CupidGCHere ) ) {
iter = 0;
msgBlankif( MSG__QUIET, status );
msgOutif( MSG__QUIET, "",
"Interupt detected. Clumps found so far will be saved",
status );
msgBlankif( MSG__QUIET, status );
}
}
/* Set up a signal handler for the SIGINT (interupt) signal. If this
signal occurs, the function "cupidGCHandler" will be called. */
signal( SIGINT, cupidGCHandler );
/* Loop round fitting a gaussian to the largest remaining peak in the
residuals array. */
while( iter && *status == SAI__OK ) {
/* Report the iteration number to the user if required. */
++niter;
msgBlankif( MSG__DEBUG1, status );
msgOutiff( MSG__DEBUG1, "", "Iteration %d:", status, niter );
/* Find the 1D vector index of the elements with the largest value in the
residuals array. */
allbad = cupidGCFindMax( type, res, el, &imax, &sumdata, status );
/* Finish iterating if all the residuals are bad, or if too many iterations
have been performed since the last succesfully fitted clump. */
if( allbad ) {
iter = 0;
niter--;
msgBlankif( MSG__DEBUG, status );
msgOutif( MSG__DEBUG1, "",
"There are no good pixels left to be fitted.",
status );
msgBlankif( MSG__DEBUG1, status );
} else if( nskip > maxskip ){
iter = 0;
niter--;
msgBlankif( MSG__DEBUG, status );
msgOutiff( MSG__DEBUG1, "",
"The previous %d fits were unusable.", status, maxskip );
msgBlankif( MSG__DEBUG1, status );
}
/* If not, make an initial guess at the Gaussian clump parameters centred
on the current peak. */
if( iter ) {
cupidGCSetInit( type, res, ipv, ndim, dims, imax, rms, gcconfig,
( niter == 1 ), velax, x, slbnd, status );
/* Find the best fitting parameters, starting from the above initial guess.
This returns a function value of zero if no fit could be performed. */
if( cupidGCFit( type, res, imax, x, &chisq, status ) ) {
/* Skip this fit if we have an estimate of the standard deviation of the
"npeak" most recent clump peak values, and the peak value of the clump
just fitted is a long way (more than NSIGMA standard deviations) from the
peak value of the previously fitted clump. Also skip it if the peak
value is less than the "mlim" value. */
if( ( npeak == 0 || iclump < npeak ||
fabs( x[ 0 ] - new_peak ) < nsig*sigma_peak ) &&
x[ 0 ] > mlim ) {
/* Record the new peak value for use with the next peak, and update the
standard deviation of the "npeak" most recent peaks. These values are
stored cyclically in the "peaks" array. */
if( npeak > 0 ) {
new_peak = x[ 0 ];
old_peak = peaks[ ipeak ];
peaks[ ipeak ] = new_peak;
if( ++ipeak == npeak ) ipeak = 0;
sum_peak += new_peak - old_peak;
sum_peak2 += new_peak*new_peak - old_peak*old_peak;
if( sum_peak2 < 0.0 ) sum_peak2 = 0.0;
mean_peak = sum_peak/npeak;
sigma_peak = sqrt( sum_peak2/npeak - mean_peak*mean_peak );
}
/* Increment the number of peaks found. */
iclump++;
/* Reset the number of failed fits since the last good fit. */
nskip = 0;
/* Remove the model fit (excluding the background) from the residuals
array. This also creates an NDF containing the data values associated
with the clump. This NDF is stored in the HDS array of NDFs in the
returned HDS object. The standard deviation of the new residuals is
returned. */
cupidGCUpdateArrays( type, res, ipd, el, ndim, dims,
x, rms, mlim, imax, peak_thresh, slbnd,
&ret, iclump, excols, mean_peak,
maxbad, &area, &sumclumps, status );
/* Dump the modified residuals if required. */
sprintf( buf, "residuals%lu", iclump );
cupidGCDump( type, MSG__DEBUG3, res, ndim, dims, buf,
status );
/* Display the clump parameters on the screen if required. */
cupidGCListClump( iclump, ndim, x, chisq, slbnd,
rms, status );
/* If this clump has a peak value which is below the threshold, increment
the count of consecutive clumps with peak value below the threshold.
Otherwise, reset this count to zero. */
if( x[ 0 ] < peak_thresh ) {
peaks_below++;
} else {
peaks_below = 0;
}
/* If this clump has an area which is below the threshold, increment
the count of consecutive clumps with area below the threshold.
Otherwise, reset this count to zero. */
if( area < area_thresh ) {
area_below++;
} else {
area_below = 0;
}
/* If the maximum number of clumps have now been found, exit.*/
if( iclump == maxclump ) {
iter = 0;
msgBlankif( MSG__DEBUG, status );
msgOutiff( MSG__DEBUG1, "",
"The specified maximum number of "
"clumps (%lu) have been found.", status,
maxclump );
msgBlankif( MSG__DEBUG1, status );
/* If the integrated data sum in the fitted gaussians exceeds or equals
the integrated data sum in th einput, exit. */
} else if( sumclumps >= sumdata ) {
iter = 0;
msgBlankif( MSG__DEBUG, status );
msgOutiff( MSG__DEBUG1,"",
"The total data sum of the fitted "
"Gaussians (%g) has reached the total "
"data sum in the supplied data (%g).",
status, (float)sumclumps, (float)sumdata );
msgBlankif( MSG__DEBUG1, status );
/* If the count of consecutive peaks below the threshold has reached
"Npad", terminate. */
} else if( peaks_below == npad ) {
iter = 0;
msgBlankif( MSG__DEBUG, status );
msgOutiff( MSG__DEBUG1, "",
"The previous %d clumps all had peak "
"values below the threshold.", status,
npad );
msgBlankif( MSG__DEBUG1, status );
/* If the count of consecutive clumps with area below the threshold has reached
"Npad", terminate. */
} else if( area_below == npad ) {
iter = 0;
msgBlankif( MSG__DEBUG, status );
msgOutiff( MSG__DEBUG1, "",
"The previous %d clumps all had areas "
"below the threshold.", status, npad );
msgBlankif( MSG__DEBUG1, status );
}
/* If the peak value fitted is very different from the previous fitted peak
value, set the residuals array element bad in order to prevent the
algorithm from trying to fit a peak to the same pixel again. */
} else {
if( type == CUPID__DOUBLE ) {
((double *)res)[ imax ] = VAL__BADD;
} else {
((float *)res)[ imax ] = VAL__BADR;
}
new_peak = 0.5*( new_peak + x[ 0 ] );
nskip++;
msgOutif( MSG__DEBUG1, "", " Clump rejected due to "
"aberrant peak value.", status );
}
/* Tell the user if no clump could be fitted around the current peak
pixel value */
} else {
nskip++;
msgOutif( MSG__DEBUG1, "", " No clump fitted.", status );
/* Set the specified element of the residuals array bad if no fit was
performed. This prevents the any subsequent attempt to fit a Gaussian
to the same peak value.*/
if( type == CUPID__DOUBLE ) {
((double *)res)[ imax ] = VAL__BADD;
} else {
((float *)res)[ imax ] = VAL__BADR;
}
}
/* Tell the user if one of the trmination criteria has ben met. */
} else {
msgOutif( MSG__DEBUG1, "",
" At least one termination criterion has been reached.",
status );
msgBlankif( MSG__DEBUG1, status );
}
}
/* Tell the user how clumps are being returned. */
if( ret ) {
datSize( ret, &nclump, status );
} else {
nclump = 0;
}
if( nclump == 0 ) msgOutif( MSG__NORM, "", "No usable clumps found.", status );
if( iclump - nclump == 1 ) {
msgOutif( MSG__NORM, "",
"1 clump rejected because it touches an edge of "
"the data array.", status );
} else if( iclump - nclump > 1 ) {
msgOutiff( MSG__NORM, "",
"%d clumps rejected because they touch an edge of "
"the data array.", status, (int)( iclump - nclump ) );
}
/* Tell the user how many iterations have been performed (i.e. how many
attempts there have been to fit a Gaussian peak). */
if( niter == 1 ){
msgOutif( MSG__DEBUG1, "", "No fit attempted.", status );
} else {
msgOutiff( MSG__DEBUG1, "",
"Fits attempted for %d candidate clumps (%d failed).",
status, (int)( niter - iclump ), niter );
}
/* Free resources */
peaks = astFree( peaks );
}
/* Remove the secondary KeyMap added to the KeyMap containing configuration
parameters for this algorithm. This prevents the values in the secondary
KeyMap being written out to the CUPID extension when cupidStoreConfig is
called. */
astMapRemove( gcconfig, CUPID__CONFIG );
/* Free resources */
res = astFree( res );
dims = astFree( dims );
cupidGC.data = astFree( cupidGC.data );
cupidGC.weight = astFree( cupidGC.weight );
cupidGC.res = astFree( cupidGC.res );
cupidGC.resu = astFree( cupidGC.resu );
cupidGC.initmodel = astFree( cupidGC.initmodel );
cupidGC.model = astFree( cupidGC.model );
cupidGC.resids = astFree( cupidGC.resids );
gcconfig = astAnnul( gcconfig );
/* Return the list of clump NDFs. */
return ret;
}
void smurf_sc2fft( int *status ) {
int avpspec=0; /* Flag for doing average power spectrum */
double avpspecthresh=0; /* Threshold noise for detectors in avpspec */
Grp * basegrp = NULL; /* Basis group for output filenames */
smfArray *bbms = NULL; /* Bad bolometer masks */
smfArray *concat=NULL; /* Pointer to a smfArray */
size_t contchunk; /* Continuous chunk counter */
smfArray *darks = NULL; /* dark frames */
int ensureflat; /* Flag for flatfielding data */
Grp *fgrp = NULL; /* Filtered group, no darks */
smfArray *flatramps = NULL;/* Flatfield ramps */
AstKeyMap *heateffmap = NULL; /* Heater efficiency data */
size_t gcount=0; /* Grp index counter */
size_t i; /* Loop counter */
smfGroup *igroup=NULL; /* smfGroup corresponding to igrp */
Grp *igrp = NULL; /* Input group of files */
int inverse=0; /* If set perform inverse transform */
int isfft=0; /* Are data fft or real space? */
dim_t maxconcat=0; /* Longest continuous chunk length in samples */
size_t ncontchunks=0; /* Number continuous chunks outside iter loop */
smfData *odata=NULL; /* Pointer to output smfData to be exported */
Grp *ogrp = NULL; /* Output group of files */
size_t outsize; /* Total number of NDF names in the output group */
int polar=0; /* Flag for FFT in polar coordinates */
int power=0; /* Flag for squaring amplitude coeffs */
size_t size; /* Number of files in input group */
smfData *tempdata=NULL; /* Temporary smfData pointer */
int weightavpspec=0; /* Flag for 1/noise^2 weighting */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
int zerobad; /* Zero VAL__BADD before taking FFT? */
/* Main routine */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get input file(s) */
kpg1Rgndf( "IN", 0, 1, "", &igrp, &size, status );
/* Filter out darks */
smf_find_science( igrp, &fgrp, 1, NULL, NULL, 1, 1, SMF__NULL, &darks,
&flatramps, &heateffmap, NULL, status );
/* input group is now the filtered group so we can use that and
free the old input group */
size = grpGrpsz( fgrp, status );
grpDelet( &igrp, status);
igrp = fgrp;
fgrp = NULL;
/* We now need to combine files from the same subarray and same sequence
to form a continuous time series */
smf_grp_related( igrp, size, 1, 0, 0, NULL, NULL, &maxconcat, NULL, &igroup,
&basegrp, NULL, status );
/* Get output file(s) */
size = grpGrpsz( basegrp, status );
if( size > 0 ) {
kpg1Wgndf( "OUT", basegrp, size, size, "More output files required...",
&ogrp, &outsize, status );
} else {
msgOutif(MSG__NORM, " ", TASK_NAME ": All supplied input frames were DARK,"
" nothing to do", status );
}
/* Get group of bolometer masks and read them into a smfArray */
smf_request_mask( "BBM", &bbms, status );
/* Obtain the number of continuous chunks and subarrays */
if( *status == SAI__OK ) {
ncontchunks = igroup->chunk[igroup->ngroups-1]+1;
}
msgOutiff( MSG__NORM, "", "Found %zu continuous chunk%s", status, ncontchunks,
(ncontchunks > 1 ? "s" : "") );
/* Are we flatfielding? */
parGet0l( "FLAT", &ensureflat, status );
/* Are we doing an inverse transform? */
parGet0l( "INVERSE", &inverse, status );
/* Are we using polar coordinates instead of cartesian for the FFT? */
parGet0l( "POLAR", &polar, status );
/* Are we going to assume amplitudes are squared? */
parGet0l( "POWER", &power, status );
/* Are we going to zero bad values first? */
parGet0l( "ZEROBAD", &zerobad, status );
/* Are we calculating the average power spectrum? */
parGet0l( "AVPSPEC", &avpspec, status );
if( avpspec ) {
power = 1;
parGet0d( "AVPSPECTHRESH", &avpspecthresh, status );
parGet0l( "WEIGHTAVPSPEC", &weightavpspec, status );
}
/* If power is true, we must be in polar form */
if( power && !polar) {
msgOutif( MSG__NORM, " ", TASK_NAME
": power spectrum requested so setting POLAR=TRUE", status );
polar = 1;
}
gcount = 1;
for( contchunk=0;(*status==SAI__OK)&&contchunk<ncontchunks; contchunk++ ) {
size_t idx;
/* Concatenate this continuous chunk but forcing a raw data read.
We will need quality. */
smf_concat_smfGroup( wf, NULL, igroup, darks, NULL, flatramps, heateffmap,
contchunk, ensureflat, 1, NULL, 0, NULL, NULL, 0, 0, 0,
&concat, NULL, status );
/* Now loop over each subarray */
/* Export concatenated data for each subarray to NDF file */
for( idx=0; (*status==SAI__OK)&&idx<concat->ndat; idx++ ) {
if( concat->sdata[idx] ) {
smfData * idata = concat->sdata[idx];
int provid = NDF__NOID;
dim_t nbolo; /* Number of detectors */
dim_t ndata; /* Number of data points */
/* Apply a mask to the quality array and data array */
smf_apply_mask( idata, bbms, SMF__BBM_QUAL|SMF__BBM_DATA, 0, status );
smf_get_dims( idata, NULL, NULL, &nbolo, NULL, &ndata, NULL, NULL,
status );
/* Check for double precision data */
if( idata->dtype != SMF__DOUBLE ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": data are not double precision.", status );
}
/* Are we zeroing VAL__BADD? */
if( (*status==SAI__OK) && zerobad ) {
double *data= (double *) idata->pntr[0];
for( i=0; i<ndata; i++ ) {
if( data[i] == VAL__BADD ) {
data[i] = 0;
}
}
}
/* Check whether we need to transform the data at all */
isfft = smf_isfft(idata,NULL,NULL,NULL,NULL,NULL,status);
if( isfft && avpspec && (*status == SAI__OK) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": to calculate average power spectrum input data cannot "
"be FFT", status );
}
if( (*status == SAI__OK) && (isfft == inverse) ) {
if( avpspec ) {
/* If calculating average power spectrum do the transforms with
smf_bolonoise so that we can also measure the noise of
each detector */
double *whitenoise=NULL;
smf_qual_t *bolomask=NULL;
double mean, sig, freqlo;
size_t ngood, newgood;
whitenoise = astCalloc( nbolo, sizeof(*whitenoise) );
bolomask = astCalloc( nbolo, sizeof(*bolomask) );
freqlo = 1. / (idata->hdr->steptime * idata->hdr->nframes);
smf_bolonoise( wf, idata, 1, freqlo, SMF__F_WHITELO,
SMF__F_WHITEHI, 1, 0, whitenoise, NULL, &odata,
status );
/* Initialize quality */
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] == VAL__BADD ) {
bolomask[i] = SMF__Q_BADB;
} else {
/* smf_bolonoise returns a variance, so take sqrt */
whitenoise[i] = sqrt(whitenoise[i]);
}
}
ngood=-1;
newgood=0;
/* Iteratively cut n-sigma noisy outlier detectors */
while( ngood != newgood ) {
ngood = newgood;
smf_stats1D( whitenoise, 1, nbolo, bolomask, 1, SMF__Q_BADB,
&mean, &sig, NULL, NULL, status );
msgOutiff( MSG__DEBUG, "", TASK_NAME
": mean=%lf sig=%lf ngood=%li\n", status,
mean, sig, ngood);
newgood=0;
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] != VAL__BADD ){
if( (whitenoise[i] - mean) > avpspecthresh *sig ) {
whitenoise[i] = VAL__BADD;
bolomask[i] = SMF__Q_BADB;
} else {
newgood++;
}
}
}
}
msgOutf( "", TASK_NAME
": Calculating average power spectrum of best %li "
" bolometers.", status, newgood);
/* If using 1/noise^2 weights, calculate 1/whitenoise^2 in-place
to avoid allocating another array */
if( weightavpspec ) {
msgOutif( MSG__VERB, "", TASK_NAME ": using 1/noise^2 weights",
status );
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] && (whitenoise[i] != VAL__BADD) ) {
whitenoise[i] = 1/(whitenoise[i]*whitenoise[i]);
}
}
}
/* Calculate the average power spectrum of good detectors */
tempdata = smf_fft_avpspec( odata, bolomask, 1, SMF__Q_BADB,
weightavpspec ? whitenoise : NULL,
status );
smf_close_file( &odata, status );
whitenoise = astFree( whitenoise );
bolomask = astFree( bolomask );
odata = tempdata;
tempdata = NULL;
/* Store the number of good bolometers */
parPut0i( "NGOOD", newgood, status );
} else {
/* Otherwise do forward/inverse transforms here as needed */
/* If inverse transform convert to cartesian representation first */
if( inverse && polar ) {
smf_fft_cart2pol( wf, idata, 1, power, status );
}
/* Tranform the data */
odata = smf_fft_data( wf, idata, NULL, inverse, 0, status );
smf_convert_bad( wf, odata, status );
if( inverse ) {
/* If output is time-domain, ensure that it is ICD bolo-ordered */
smf_dataOrder( odata, 1, status );
} else if( polar ) {
/* Store FFT of data in polar form */
smf_fft_cart2pol( wf, odata, 0, power, status );
}
}
/* open a reference input file for provenance propagation */
ndgNdfas( basegrp, gcount, "READ", &provid, status );
/* Export the data to a new file */
smf_write_smfData( odata, NULL, NULL, ogrp, gcount, provid,
MSG__VERB, 0, status );
/* Free resources */
ndfAnnul( &provid, status );
smf_close_file( &odata, status );
} else {
msgOutif( MSG__NORM, " ",
"Data are already transformed. No output will be produced",
status );
}
}
/* Update index into group */
gcount++;
}
/* Close the smfArray */
smf_close_related( &concat, status );
}
/* Write out the list of output NDF names, annulling the error if a null
parameter value is supplied. */
if( *status == SAI__OK ) {
grpList( "OUTFILES", 0, 0, NULL, ogrp, status );
if( *status == PAR__NULL ) errAnnul( status );
}
/* Tidy up after ourselves: release the resources used by the grp routines */
grpDelet( &igrp, status);
grpDelet( &ogrp, status);
if (basegrp) grpDelet( &basegrp, status );
if( igroup ) smf_close_smfGroup( &igroup, status );
if( flatramps ) smf_close_related( &flatramps, status );
if (heateffmap) heateffmap = smf_free_effmap( heateffmap, status );
if (bbms) smf_close_related( &bbms, status );
ndfEnd( status );
/* Ensure that FFTW doesn't have any used memory kicking around */
fftw_cleanup();
}
void smf_filter_mce( smfFilter *filt, int noinverse, int *status ) {
/* Filter parameters */
double B_1_1;
double B_1_2;
double B_2_1;
double B_2_2;
double CLOCK_PERIOD;
double ROW_DWELL;
double NUM_ROWS=41;
double DELTA_TIME;
double SRATE;
double datechange; /* UTC MJD for change in MCE filter parameters */
AstTimeFrame *tf=NULL; /* time frame for date conversion */
size_t i; /* Loop counter */
if( *status != SAI__OK ) return;
if( !filt ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "NULL smfFilter supplied.", status );
return;
}
if( filt->ndims != 1 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": function only generates filters for time-series",
status );
return;
}
if( !filt->fdims[0] ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": 0-length smfFilter supplied.",
status );
return;
}
if( filt->dateobs == VAL__BADD ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": dateobs (date of data to which filter will be "
"applied) is not set - can't determine correct MCE filter "
"parameters.", status );
return;
}
/* If filt->real is NULL, create a complex identity filter first. Similarly,
if the filter is currently only real-valued, add an imaginary part. */
if( !filt->real ) {
smf_filter_ident( filt, 1, status );
if( *status != SAI__OK ) return;
} else if( !filt->imag ) {
filt->imag = astCalloc( filt->fdims[0], sizeof(*filt->imag) );
if( *status != SAI__OK ) return;
filt->isComplex = 1;
}
/* Set up filter parameters */
tf = astTimeFrame( " " );
astSet( tf, "TimeScale=UTC" );
astSet( tf, "TimeOrigin=%s", "2011-06-03T00:00:00" );
datechange = astGetD( tf, "TimeOrigin" );
tf = astAnnul( tf );
if( filt->dateobs > datechange ) {
/* Data taken after 20110603 */
B_1_1 = -1.9712524; /* -2.*32297./2.^15. */
B_1_2 = 0.97253418; /* 2.*15934./2.^15. */
B_2_1 = -1.9337769; /* -2.*31683./2.^15. */
B_2_2 = 0.93505859; /* 2.*15320./2.^15. */
ROW_DWELL = 94.; /* time to dwell at each row (in clocks) */
msgOutiff(MSG__DEBUG, "", FUNC_NAME
": filter for data UTC MJD %lf after %lf", status,
filt->dateobs, datechange );
} else {
/* Older data */
B_1_1 = -1.9587402; /* -2.*32092./2.^15. */
B_1_2 = 0.96130371; /* 2.*15750./2.^15. */
B_2_1 = -1.9066162; /* -2.*31238./2.^15. */
B_2_2 = 0.90911865; /* 2.*14895./2.^15. */
ROW_DWELL = 128.; /* time to dwell at each row (in clocks) */
msgOutiff(MSG__DEBUG, "", FUNC_NAME
": filter for data UTC MJD %lf before %lf", status,
filt->dateobs, datechange );
}
CLOCK_PERIOD = 20E-9; /* 50 MHz clock */
NUM_ROWS = 41.; /* number of rows addressed */
DELTA_TIME = (CLOCK_PERIOD*ROW_DWELL*NUM_ROWS); /* sample length */
SRATE = (1./DELTA_TIME); /* sample rate */
/* Loop over all frequencies in the filter */
for( i=0; i<filt->fdims[0]; i++ ) {
double cos_m_o;
double sin_m_o;
double cos_m_2o;
double sin_m_2o;
double f;
gsl_complex den;
gsl_complex h1_omega;
gsl_complex h2_omega;
gsl_complex h_omega;
gsl_complex num;
gsl_complex temp;
double omega;
f = filt->df[0]*i; /* Frequency at this step */
omega = (f / SRATE)*2*AST__DPI; /* Angular frequency */
cos_m_o = cos(-omega);
sin_m_o = sin(-omega);
cos_m_2o = cos(-2*omega);
sin_m_2o = sin(-2*omega);
/*
h1_omega=(1 + 2*complex(cos_m_o,sin_m_o) + complex(cos_m_2o,sin_m_2o)) /
(1 + b_1_1*complex(cos_m_o,sin_m_o) + b_1_2 * complex(cos_m_2o,sin_m_2o))
*/
/* numerator */
GSL_SET_COMPLEX(&num, 1, 0);
GSL_SET_COMPLEX(&temp, cos_m_o, sin_m_o);
num = gsl_complex_add( num, gsl_complex_mul_real(temp, 2) );
GSL_SET_COMPLEX(&temp, cos_m_2o, sin_m_2o);
num = gsl_complex_add( num, temp );
/* denominator */
GSL_SET_COMPLEX(&den, 1, 0);
GSL_SET_COMPLEX(&temp, cos_m_o, sin_m_o);
den = gsl_complex_add( den, gsl_complex_mul_real(temp,B_1_1) );
GSL_SET_COMPLEX(&temp, cos_m_2o, sin_m_2o);
den = gsl_complex_add( den, gsl_complex_mul_real(temp,B_1_2) );
/* quotient */
h1_omega = gsl_complex_div( num, den );
/*
h2_omega=(1 + 2*complex(cos_m_o,sin_m_o) + complex(cos_m_2o,sin_m_2o)) /
(1 + b_2_1*complex(cos_m_o,sin_m_o) + b_2_2*complex(cos_m_2o,sin_m_2o))
note: we can re-use numerator from above
*/
/* denominator */
GSL_SET_COMPLEX(&den, 1, 0);
GSL_SET_COMPLEX(&temp, cos_m_o, sin_m_o);
den = gsl_complex_add( den, gsl_complex_mul_real(temp,B_2_1) );
GSL_SET_COMPLEX(&temp, cos_m_2o, sin_m_2o);
den = gsl_complex_add( den, gsl_complex_mul_real(temp,B_2_2) );
/* quotient */
h2_omega = gsl_complex_div( num, den );
/* And finally...
h_omega=h1_omega*h2_omega/2048.
*/
h_omega = gsl_complex_mul( h1_omega, gsl_complex_div_real(h2_omega,2048.) );
/* Normally we are applying the inverse of the filter to remove
its effect from the time-series. */
if( !noinverse ) {
h_omega = gsl_complex_inverse( h_omega );
}
/* Then apply this factor to the filter. */
GSL_SET_COMPLEX( &temp, filt->real[i], filt->imag[i] );
temp = gsl_complex_mul( temp, h_omega );
filt->real[i] = GSL_REAL( temp );
filt->imag[i] = GSL_IMAG( temp );
}
}
void smf_mapbounds( int fast, Grp *igrp, int size, const char *system,
AstFrameSet *spacerefwcs, int alignsys, int *lbnd_out,
int *ubnd_out, AstFrameSet **outframeset, int *moving,
smfBox ** boxes, fts2Port fts_port, int *status ) {
/* Local Variables */
AstSkyFrame *abskyframe = NULL; /* Output Absolute SkyFrame */
int actval; /* Number of parameter values supplied */
AstMapping *bolo2map = NULL; /* Combined mapping bolo->map
coordinates, WCS->GRID Mapping from
input WCS FrameSet */
smfBox *box = NULL; /* smfBox for current file */
smfData *data = NULL; /* pointer to SCUBA2 data struct */
double dlbnd[ 2 ]; /* Floating point lower bounds for output map */
drcntrl_bits drcntrl_mask = 0;/* Mask to use for DRCONTROL on this instrument */
double dubnd[ 2 ]; /* Floating point upper bounds for output map */
AstMapping *fast_map = NULL; /* Mapping from tracking to absolute map coords */
smfFile *file = NULL; /* SCUBA2 data file information */
int first; /* Is this the first good subscan ? */
AstFitsChan *fitschan = NULL;/* Fits channels to construct WCS header */
AstFrameSet *fs = NULL; /* A general purpose FrameSet pointer */
smfHead *hdr = NULL; /* Pointer to data header this time slice */
int i; /* Loop counter */
dim_t j; /* Loop counter */
AstSkyFrame *junksky = NULL; /* Unused SkyFrame argument */
dim_t k; /* Loop counter */
int lbnd0[ 2 ]; /* Defaults for LBND parameter */
double map_pa=0; /* Map PA in output coord system (rads) */
dim_t maxloop; /* Number of times to go round the time slice loop */
dim_t nbadt = 0; /* Number of bad time slices */
dim_t ngoodt = 0; /* Number of good time slices */
double par[7]; /* Projection parameters */
double shift[ 2 ]; /* Shifts from PIXEL to GRID coords */
AstMapping *oskymap = NULL; /* Mapping celestial->map coordinates,
Sky <> PIXEL mapping in output
FrameSet */
AstSkyFrame *oskyframe = NULL;/* Output SkyFrame */
char *refsys = NULL; /* Sky system from supplied reference FrameSet */
dim_t textreme[4]; /* Time index corresponding to minmax TCS posn */
AstFrame *skyin = NULL; /* Sky Frame in input FrameSet */
double skyref[ 2 ]; /* Values for output SkyFrame SkyRef attribute */
struct timeval tv1; /* Timer */
struct timeval tv2; /* Timer */
AstMapping *tmap; /* Temporary Mapping */
int trim; /* Trim borders of bad pixels from o/p image? */
int ubnd0[ 2 ]; /* Defaults for UBND parameter */
double x_array_corners[4]; /* X-Indices for corner bolos in array */
double x_map[4]; /* Projected X-coordinates of corner bolos */
double y_array_corners[4]; /* Y-Indices for corner pixels in array */
double y_map[4]; /* Projected X-coordinates of corner bolos */
/* Main routine */
if (*status != SAI__OK) return;
/* Start a timer to see how long this takes */
smf_timerinit( &tv1, &tv2, status );
/* Initialize pointer to output FrameSet and moving-source flag */
*outframeset = NULL;
*moving = 0;
/* initialize double precision output bounds and the proj pars */
for( i = 0; i < 7; i++ ) par[ i ] = AST__BAD;
dlbnd[ 0 ] = VAL__MAXD;
dlbnd[ 1 ] = VAL__MAXD;
dubnd[ 0 ] = VAL__MIND;
dubnd[ 1 ] = VAL__MIND;
/* If we have a supplied reference WCS we can use that directly
without having to calculate it from the data. Replace the requested
system with the system from the reference FrameSet (take a copy of the
string since astGetC may re-use its buffer). */
if (spacerefwcs) {
oskyframe = astGetFrame( spacerefwcs, AST__CURRENT );
int nc = 0;
refsys = astAppendString( NULL, &nc, astGetC( oskyframe, "System" ) );
system = refsys;
}
/* Create array of returned smfBox structures and store a pointer
to the next one to be initialised. */
*boxes = astMalloc( sizeof( smfBox ) * size );
box = *boxes;
astBegin;
/* Loop over all files in the Grp */
first = 1;
for( i=1; i<=size; i++, box++ ) {
/* Initialise the spatial bounds of section of the the output cube that is
contributed to by the current ionput file. */
box->lbnd[ 0 ] = VAL__MAXD;
box->lbnd[ 1 ] = VAL__MAXD;
box->ubnd[ 0 ] = VAL__MIND;
box->ubnd[ 1 ] = VAL__MIND;
/* Read data from the ith input file in the group */
smf_open_file( NULL, igrp, i, "READ", SMF__NOCREATE_DATA, &data, status );
if (*status != SAI__OK) {
msgSeti( "I", i );
errRep( "smf_mapbounds", "Could not open data file no ^I.", status );
break;
} else {
if( *status == SAI__OK ) {
if( data->file == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "No smfFile associated with smfData.",
status );
break;
} else if( data->hdr == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "No smfHead associated with smfData.",
status );
break;
} else if( data->hdr->fitshdr == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "No FITS header associated with smfHead.",
status );
break;
}
}
}
/* convenience pointers */
file = data->file;
hdr = data->hdr;
/* report name of the input file */
smf_smfFile_msg( file, "FILE", 1, "<unknown>" );
msgSeti("I", i);
msgSeti("N", size);
msgOutif(MSG__VERB, " ",
"SMF_MAPBOUNDS: Processing ^I/^N ^FILE",
status);
/* Check that there are 3 pixel axes. */
if( data->ndims != 3 ) {
smf_smfFile_msg( file, "FILE", 1, "<unknown>" );
msgSeti( "NDIMS", data->ndims );
*status = SAI__ERROR;
errRep( FUNC_NAME, "^FILE has ^NDIMS pixel axes, should be 3.",
status );
break;
}
/* Check that the data dimensions are 3 (for time ordered data) */
if( *status == SAI__OK ) {
/* If OK Decide which detectors (GRID coord) to use for
checking bounds, depending on the instrument in use. */
switch( hdr->instrument ) {
case INST__SCUBA2:
drcntrl_mask = DRCNTRL__POSITION;
/* 4 corner bolometers of the subarray */
x_array_corners[0] = 1;
x_array_corners[1] = 1;
x_array_corners[2] = (data->dims)[0];
x_array_corners[3] = (data->dims)[0];
y_array_corners[0] = 1;
y_array_corners[1] = (data->dims)[1];
y_array_corners[2] = 1;
y_array_corners[3] = (data->dims)[1];
break;
case INST__AZTEC:
/* Rough guess for extreme bolometers around the edge */
x_array_corners[0] = 22;
x_array_corners[1] = 65;
x_array_corners[2] = 73;
x_array_corners[3] = 98;
y_array_corners[0] = 1; /* Always 1 for AzTEC */
y_array_corners[1] = 1;
y_array_corners[2] = 1;
y_array_corners[3] = 1;
break;
case INST__ACSIS:
smf_find_acsis_corners( data, x_array_corners, y_array_corners,
status);
break;
default:
*status = SAI__ERROR;
errRep(FUNC_NAME, "Don't know how to calculate mapbounds for data created with this instrument", status);
}
}
if( *status == SAI__OK) {
size_t goodidx = SMF__BADSZT;
/* Need to build up a frameset based on good telescope position.
We can not assume that we the first step will be a good TCS position
so we look for one. If we can not find anything we skip to the
next file. */
maxloop = (data->dims)[2];
for (j=0; j<maxloop; j++) {
JCMTState state = (hdr->allState)[j];
if (state.jos_drcontrol >= 0 && state.jos_drcontrol & drcntrl_mask ) {
/* bad TCS - so try again */
} else {
/* Good tcs */
goodidx = j;
break;
}
}
if (goodidx == SMF__BADSZT) {
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutif( MSG__QUIET, "", "No good telescope positions found in file ^FILE. Ignoring",
status );
smf_close_file( NULL, &data, status );
continue;
}
/* If we are dealing with the first good file, create the output
SkyFrame. */
if( first ) {
first = 0;
/* Create output SkyFrame if it has not come from a reference */
if ( oskyframe == NULL ) {
/* smf_tslice_ast only needs to get called once to set up framesets */
if( hdr->wcs == NULL ) {
smf_tslice_ast( data, goodidx, 1, fts_port, status);
}
/* Retrieve input SkyFrame */
skyin = astGetFrame( hdr->wcs, AST__CURRENT );
smf_calc_skyframe( skyin, system, hdr, alignsys, &oskyframe, skyref,
moving, status );
/* Get the orientation of the map vertical within the output celestial
coordinate system. This is derived form the MAP_PA FITS header, which
gives the orientation of the map vertical within the tracking system. */
map_pa = smf_calc_mappa( hdr, system, skyin, status );
/* Provide a sensible default for the pixel size based on wavelength */
par[4] = smf_calc_telres( hdr->fitshdr, status );
par[4] *= AST__DD2R/3600.0;
par[5] = par[4];
/* Calculate the projection parameters. We do not enable autogrid determination
for SCUBA-2 so we do not need to obtain all the data before calculating
projection parameters. */
smf_get_projpar( oskyframe, skyref, *moving, 0, 0, NULL, 0,
map_pa, par, NULL, NULL, status );
if (skyin) skyin = astAnnul( skyin );
/* If the output skyframe has been supplied, we still need to
determine whether the source is moving or not, and set the
reference position. */
} else {
/* smf_tslice_ast only needs to get called once to set up framesets */
if( hdr->wcs == NULL ) {
smf_tslice_ast( data, goodidx, 1, fts_port, status);
}
/* Retrieve input SkyFrame */
skyin = astGetFrame( hdr->wcs, AST__CURRENT );
smf_calc_skyframe( skyin, system, hdr, alignsys, &junksky, skyref,
moving, status );
/* Store the sky reference position. If the target is moving,
ensure the returned SkyFrame represents offsets from the
reference position rather than absolute coords. */
astSetD( oskyframe, "SkyRef(1)", skyref[ 0 ] );
astSetD( oskyframe, "SkyRef(2)", skyref[ 1 ] );
if( *moving ) astSet( oskyframe, "SkyRefIs=Origin" );
/* Ensure the Epoch attribute in the map is set to the date of
the first data in the map, rather than the date in supplied
reference WCS. */
astSetD( oskyframe, "Epoch", astGetD( junksky, "Epoch" ) );
}
if ( *outframeset == NULL && oskyframe != NULL && (*status == SAI__OK)){
/* Now created a spatial Mapping. Use the supplied reference frameset
if supplied */
if (spacerefwcs) {
oskymap = astGetMapping( spacerefwcs, AST__BASE, AST__CURRENT );
} else {
/* Now populate a FitsChan with FITS-WCS headers describing
the required tan plane projection. The longitude and
latitude axis types are set to either (RA,Dec) or (AZ,EL)
to get the correct handedness. */
fitschan = astFitsChan ( NULL, NULL, " " );
smf_makefitschan( astGetC( oskyframe, "System"), &(par[0]),
&(par[2]), &(par[4]), par[6], fitschan, status );
astClear( fitschan, "Card" );
fs = astRead( fitschan );
/* Extract the output PIXEL->SKY Mapping. */
oskymap = astGetMapping( fs, AST__BASE, AST__CURRENT );
/* Tidy up */
fs = astAnnul( fs );
}
/* Create the output FrameSet */
*outframeset = astFrameSet( astFrame(2, "Domain=GRID"), " " );
/* Now add the SkyFrame to it */
astAddFrame( *outframeset, AST__BASE, oskymap, oskyframe );
/* Now add a POLANAL Frame if required (i.e. if the input time
series are POL-2 Q/U values). */
smf_addpolanal( *outframeset, hdr, status );
/* Invert the oskymap mapping */
astInvert( oskymap );
} /* End WCS FrameSet construction */
}
/* Get a copy of the output SkyFrame and ensure it represents
absolute coords rather than offset coords. */
abskyframe = astCopy( oskyframe );
astClear( abskyframe, "SkyRefIs" );
astClear( abskyframe, "AlignOffset" );
maxloop = (data->dims)[2];
if (fast) {
/* For scan map we scan through looking for largest telescope moves.
For dream/stare we just look at the start and end time slices to
account for sky rotation. */
if (hdr->obsmode != SMF__OBS_SCAN) {
textreme[0] = 0;
textreme[1] = (data->dims)[2] - 1;
maxloop = 2;
} else {
const char *tracksys;
double *ac1list, *ac2list, *bc1list, *bc2list, *p1, *p2, *p3, *p4;
double flbnd[4], fubnd[4];
JCMTState state;
/* If the output and tracking systems are different, get a
Mapping between them. */
tracksys = sc2ast_convert_system( (hdr->allState)[goodidx].tcs_tr_sys,
status );
if( strcmp( system, tracksys ) ) {
AstSkyFrame *tempsf = astCopy( abskyframe );
astSetC( tempsf, "System", tracksys );
AstFrameSet *tempfs = astConvert( tempsf, abskyframe, "" );
tmap = astGetMapping( tempfs, AST__BASE, AST__CURRENT );
fast_map = astSimplify( tmap );
tmap = astAnnul( tmap );
tempsf = astAnnul( tempsf );
tempfs = astAnnul( tempfs );
} else {
fast_map = NULL;
}
/* Copy all ac1/2 positions into two array, and transform them
from tracking to absolute output sky coords. */
ac1list = astMalloc( maxloop*sizeof( *ac1list ) );
ac2list = astMalloc( maxloop*sizeof( *ac2list ) );
if( *status == SAI__OK ) {
p1 = ac1list;
p2 = ac2list;
for( j = 0; j < maxloop; j++ ) {
state = (hdr->allState)[ j ];
*(p1++) = state.tcs_tr_ac1;
*(p2++) = state.tcs_tr_ac2;
}
if( fast_map ) astTran2( fast_map, maxloop, ac1list, ac2list, 1,
ac1list, ac2list );
}
/* If the target is moving, we need to adjust these ac1/2 values
to represent offsets from the base position. */
if( *moving ) {
/* Copy all bc1/2 positions into two arrays. */
bc1list = astMalloc( maxloop*sizeof( *bc1list ) );
bc2list = astMalloc( maxloop*sizeof( *bc2list ) );
if( *status == SAI__OK ) {
p1 = bc1list;
p2 = bc2list;
for( j = 0; j < maxloop; j++ ) {
state = (hdr->allState)[ j ];
*(p1++) = state.tcs_tr_bc1;
*(p2++) = state.tcs_tr_bc2;
}
/* Transform them from tracking to absolute output sky coords. */
if( fast_map ) astTran2( fast_map, maxloop, bc1list, bc2list,
1, bc1list, bc2list );
/* Replace each ac1/2 position with the offsets from the
corresponding base position. */
p1 = bc1list;
p2 = bc2list;
p3 = ac1list;
p4 = ac2list;
for( j = 0; j < maxloop; j++ ) {
smf_offsets( *(p1++), *(p2++), p3++, p4++, status );
}
}
/* We no longer need the base positions. */
bc1list = astFree( bc1list );
bc2list = astFree( bc2list );
}
/* Transform the ac1/2 position from output sky coords to
output pixel coords. */
astTran2( oskymap, maxloop, ac1list, ac2list, 1, ac1list, ac2list );
/* Find the bounding box containing these pixel coords and the
time slices at which the boresight touches each edge of this
box. */
flbnd[ 0 ] = VAL__MAXD;
flbnd[ 1 ] = VAL__MAXD;
fubnd[ 0 ] = VAL__MIND;
fubnd[ 1 ] = VAL__MIND;
for( j = 0; j < 4; j++ ) textreme[ j ] = (dim_t) VAL__BADI;
if( *status == SAI__OK ) {
p1 = ac1list;
p2 = ac2list;
for( j = 0; j < maxloop; j++,p1++,p2++ ) {
if( *p1 != VAL__BADD && *p2 != VAL__BADD ){
if ( *p1 < flbnd[0] ) { flbnd[0] = *p1; textreme[0] = j; }
if ( *p2 < flbnd[1] ) { flbnd[1] = *p2; textreme[1] = j; }
if ( *p1 > fubnd[0] ) { fubnd[0] = *p1; textreme[2] = j; }
if ( *p2 > fubnd[1] ) { fubnd[1] = *p2; textreme[3] = j; }
}
}
}
maxloop = 4;
msgSetd("X1", textreme[0]);
msgSetd("X2", textreme[1]);
msgSetd("X3", textreme[2]);
msgSetd("X4", textreme[3]);
msgOutif( MSG__DEBUG, " ",
"Extrema time slices are ^X1, ^X2, ^X3 and ^X4",
status);
ac1list = astFree( ac1list );
ac2list = astFree( ac2list );
}
}
/* Get the astrometry for all the relevant time slices in this data file */
for( j=0; j<maxloop; j++ ) {
dim_t ts; /* Actual time slice to use */
/* if we are doing the fast loop, we need to read the time slice
index from textreme. Else we just use the index */
if (fast) {
/* get the index but make sure it is good */
ts = textreme[j];
if (ts == (dim_t)VAL__BADI) continue;
} else {
ts = j;
}
/* Calculate the bolo to map-pixel transformation for this tslice */
bolo2map = smf_rebin_totmap( data, ts, abskyframe, oskymap,
*moving, fts_port, status );
if ( *status == SAI__OK ) {
/* skip if we did not get a mapping this time round */
if (!bolo2map) continue;
/* Check corner pixels in the array for their projected extent
on the sky to set the pixel bounds */
astTran2( bolo2map, 4, x_array_corners, y_array_corners, 1,
x_map, y_map );
/* Update min/max for this time slice */
for( k=0; k<4; k++ ) {
if( x_map[k] != AST__BAD && y_map[k] != AST__BAD ) {
if( x_map[k] < dlbnd[0] ) dlbnd[0] = x_map[k];
if( y_map[k] < dlbnd[1] ) dlbnd[1] = y_map[k];
if( x_map[k] > dubnd[0] ) dubnd[0] = x_map[k];
if( y_map[k] > dubnd[1] ) dubnd[1] = y_map[k];
if( x_map[k] < box->lbnd[0] ) box->lbnd[0] = x_map[k];
if( y_map[k] < box->lbnd[1] ) box->lbnd[1] = y_map[k];
if( x_map[k] > box->ubnd[0] ) box->ubnd[0] = x_map[k];
if( y_map[k] > box->ubnd[1] ) box->ubnd[1] = y_map[k];
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Extreme positions are bad.", status );
break;
}
}
}
/* Explicitly annul these mappings each time slice for reduced
memory usage */
if (bolo2map) bolo2map = astAnnul( bolo2map );
if (fs) fs = astAnnul( fs );
/* Break out of loop over time slices if bad status */
if (*status != SAI__OK) goto CLEANUP;
}
/* Annul any remaining Ast objects before moving on to the next file */
if (fs) fs = astAnnul( fs );
if (bolo2map) bolo2map = astAnnul( bolo2map );
}
/* Close the data file */
smf_close_file( NULL, &data, status);
/* Break out of loop over data files if bad status */
if (*status != SAI__OK) goto CLEANUP;
}
/* make sure we got values - should not be possible with good status */
if (dlbnd[0] == VAL__MAXD || dlbnd[1] == VAL__MAXD) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRep( " ", "Unable to find any valid map bounds", status );
}
}
if (nbadt > 0) {
msgOutf( "", " Processed %zu time slices to calculate bounds,"
" of which %zu had bad telescope data and were skipped",
status, (size_t)(ngoodt+nbadt), (size_t)nbadt );
}
/* If spatial reference wcs was supplied, store par values that result in
no change to the pixel origin. */
if( spacerefwcs ){
par[ 0 ] = 0.5;
par[ 1 ] = 0.5;
}
/* Need to re-align with the interim GRID coordinates */
lbnd_out[0] = ceil( dlbnd[0] - par[0] + 0.5 );
ubnd_out[0] = ceil( dubnd[0] - par[0] + 0.5 );
lbnd_out[1] = ceil( dlbnd[1] - par[1] + 0.5 );
ubnd_out[1] = ceil( dubnd[1] - par[1] + 0.5 );
/* Do the same with the individual input file bounding boxes */
box = *boxes;
for (i = 1; i <= size; i++, box++) {
box->lbnd[0] = ceil( box->lbnd[0] - par[0] + 0.5);
box->ubnd[0] = ceil( box->ubnd[0] - par[0] + 0.5);
box->lbnd[1] = ceil( box->lbnd[1] - par[1] + 0.5);
box->ubnd[1] = ceil( box->ubnd[1] - par[1] + 0.5);
}
/* Apply a ShiftMap to the output FrameSet to re-align the GRID
coordinates */
shift[0] = 2.0 - par[0] - lbnd_out[0];
shift[1] = 2.0 - par[1] - lbnd_out[1];
astRemapFrame( *outframeset, AST__BASE, astShiftMap( 2, shift, " " ) );
/* Set the dynamic defaults for lbnd/ubnd */
lbnd0[ 0 ] = lbnd_out[ 0 ];
lbnd0[ 1 ] = lbnd_out[ 1 ];
parDef1i( "LBND", 2, lbnd0, status );
ubnd0[ 0 ] = ubnd_out[ 0 ];
ubnd0[ 1 ] = ubnd_out[ 1 ];
parDef1i( "UBND", 2, ubnd0, status );
parGet1i( "LBND", 2, lbnd_out, &actval, status );
if( actval == 1 ) lbnd_out[ 1 ] = lbnd_out[ 0 ];
parGet1i( "UBND", 2, ubnd_out, &actval, status );
if( actval == 1 ) ubnd_out[ 1 ] = ubnd_out[ 0 ];
/* Ensure the bounds are the right way round. */
if( lbnd_out[ 0 ] > ubnd_out[ 0 ] ) {
int itmp = lbnd_out[ 0 ];
lbnd_out[ 0 ] = ubnd_out[ 0 ];
ubnd_out[ 0 ] = itmp;
}
if( lbnd_out[ 1 ] > ubnd_out[ 1 ] ) {
int itmp = lbnd_out[ 1 ];
lbnd_out[ 1 ] = ubnd_out[ 1 ];
ubnd_out[ 1 ] = itmp;
}
/* If borders of bad pixels are being trimmed from the output image,
then do not allow the user-specified bounds to extend outside the
default bounding box (since we know that the default bounding box
encloses all available data). */
parGet0l( "TRIM", &trim, status );
if( trim ) {
if( lbnd_out[ 0 ] < lbnd0[ 0 ] ) lbnd_out[ 0 ] = lbnd0[ 0 ];
if( lbnd_out[ 1 ] < lbnd0[ 1 ] ) lbnd_out[ 1 ] = lbnd0[ 1 ];
if( ubnd_out[ 0 ] > ubnd0[ 0 ] ) ubnd_out[ 0 ] = ubnd0[ 0 ];
if( ubnd_out[ 1 ] > ubnd0[ 1 ] ) ubnd_out[ 1 ] = ubnd0[ 1 ];
}
/* Modify the returned FrameSet to take account of the new pixel origin. */
shift[ 0 ] = lbnd0[ 0 ] - lbnd_out[ 0 ];
shift[ 1 ] = lbnd0[ 1 ] - lbnd_out[ 1 ];
if( shift[ 0 ] != 0.0 || shift[ 1 ] != 0.0 ) {
astRemapFrame( *outframeset, AST__BASE, astShiftMap( 2, shift, " " ) );
}
/* Report the pixel bounds of the cube. */
if( *status == SAI__OK ) {
msgOutif( MSG__NORM, " ", " ", status );
msgSeti( "XL", lbnd_out[ 0 ] );
msgSeti( "YL", lbnd_out[ 1 ] );
msgSeti( "XU", ubnd_out[ 0 ] );
msgSeti( "YU", ubnd_out[ 1 ] );
msgOutif( MSG__NORM, " ", " Output map pixel bounds: ( ^XL:^XU, ^YL:^YU )",
status );
if( ( ubnd_out[ 0 ] - lbnd_out[ 0 ] + 1 ) > MAX_DIM ||
( ubnd_out[ 1 ] - lbnd_out[ 1 ] + 1 ) > MAX_DIM ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": The map is too big. Check your list of input "
"data files does not include widely separated observations.",
status );
}
}
/* If no error has occurred, export the returned FrameSet pointer from the
current AST context so that it will not be annulled when the AST
context is ended. Otherwise, ensure a null pointer is returned. */
if( *status == SAI__OK ) {
astExport( *outframeset );
} else {
*outframeset = astAnnul( *outframeset );
}
msgOutiff( SMF__TIMER_MSG, "",
"Took %.3f s to calculate map bounds",
status, smf_timerupdate( &tv1, &tv2, status ) );
/* Clean Up */
CLEANUP:
if (*status != SAI__OK) {
errRep(FUNC_NAME, "Unable to determine map bounds", status);
}
if (oskymap) oskymap = astAnnul( oskymap );
if (bolo2map) bolo2map = astAnnul( bolo2map );
if (fitschan) fitschan = astAnnul( fitschan );
if( data != NULL )
smf_close_file( NULL, &data, status );
refsys = astFree( refsys );
astEnd;
}
void
smf_flat_params( const smfData * refdata, const char resistpar[],
const char methpar[], const char orderpar[], const char snrminpar[],
double * refohms, double **resistance, int * outrows,
int * outcols, smf_flatmeth *flatmeth,
int * order, double * snrmin, smfData ** heateff,
int * status ) {
dim_t datarows = 0; /* Number of rows in refdata */
dim_t datacols = 0; /* Number of columns in refdata */
size_t j = 0; /* Counter, index */
char method[SC2STORE_FLATLEN]; /* flatfield method string */
size_t nbols; /* Number of bolometers */
double refohmsval = 0.0; /* Internal version of refohms */
AstKeyMap * resmap = NULL; /* Resistor map */
AstKeyMap * subarrays = NULL; /* Subarray lookup table */
char thissub[32]; /* This sub-instrument string */
if (resistance) *resistance = NULL;
if (*status != SAI__OK) return;
if (!refdata) {
*status = SAI__ERROR;
errRep( "", "Must provide reference data file to calculate flatfield parameters"
" (possible programming error)", status );
return;
}
/* Based on refdata we now need to calculate the default reference
resistance and retrieve the correct heater efficiency file for each array.
We need the unique subarray string so that we can set up a look up keymap.
There is no code in SMURF to return all the known subarrays but
we need to know all the options in order to use kpg1Config. */
subarrays = astKeyMap( " " );
astMapPut0I( subarrays, "CG450MK2_M0907D0501", 0, NULL );
astMapPut0I( subarrays, "CG850MK2_M0904D0503", 0, NULL );
astMapPut0I( subarrays, "SG850_M0906D1005", 0, NULL );
astMapPut0I( subarrays, "SG850_M1002D1006", 0, NULL );
astMapPut0I( subarrays, "SG850_M1005D1007", 0, NULL );
astMapPut0I( subarrays, "SG850_M1003D1004", 0, NULL );
astMapPut0I( subarrays, "SG450_M1004D1000", 0, NULL );
astMapPut0I( subarrays, "SG450_M1007D1002", 0, NULL );
astMapPut0I( subarrays, "SG450_M1006D1003", 0, NULL );
astMapPut0I( subarrays, "SG450_M1009D1008", 0, NULL );
/* and indicate which subarray we are interested in (uppercased) */
smf_fits_getS( refdata->hdr, "ARRAYID", thissub, sizeof(thissub), status );
{ /* need to uppercase */
size_t l = strlen(thissub);
for (j=0;j<l;j++) {
thissub[j] = toupper(thissub[j]);
}
}
astMapPut0I( subarrays, thissub, 1, NULL );
/* Read the config file */
resmap = kpg1Config( resistpar, "$SMURF_DIR/smurf_calcflat.def",
subarrays, 1, status );
subarrays = astAnnul( subarrays );
if (*status != SAI__OK) goto CLEANUP;
/* Read the reference resistance */
astMapGet0D( resmap, "REFRES", &refohmsval );
if (refohms && *status == SAI__OK) {
*refohms = refohmsval;
msgOutiff(MSG__VERB, "",
"Read reference resistance for subarray %s of %g ohms\n",
status, thissub, *refohms );
}
/* We no longer want to read per-bolometer resistor values from the
config file. To retain backwards compatibility with the current
implementation of smf_flat_standardpow we simply fill the
per-bol resistance array with the reference resistance which
effectively disables smf_flat_standardpow */
smf_get_dims( refdata, &datarows, &datacols, NULL, NULL, NULL, NULL, NULL, status );
nbols = datacols * datarows;
if (*status == SAI__OK && resistance ) {
*resistance = astMalloc( nbols*sizeof(**resistance) );
for (j = 0; j < (size_t)nbols; j++) {
(*resistance)[j] = refohmsval;
}
}
/* Get the heater efficiency file */
if (heateff && astMapHasKey( resmap, "HEATEFF" ) ) {
const char * heateffstr = NULL;
if (astMapGet0C( resmap, "HEATEFF", &heateffstr )) {
Grp * heateffgrp = NULL;
smfData * heatefftmp = NULL;
heateffgrp = grpNew( "heateff", status );
grpPut1( heateffgrp, heateffstr, 0, status );
smf_open_file( NULL, heateffgrp, 1, "READ", SMF__NOTTSERIES|SMF__NOFIX_METADATA, &heatefftmp, status );
/* Divorce the smfData from the underlying file. This file stays open for the entire
duration of the data processing and can some times lead to issues when we attempt
to close it an hour after we opened it (it's usually on an NFS disk) */
if (*status == SAI__OK) {
*heateff = smf_deepcopy_smfData( NULL, heatefftmp, 0,
SMF__NOCREATE_FILE | SMF__NOCREATE_FTS |
SMF__NOCREATE_DA,
0, 0, status );
smf_close_file(NULL, &heatefftmp, status);
}
/* Check the dimensions */
if (*status == SAI__OK) {
dim_t heatrows = 0;
dim_t heatcols = 0;
smf_get_dims( *heateff, &heatrows, &heatcols, NULL, NULL, NULL, NULL, NULL, status );
if (*status == SAI__OK) {
if ( datarows != heatrows || datacols != heatcols ) {
*status = SAI__ERROR;
errRepf( "", "Dimensions of heater efficiency file %s are (%zu, %zu)"
" but flatfield has dimensions (%zu, %zu)",
status, heateffstr, (size_t)heatrows, (size_t)heatcols,
(size_t)datarows, (size_t)datacols);
}
}
if (*status == SAI__OK) {
smf_dtype_check_fatal( *heateff, NULL, SMF__DOUBLE, status );
if (*status == SMF__BDTYP) {
errRepf("", "Heater efficiency data in %s should be double precision",
status, heateffstr);
}
}
if (*status == SAI__OK) {
char heateffarrid[32];
smf_fits_getS( refdata->hdr, "ARRAYID", heateffarrid, sizeof(heateffarrid), status );
if (*status != SAI__OK) errAnnul( status );
if (strcasecmp( thissub, heateffarrid ) != 0 ) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRepf("", "Subarray associated with heater efficiency image (%s)"
" does not match that of the data to be flatfielded (%s)",
status, heateffarrid, thissub );
}
}
}
}
if (heateffgrp) grpDelet( &heateffgrp, status );
}
}
if (methpar && flatmeth) {
/* See if we want to use TABLE or POLYNOMIAL mode */
parChoic( methpar, "POLYNOMIAL", "POLYNOMIAL, TABLE", 1,
method, sizeof(method), status );
*flatmeth = smf_flat_methcode( method, status );
if (*flatmeth == SMF__FLATMETH_POLY) {
/* need an order for the polynomial */
if (order && orderpar) {
parGdr0i( orderpar, 1, 1, 3, 1, order, status );
/* and if the order is 1 then we can ask for the snr min */
if (snrminpar && *order == 1) {
parGet0d( snrminpar, snrmin, status );
}
}
} else {
/* need an snr min for table mode responsivities */
if (snrminpar) parGet0d( snrminpar, snrmin, status );
}
}
if (outrows) *outrows = datarows;
if (outcols) *outcols = datacols;
CLEANUP:
resmap = astAnnul( resmap );
if (*status != SAI__OK) {
if (resistance && *resistance) *resistance = astFree( *resistance );
if (heateff && *heateff) smf_close_file( NULL, heateff, status );
}
return;
}
void smf_rebincube_nn( ThrWorkForce *wf, smfData *data, int first, int last,
int *ptime, dim_t nchan, dim_t ndet, dim_t nslice,
dim_t nxy, dim_t nout, dim_t dim[3],
int badmask, int is2d, AstMapping *ssmap,
AstSkyFrame *abskyfrm, AstMapping *oskymap,
Grp *detgrp, int moving, int usewgt, int genvar,
double tfac, double fcon, float *data_array,
float *var_array, double *wgt_array,
float *texp_array, float *teff_array, int *nused,
int *nreject, int *naccept, int *good_tsys,
int *status ){
/* Local Variables */
AstMapping *totmap = NULL; /* WCS->GRID Mapping from input WCS FrameSet */
const char *name = NULL; /* Pointer to current detector name */
const double *tsys = NULL; /* Pointer to Tsys value for first detector */
dim_t gxout; /* Output X grid index */
dim_t gyout; /* Output Y grid index */
dim_t ichan; /* Input channel index */
dim_t idet; /* detector index */
dim_t itime; /* Index of current time slice */
dim_t nchanout; /* No of spectral channels in the output */
dim_t timeslice_size; /* No of detector values in one time slice */
double *detxin = NULL; /* Work space for input X grid coords */
double *detxout = NULL; /* Work space for output X grid coords */
double *detyin = NULL; /* Work space for input Y grid coords */
double *detyout = NULL; /* Work space for output Y grid coords */
double invar; /* Input variance */
double tcon; /* Variance factor for whole time slice */
double wgt; /* Weight for input value */
float *ddata = NULL; /* Pointer to start of input detector data */
float *tdata = NULL; /* Pointer to start of input time slice data */
float *work = NULL; /* Pointer to start of work array */
float rtsys; /* Tsys value */
float teff; /* Effective integration time */
float texp; /* Total time ( = ton + toff ) */
int *nexttime; /* Pointer to next time slice index to use */
int *specpop = NULL; /* Input channels per output channel */
int *spectab = NULL; /* I/p->o/p channel number conversion table */
int first_ts; /* Is this the first time slice? */
int found; /* Was current detector name found in detgrp? */
int ignore; /* Ignore this time slice? */
int init_detector_data; /* Should detector_data be initialised? */
int iv0; /* Offset for pixel in 1st o/p spectral channel */
int jdet; /* Detector index */
int naccept_old; /* Previous number of accepted spectra */
int ochan; /* Output channel index */
int use_threads; /* Use multiple threads? */
smfHead *hdr = NULL; /* Pointer to data header for this time slice */
static smfRebincubeNNArgs1 *common_data = NULL; /* Holds data common to all detectors */
static smfRebincubeNNArgs2 *detector_data = NULL; /* Holds data for each detector */
static int *pop_array = NULL;/* I/p spectra pasted into each output spectrum */
static dim_t ndet_max = 0; /* Max number of detectors */
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Store a pointer to the input NDFs smfHead structure. */
hdr = data->hdr;
/* Store the number of pixels in one time slice */
timeslice_size = ndet*nchan;
/* Use this mapping to get the zero-based output channel number corresponding
to each input channel number. */
smf_rebincube_spectab( nchan, dim[ 2 ], ssmap, &spectab, status );
if( !spectab ) goto L999;
/* The 2D weighting scheme assumes that each output channel receives
contributions from one and only one input channel in each input file.
Create an array with an element for each output channel, holding the
number of input channels that contribute to the output channel. */
nchanout = dim[ 2 ];
if( is2d ) {
specpop = astMalloc( nchanout*sizeof( int ) );
memset( specpop, 0, nchanout*sizeof( int ) );
for( ichan = 0; ichan < nchan; ichan++ ) {
ochan = spectab[ ichan ];
if( ochan != -1 ) specpop[ ochan ]++;
}
}
/* If this is the first pass through this file, initialise the arrays. */
if( first ){
smf_rebincube_init( is2d, nxy, nout, genvar, data_array, var_array,
wgt_array, texp_array, teff_array, nused, status );
/* Allocate an extra work array and initialise it to zero. This holds the
total number of input spectra pasted into each output spectrum. It is
not needed by the AST-based function and so has not been put into
smf_rebincube_init. */
if( is2d ) {
pop_array = astMalloc( nxy*sizeof( int ) );
memset( pop_array, 0, nxy*sizeof( int ) );
}
}
/* Allocate work arrays to hold the input and output grid coords for each
detector. */
detxin = astMalloc( ndet*sizeof( double ) );
detyin = astMalloc( ndet*sizeof( double ) );
detxout = astMalloc( ndet*sizeof( double ) );
detyout = astMalloc( ndet*sizeof( double ) );
/* Initialise a string to point to the name of the first detector for which
data is available */
name = hdr->detname;
/* Fill the input arrays with the grid coords of each detector. */
for( idet = 0; idet < ndet; idet++ ) {
detxin[ idet ] = (double) idet + 1.0;
detyin[ idet ] = 1.0;
/* If a group of detectors to be used was supplied, search the group for
the name of the current detector. If not found, set the GRID coord
bad. */
if( detgrp ) {
found = grpIndex( name, detgrp, 1, status );
if( !found ) {
detxin[ idet ] = AST__BAD;
detyin[ idet ] = AST__BAD;
}
}
/* Move on to the next available detector name. */
name += strlen( name ) + 1;
}
/* Initialise a pointer to the ntex time slice index to be used. */
nexttime = ptime;
/* Count the number of time slices to be processed. */
if( ptime ) {
itime = 0;
while( ptime[ itime ] != VAL__MAXI ) itime++;
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Selecting %d time "
"slices from data file '%s'.", status, (int) itime,
data->file->name );
}
} else {
itime = nslice;
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Using all %d time "
"slices from data file '%s'.", status, (int) itime,
data->file->name );
}
}
/* Initialise the progress meter. */
smf_reportprogress( itime, status );
/* Loop round all time slices in the input NDF. */
use_threads = 0;
first_ts = 1;
for( itime = 0; itime < nslice && *status == SAI__OK; itime++ ) {
/* If this time slice is not being pasted into the output cube, pass on. */
if( nexttime ){
if( *nexttime != (int) itime ) continue;
nexttime++;
}
/* Store a pointer to the first input data value in this time slice. */
tdata = ( (float *) (data->pntr)[ 0 ] ) + itime*timeslice_size;
/* Begin an AST context. Having this context within the time slice loop
helps keep the number of AST objects in use to a minimum. */
astBegin;
/* Get a Mapping from the spatial GRID axes in the input the spatial
GRID axes in the output for the current time slice. Note this has
to be done first since it stores details of the current time slice
in the "smfHead" structure inside "data", and this is needed by
subsequent functions. */
totmap = smf_rebin_totmap( data, itime, abskyfrm, oskymap, moving,
status );
if( !totmap ) {
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Cannot get "
"Mapping for slice %d from data file '%s'.", status,
(int) itime, data->file->name );
}
astEnd;
break;
}
/* Get the effective exposure time, the total exposure time, and the
Tsys->Variance onversion factor for this time slice. Also get a
pointer to the start of the Tsys array. */
tsys = smf_rebincube_tcon( hdr, itime, fcon, &texp, &teff, &tcon,
status );
/* Use this Mapping to get the output spatial grid coords for each input
detector. */
astTran2( totmap, ndet, detxin, detyin, 1, detxout, detyout );
/* If this is the first time slice from the current input file to be pasted
into the output, see if any of the spectra will end up being pasted on
top of each other in the output. If not we can use a separate thread to
paste each spectrum. Otherwise, we cannot use multiple threads since they
may end up trying to write to the same output pixel at the same time. */
if( first_ts ) {
first_ts = 0;
use_threads = wf ? 1 : 0;
for( idet = 0; idet < ndet - 1 && use_threads; idet++ ) {
if( detxout[ idet ] != AST__BAD && detyout[ idet ] != AST__BAD ){
gxout = floor( detxout[ idet ] + 0.5 );
gyout = floor( detyout[ idet ] + 0.5 );
if( gxout >= 1 && gxout <= dim[ 0 ] &&
gyout >= 1 && gyout <= dim[ 1 ] ) {
for( jdet = idet + 1; idet < ndet; idet++ ) {
if( detxout[ jdet ] != AST__BAD &&
detyout[ jdet ] != AST__BAD ){
if( floor( detxout[ jdet ] + 0.5 ) == gxout &&
floor( detyout[ jdet ] + 0.5 ) == gyout ) {
use_threads = 0;
break;
}
}
}
}
}
}
/* If we will be using mutiple threads, do some preparation. */
if( use_threads ) {
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Using multiple "
"threads to process data file '%s'.", status,
data->file->name );
}
/* Ensure we have a structure holding information which is common to all
detectors and time slices. */
common_data = astGrow( common_data, sizeof( smfRebincubeNNArgs1 ), 1 );
if( astOK ) {
common_data->badmask = badmask;
common_data->nchan = nchan;
common_data->nchanout = nchanout;
common_data->spectab = spectab;
common_data->specpop = specpop;
common_data->nxy = nxy;
common_data->genvar = genvar;
common_data->data_array = data_array;
common_data->var_array = var_array;
common_data->wgt_array = wgt_array;
common_data->pop_array = pop_array;
common_data->nout = nout;
common_data->is2d = is2d;
}
/* Ensure we have a structure for each detector to hold the detector-specific
data, plus a pointer to the common data. */
init_detector_data = ( detector_data == NULL );
if( init_detector_data ) ndet_max = 0;
detector_data = astGrow( detector_data, sizeof( smfRebincubeNNArgs2 ),
ndet ) ;
/* Initialise pointers stored within any new elements added to the
"detector_data" array. */
if( ndet > ndet_max && astOK ) {
for( idet = ndet_max; idet < ndet; idet++ ) {
detector_data[ idet ].common = NULL;
detector_data[ idet ].work = NULL;
detector_data[ idet ].ddata = NULL;
}
ndet_max = ndet;
}
/* Allocate work space for each detector and store the common data
pointer. */
if( astOK ) {
for( idet = 0; idet < ndet; idet++ ) {
detector_data[ idet ].common = common_data;
detector_data[ idet ].work = astGrow( detector_data[ idet ].work,
sizeof( float ), nchanout );
}
}
/* If we are using a single threads, do some alternative preparation. */
} else {
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Using a single "
"thread to process data file '%s'.", status,
data->file->name );
}
/* We need an extra work array for 2D weighting that can hold a single
output spectrum. This is used as a staging post for each input
spectrum prior to pasting it into the output cube. */
work = astMalloc( nchanout*sizeof( float ) );
}
}
/* Loop round each detector, pasting its spectral values into the output
cube. */
for( idet = 0; idet < ndet; idet++ ) {
/* If multi-threaded, initialise a bad value for the detector's weight
to indicate that it is not being used. */
if( use_threads ) detector_data[ idet ].wgt = VAL__BADD;
/* See if any good tsys values are present. */
rtsys = tsys ? (float) tsys[ idet ] : VAL__BADR;
if( rtsys <= 0.0 ) rtsys = VAL__BADR;
if( rtsys != VAL__BADR ) *good_tsys = 1;
/* Check the detector has a valid position in output grid coords */
if( detxout[ idet ] != AST__BAD && detyout[ idet ] != AST__BAD ){
/* Find the closest output pixel and check it is within the bounds of the
output cube. */
gxout = floor( detxout[ idet ] + 0.5 );
gyout = floor( detyout[ idet ] + 0.5 );
if( gxout >= 1 && gxout <= dim[ 0 ] &&
gyout >= 1 && gyout <= dim[ 1 ] ) {
/* Get the offset of the output array element that corresponds to this
pixel in the first spectral channel. */
iv0 = ( gyout - 1 )*dim[ 0 ] + ( gxout - 1 );
/* If required calculate the variance associated with this detector, based on
the input Tsys values. */
invar = VAL__BADR;
if( usewgt || genvar == 2 ) {
if( rtsys != VAL__BADR ) {
if( tcon != VAL__BADD ) invar = tcon*rtsys*rtsys;
}
}
/* Calculate the weight for this detector. If we need the input variance,
either to weight the input or to calculate output variances, but the
input variance is not available, then ignore this detector. */
ignore = 0;
if( usewgt ) {
if( invar > 0.0 && invar != VAL__BADR ) {
wgt = 1.0/invar;
} else {
ignore = 1;
}
} else if( genvar == 2 ) {
ignore = ( invar <= 0.0 || invar == VAL__BADR );
wgt = 1.0;
} else {
wgt = 1.0;
}
/* If we are not ignoring this input spectrum, get a pointer to the start
of the input spectrum data and paste it into the output cube using
either the 2D or 3D algorithm. */
if( !ignore ) {
ddata = tdata + idet*nchan;
/* First deal with cases where we are using a single thread (the current
thread). */
if( !use_threads ) {
naccept_old = *naccept;
if( is2d ) {
smf_rebincube_paste2d( badmask, nchan, nchanout, spectab,
specpop, iv0, nxy, wgt, genvar,
invar, ddata, data_array,
var_array, wgt_array, pop_array,
nused, nreject, naccept, work,
status );
} else {
smf_rebincube_paste3d( nchan, nout, spectab, iv0, nxy,
wgt, genvar, invar, ddata,
data_array, var_array,
wgt_array, nused, status );
(*naccept)++;
}
/* Now we update the total and effective exposure time arrays for the
output spectrum that receives this input spectrum. Scale the exposure
times of this time slice in order to reduce its influence on the
output expsoure times if it does not have much spectral overlap with
the output cube. Only update the exposure time arrays if the spectrum
was used (as shown by an increase in the number of accepted spectra). */
if( texp != VAL__BADR && *naccept > naccept_old ) {
texp_array[ iv0 ] += texp*tfac;
teff_array[ iv0 ] += teff*tfac;
}
/* Now deal with cases where we are using several threads. */
} else {
/* Set up the detector specific data. */
detector_data[ idet ].iv0 = iv0;
detector_data[ idet ].wgt = wgt;
detector_data[ idet ].invar = invar;
detector_data[ idet ].ddata = ddata;
detector_data[ idet ].nused = 0;
detector_data[ idet ].nreject = 0;
detector_data[ idet ].naccept = 0;
/* Add a job to the workforce's job list. This job calls smf_rebincube_paste2d
or smf_rebincube_paste3d to paste the detector input spectrum into the
output cube. */
thrAddJob( wf, 0, detector_data + idet,
smf_rebincube_paste_thread, 0, NULL, status );
}
} else if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Detector %d "
"is being ignored when processing data file '%s'.",
status, idet, data->file->name );
}
} else if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Detector %d "
"fell outside the output cube when processing "
"data file '%s'.", status, idet, data->file->name );
}
} else if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Detector %d has "
"an unknown position in the output cube when processing "
"data file '%s'.", status, idet, data->file->name );
}
}
/* If using multiple threads, wait until all spectra for this time slice
have been pasted into the output cube. Then transfer the output values
from the detector data structures to the returned variables. */
if( use_threads ) {
thrWait( wf, status );
for( idet = 0; idet < ndet; idet++ ) {
if( detector_data[ idet ].wgt != VAL__BADD ) {
(*nused) += detector_data[ idet ].nused;
(*nreject) += detector_data[ idet ].nreject;
(*naccept) += detector_data[ idet ].naccept;
if( texp != VAL__BADR && detector_data[ idet ].naccept > 0 ) {
texp_array[ detector_data[ idet ].iv0 ] += texp*tfac;
teff_array[ detector_data[ idet ].iv0 ] += teff*tfac;
}
}
}
}
/* Update the progress meter. */
smf_reportprogress( 0, status );
/* End the AST context. */
astEnd;
}
/* If this is the final pass through this function, normalise the returned
data and variance values, and release any static resources allocated
within this function. */
if( last ) {
if( is2d ) {
smf_rebincube_norm2d( nout, nxy, genvar, data_array,
var_array, wgt_array, pop_array, status );
} else {
smf_rebincube_norm3d( nout, genvar, *nused, data_array,
var_array, wgt_array, status );
}
pop_array = astFree( pop_array );
if( use_threads ) {
common_data = astFree( common_data );
for( idet = 0; idet < ndet_max; idet++ ) {
detector_data[ idet ].work = astFree( detector_data[ idet ].work );
}
detector_data = astFree( detector_data );
}
}
/* Free non-static resources. */
L999:;
work = astFree( work );
spectab = astFree( spectab );
specpop = astFree( specpop );
detxin = astFree( detxin );
detyin = astFree( detyin );
detxout = astFree( detxout );
detyout = astFree( detyout );
}
void smf_clipnoise( double *clipdata, size_t ndata, int cliplog,
double cliplow, double cliphigh, size_t *nclipped,
int *status ) {
const float clips[] = {5,3,1};
const size_t nclips = sizeof(clips)/sizeof(*clips);
size_t i;
size_t nlow=0;
size_t nhigh=0;
double *work=NULL;
double median, mean, sigma;
if( *status != SAI__OK ) return;
if( cliplog ) msgOutif( MSG__DEBUG, "", FUNC_NAME
": taking log10 of the data", status );
if( (cliphigh > 0) || (cliplow > 0 ) ) {
work = astCalloc( ndata, sizeof(*work) );
/* Copy the data, or its log, into a buffer */
if( *status == SAI__OK ) {
for( i=0; i<ndata; i++ ) {
if( (clipdata[i] != VAL__BADD) && (clipdata[i] > 0) ) {
work[i] = cliplog ? log10(clipdata[i]) : clipdata[i];
} else {
work[i] = VAL__BADD;
}
}
}
/* Measure the clipped median, mean and standard deviation. We
step down from 5- to 1-sigma, using the median rather than the
mean as our central measure to ensure robustness against large
outliers. This should end up near the mode of the distribution,
although the RMS of the sample will under-estimate, by nearly a
factor of 2, the standard deviation for a Gaussian distribution
since we've clipped so much of the wings. We scale it back up
so that it would give the right answer for a Gaussian. */
smf_clipped_stats1D( work, nclips, clips, 1, ndata, NULL, 0, 0,
&mean, &sigma, &median, 1, NULL, status );
/* Assume that we do not need to clip if we can not get good
statistics */
if (*status == SMF__INSMP) {
errAnnul( status );
msgOutif(MSG__NORM, "",
"Noise clipping disabled as there are too few bolometers",
status );
goto CLEANUP;
}
sigma *= 1.85;
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": mean=%lg median=%lg standard dev=%lg",
status, mean, median, sigma );
/* Then clip the high/low outliers relative to the median */
if( *status==SAI__OK ) {
for( i=0; i<ndata; i++ ) {
if( work[i] != VAL__BADD ) {
double d = work[i] - median;
if( (cliphigh>0) &&
(d >= sigma*cliphigh) ) {
clipdata[i] = VAL__BADD;
nhigh++;
}
if( (cliplow>0) &&
(-d >= sigma*cliplow) ) {
clipdata[i] = VAL__BADD;
nlow++;
}
}
}
}
if( nhigh ) msgOutiff( MSG__VERB, "", FUNC_NAME
": clipped %zu values >= %lg",
status, nhigh,
cliplog ? pow(10,median + sigma*cliphigh) :
median + sigma*cliphigh );
if( nlow ) msgOutiff( MSG__VERB, "", FUNC_NAME
": clipped %zu values <= %lg",
status, nlow,
cliplog ? pow(10,median - sigma*cliplow) :
median - sigma*cliplow );
/* Return number of clipped values */
if( nclipped ) *nclipped = nhigh+nlow;
CLEANUP:
/* Free temporary buffer */
if( work ) work = astFree( work );
}
}
void cupidFindback0( void *data, int *status ){
/*
*+
* Name:
* cupidFindback0
* Purpose:
* Initiate the spatial filtering of a 3D array within a thread.
* Language:
* Starlink C
* Synopsis:
* void cupidFindback0( void *data, int *status ){
* Description:
* This is the main function that gets run in a worker thread in order
* to filter a slice of the supplied base NDF.
* Parameters:
* data
* Pointer to the data structure containing the information needed
* by this function.
* status
* Pointer to the inherited status value.
* Copyright:
* Copyright (C) 2011 Science & Technology Facilities Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* {enter_new_authors_here}
* History:
* 13-SEP-2011 (DSB):
* Original version.
* {enter_further_changes_here}
*-
*/
/* Local Variables: */
CupidFindback0Data *pdata;/* Pointer to structure holding requied info */
double rms; /* Global rms error in data */
int box[ 3 ]; /* Dimensions of each cell in pixels */
int islice; /* Slice index */
int ndim; /* Total number of pixel axes in NDF */
int newalg; /* Use experimental algorithm variations? */
int nslice; /* Number of slices to process */
int slice_dim[ 3 ]; /* Dimensions of each significant slice axis */
int slice_lbnd[ 3 ]; /* Lower bounds of each significant slice axis */
int slice_size; /* Number of pixels in each slice */
int type; /* Integer identifier for data type */
void *ipd1; /* Pointer to input Data array */
void *ipd2; /* Pointer to output Data array */
void *wa; /* Pointer to work array */
void *wb; /* Pointer to work array */
/* Abort if an error has already occurred. */
if( *status != SAI__OK ) return;
/* Get a pointer to the data structure, with the expected type. */
pdata = (CupidFindback0Data *) data;
/* Store the information needed by the function (cupidFindback0) that
does the work in a thread. */
box[ 0 ] = pdata->box[ 0 ];
box[ 1 ] = pdata->box[ 1 ];
box[ 2 ] = pdata->box[ 2 ];
ipd1 = pdata->ipd1;
ipd2 = pdata->ipd2;
islice = pdata->islice;
ndim = pdata->ndim;
nslice = pdata->nslice;
rms = pdata->rms;
slice_dim[ 0 ] = pdata->slice_dim[ 0 ];
slice_dim[ 1 ] = pdata->slice_dim[ 1 ];
slice_dim[ 2 ] = pdata->slice_dim[ 2 ];
slice_lbnd[ 0 ] = pdata->slice_lbnd[ 0 ];
slice_lbnd[ 1 ] = pdata->slice_lbnd[ 1 ];
slice_lbnd[ 2 ] = pdata->slice_lbnd[ 2 ];
slice_size = pdata->slice_size;
type = pdata->type;
newalg = pdata->newalg;
/* Report the bounds of the slice if required. */
msgBlankif( MSG__VERB, status );
msgOutiff( MSG__VERB, "", " Processing slice %d of %d...", status,
islice+1, nslice );
msgBlankif( MSG__VERB, status );
/* Process this slice, then increment the pointer to the next slice. */
if( type == CUPID__FLOAT ) {
wa = astMalloc( sizeof( float )*slice_size );
wb = astMalloc( sizeof( float )*slice_size );
cupidFindback1F( ndim, slice_dim, slice_lbnd, box, rms, ipd1, ipd2,
wa, wb, newalg, status );
} else {
wa = astMalloc( sizeof( double )*slice_size );
wb = astMalloc( sizeof( double )*slice_size );
cupidFindback1D( ndim, slice_dim, slice_lbnd, box, rms, ipd1, ipd2,
wa, wb, newalg, status );
}
/* Free workspace. */
wa = astFree( wa );
wb = astFree( wb );
}
void smf_coords_lut( smfData *data, int tstep, dim_t itime_lo,
dim_t itime_hi, AstSkyFrame *abskyfrm,
AstMapping *oskymap, int moving, int olbnd[ 2 ],
int oubnd[ 2 ], fts2Port fts_port, int *lut,
double *angle,
double *lon, double *lat, int *status ) {
/* Local Variables */
AstCmpMap *bsmap = NULL; /* Tracking -> output grid Mapping */
AstFrame *trfrm = NULL; /* Tracking Frame */
AstFrameSet *fs = NULL; /* Tracking -> output sky FrameSet */
AstMapping *fullmap = NULL; /* Full Mapping from bolo GRID to output map GRID */
AstMapping *offmap = NULL; /* Mapping from absolute to offset sky coords */
AstMapping *tr2skyabs = NULL;/* Tracking -> output sky Mapping */
AstSkyFrame *offsky = NULL; /* Offset sky frame */
JCMTState *state; /* Pointer to telescope info for time slice */
dim_t ibolo; /* Vector index of bolometer */
dim_t idimx; /* Bolometers per row */
dim_t idimy; /* Bolometers per column */
dim_t ilut; /* Index of LUT element */
dim_t itime0; /* Time slice index at next full calculation */
dim_t itime; /* Time slice index */
dim_t nbolo; /* Total number of bolometers */
double *outmapcoord; /* Array holding output map GRID coords */
double *px; /* Pointer to next output map X GRID coord */
double *py; /* Pointer to next output map Y GRID coord */
double *pgx; /* Pointer to next X output grid coords */
double *pgy; /* Pointer to next Y output grid coords */
double *wgx = NULL; /* Work space to hold X output grid coords */
double *wgy = NULL; /* Work space to hold Y output grid coords */
double bsx0; /* Boresight output map GRID X at previous full calc */
double bsx; /* Boresight output map GRID X at current time slice */
double bsxlast; /* Boresight output map GRID X at previous time slice*/
double bsy0; /* Boresight output map GRID Y at previous full calc */
double bsy; /* Boresight output map GRID Y at current time slice */
double bsylast; /* Boresight output map GRID Y at previous time slice*/
double dx; /* Offset in GRID X from previous full calc */
double dxlast; /* Offset in GRID X from previous time slice */
double dy; /* Offset in GRID Y from previous full calc */
double dylast; /* Offset in GRID Y from previous time slice */
double shift[ 2 ]; /* Shift from PIXEL to GRID in output map */
double x; /* Output GRID X at current bolo in current row */
double xin[ 2 ]; /* Input X values */
double xout[ 2 ]; /* Output X values */
double y; /* Output GRID Y at current bolo in current row */
double yin[ 2 ]; /* Input Y values */
double yout[ 2 ]; /* Output Y values */
int lbnd_in[ 2 ]; /* Lower bounds of input array */
int np; /* Number of positions to transform */
int odimx; /* Output map X dimension in pixels */
int odimy; /* Output map Y dimension in pixels */
int ox; /* Output X GRID index (-1) containing current bolo */
int oy; /* Output Y GRID index (-1) containing current bolo */
int ubnd_in[ 2 ]; /* Upper bounds of input array */
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Begin an AST context. */
astBegin;
/* Get the dimensions of the output map. */
odimx = oubnd[ 0 ] - olbnd[ 0 ] + 1;
odimy = oubnd[ 1 ] - olbnd[ 1 ] + 1;
/* Get the dimensions of the bolometer array. */
idimx = (data->dims)[ 0 ];
idimy = (data->dims)[ 1 ];
/* Store integer bounds within the input bolometer GRID system. */
lbnd_in[ 0 ] = 1;
lbnd_in[ 1 ] = 1;
ubnd_in[ 0 ] = idimx;
ubnd_in[ 1 ] = idimy;
/* Determine the number of bolometers. */
nbolo = idimx*idimy;
/* Initialise the index of the next LUT element to write. */
ilut = 0;
/* Ensure tstep is at least one. */
if( tstep < 1 ) tstep = 1;
/* Get the time slice index at which to do the next full calculation. */
itime0 = itime_lo;
/* We only need the following AST objects if we will be approximating
some caclulations. */
if( tstep > 1 ) {
/* We need to find the first good TCS index in order to get the
tracking system (Which for SCUBA-2 won't be changing in the sequence) */
itime0 = VAL__BADI;
for (itime = itime_lo; itime <= itime_hi; itime++) {
JCMTState * slice = &((data->hdr->allState)[itime]);
if (!(slice->jos_drcontrol >= 0 && slice->jos_drcontrol & DRCNTRL__POSITION)) {
itime0 = itime;
break;
}
}
if ((int)itime0 != VAL__BADI) {
/* We need a Frame describing absolute tracking system coords. Take a
copy of the supplied skyframe (to inherit obslat, obslon, epoch,
etc), and then set its system to the tracking system. */
trfrm = astCopy( abskyfrm );
astSetC( trfrm, "System",
sc2ast_convert_system( (data->hdr->allState)[itime0].tcs_tr_sys,
status ) );
/* Get the Mapping from the tracking system to the output (absolute, since
abskyfrm is absolute) sky system. */
fs = astConvert( trfrm, abskyfrm, " " );
if( !fs && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( " ", "smf_coords_lut: Failed to convert from "
"tracking system to output WCS system.", status );
}
tr2skyabs = astSimplify( astGetMapping( fs, AST__BASE, AST__CURRENT ) );
/* For moving targets, we also need a Frame describing offset sky
coordinates in the output map, in which the reference point is the
current telescope base position. This will involve changing the
SkyRef attribute of the Frame for every time slice, so take a
copy of the supplied SkyFrame to avoid changing it. */
if( moving ) {
offsky = astCopy( abskyfrm );
astSet( offsky, "SkyRefIs=Origin" );
}
/* Create the Mapping from offsets within the output map sky coordinate
system output to map GRID coords to output map GRID coords. This uses a
ShiftMap to convert from output PIXEL coords (produced by "oskymap") to
output GRID coords. Note, if the target is moving, "oskymap" maps from
sky *offsets* to output map PIXEL coords. */
shift[ 0 ] = 1.5 - olbnd[ 0 ];
shift[ 1 ] = 1.5 - olbnd[ 1 ];
bsmap = astCmpMap( oskymap, astShiftMap( 2, shift, " " ), 1, " " );
} else {
/* We did not find any good TCS data so force us into tstep==1
case and end up with VAL__BADI for every element. */
tstep = 1;
itime0 = itime_lo;
msgOutiff(MSG__VERB, "", "All time slices from %zu -- %zu had bad TCS data.",
status, (size_t)itime_lo, (size_t)itime_hi );
}
}
/* Allocate memory to hold the (x,y) output map grid coords at each bolo
for a single time slice. */
outmapcoord = astMalloc( sizeof( *outmapcoord )*2*nbolo );
/* Initialise boresight position for the benefit of the tstep == 1 case. */
bsx = bsy = 0.0;
bsx0 = bsy0 = AST__BAD;
bsxlast = bsylast = AST__BAD;
/* If lon and lat arrays are to be returned, allocate memory to hold the grid
coords of every bolometer for a single sample. */
if( lon && lat ) {
wgx = astMalloc( nbolo*sizeof( *wgx ) );
wgy = astMalloc( nbolo*sizeof( *wgy ) );
}
/* Loop round each time slice. */
state = data->hdr->allState + itime_lo;
for( itime = itime_lo; itime <= itime_hi && *status == SAI__OK;
itime++,state++ ) {
/* No need to get the boresight position if we are doing full
calculations at every time slice. If this time slice has bad
TCS data then the problem will be caught in smf_rebin_totmap. */
if( tstep > 1 ) {
/* Transform the current boresight and base (if moving) positions from
tracking coords to absolute output map sky coords. */
xin[ 0 ] = state->tcs_tr_ac1;
yin[ 0 ] = state->tcs_tr_ac2;
if( moving ) {
xin[ 1 ] = state->tcs_tr_bc1;
yin[ 1 ] = state->tcs_tr_bc2;
np = 2;
} else {
np = 1;
}
astTran2( tr2skyabs, np, xin, yin, 1, xout, yout );
/* If the target is moving, find the offsets within the output map sky
coordinate system, from base to boresight at the current time slice.
These offsets become the new "boresight" position (in xin/yin). Guard
against assigning bad values to the ref pos, which can cause AST to
report errors. */
if( moving && xout[ 1 ] != AST__BAD && yout[ 1 ] != AST__BAD ) {
/* Set the current telescope base position as the reference point in
"offsky". Then get the Mapping from absolute to offset sky coords. */
astSetD( offsky, "SkyRef(1)", xout[ 1 ] );
astSetD( offsky, "SkyRef(2)", yout[ 1 ] );
offmap = astSkyOffsetMap( offsky );
/* Use this Mapping to convert the current boresight position from
absolute sky coords to offsets from the current base position. */
astTran2( offmap, 1, xout, yout, 1, xin, yin );
/* Annul the Mapping to avoid keep the number of AST objects to a minimum. */
offmap = astAnnul( offmap );
/* If the target is stationary, we can just use the absolute boresight
position as it is. */
} else {
xin[ 0 ] = xout[ 0 ];
yin[ 0 ] = yout[ 0 ];
}
/* Transform the above boresight position from output map sky coords to output
map GRID coords. */
astTran2( bsmap, 1, xin, yin, 1, &bsx, &bsy );
}
/* If we have reached the next full calculation... */
if( itime == itime0 ) {
/* Calculate the full bolometer to map-pixel transformation for the current
time slice */
fullmap = smf_rebin_totmap( data, itime, abskyfrm, oskymap, moving,
fts_port, status );
/* If succesful, use it to transform every bolometer position from bolo
GRID coords to output map GRID coords. */
if( fullmap ) {
itime0 += tstep;
astTranGrid( fullmap, 2, lbnd_in, ubnd_in, 0.1, 1000000, 1,
2, nbolo, outmapcoord );
fullmap = astAnnul( fullmap );
/* Record the boresight grid coords at this time slice. */
bsx0 = bsx;
bsy0 = bsy;
/* If we cannot determine a full Mapping for this time slice, move the
node to the next time slice (otherwise we would loose all the data to
the next node), and set the boresight position bad to indicate we
have no mapping for this time slice. */
} else {
itime0++;
bsx0 = AST__BAD;
bsy0 = AST__BAD;
}
}
/* Get the offset from the boresight position at the previous full
calculation and the current boresight position, in output map GRID
coords. */
dx = ( bsx != AST__BAD && bsx0 != AST__BAD ) ? bsx - bsx0 : AST__BAD;
dy = ( bsy != AST__BAD && bsy0 != AST__BAD ) ? bsy - bsy0 : AST__BAD;
/* Work out the scan direction based on the GRID offsets between this and
the previous time slice. Angles are calculated using atan2, with values
ranging from -pi to +pi. */
if( angle ) {
double theta = AST__BAD;
dxlast = ( bsx != AST__BAD && bsxlast != AST__BAD ) ?
bsx - bsxlast : AST__BAD;
dylast = ( bsy != AST__BAD && bsylast != AST__BAD ) ?
bsy - bsylast : AST__BAD;
if( dxlast != AST__BAD && dylast != AST__BAD &&
!( !dxlast && !dylast ) ) {
theta = atan2( dylast, dxlast );
}
angle[itime-itime_lo] = theta;
bsxlast = bsx;
bsylast = bsy;
}
/* Initialise pointers to the place to store the final grid coords for
each bolometer. */
pgx = wgx;
pgy = wgy;
/* Loop round all bolometers. */
px = outmapcoord;
py = outmapcoord + nbolo;
for( ibolo = 0; ibolo < nbolo; ibolo++ ){
/* If good, get the x and y output map GRID coords for this bolometer. */
if( dx != AST__BAD && dy != AST__BAD ) {
x = *(px++) + dx;
y = *(py++) + dy;
/* If required, store them so that we can convert them into lon/lat
values later. */
if( pgx && pgy ) {
*(pgx++) = x;
*(pgy++) = y;
}
/* Find the grid indices (minus one) of the output map pixel containing the
mapped bolo grid coords. One is subtracted in order to simplify the
subsequent calculation of the vector index. */
ox = (int) ( x - 0.5 );
oy = (int) ( y - 0.5 );
/* Check it is within the output map */
if( ox >= 0 && ox < odimx && oy >= 0 && oy < odimy ) {
/* Find the 1-dimensional vector index into the output array for this
pixel and store in the next element of the returned LUT. */
lut[ ilut++ ] = ox + oy*odimx;
/* Store a bad value for points that are off the edge of the output map. */
} else {
lut[ ilut++ ] = VAL__BADI;
}
/* If good, store a bad index in the LUT and move on to the next pixel. */
} else {
lut[ ilut++ ] = VAL__BADI;
px++;
py++;
if( pgx && pgy ) {
*(pgx++) = AST__BAD;
*(pgy++) = AST__BAD;
}
}
}
/* If required transform the grid coords into (lon,lat) coords and store in
the relevant elements of the supplied "lon" and "lat" arrays. */
if( wgx && wgy ) {
astTran2( oskymap, nbolo, wgx, wgy, 0, lon + itime*nbolo,
lat + itime*nbolo );
/* Convert from rads to degs. */
pgx = lon + itime*nbolo;
pgy = lat + itime*nbolo;
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
if( *pgx != AST__BAD && *pgy != AST__BAD ) {
*(pgx++) *= AST__DR2D;
*(pgy++) *= AST__DR2D;
} else {
pgx++;
pgy++;
}
}
}
}
/* To obtain a reasonable value for the first entry of angle, we simply
duplicate the second value. */
if( angle && (itime_hi > itime_lo) ) {
angle[0] = angle[1];
}
/* Free remaining work space. */
outmapcoord = astFree( outmapcoord );
wgx = astFree( wgx );
wgy = astFree( wgy );
/* Export the WCS pointer in the data header since it will be annulled at
a higher level. Note the FrameSet pointer may be null if the last
full calculation was for a slice with bad telescope data. */
if( data->hdr->wcs ) astExport( data->hdr->wcs );
/* End the AST context. */
astEnd;
}
size_t smf_get_freemem ( double *mbytes, size_t * pagesize,
int64_t * physsize, int * status ) {
int64_t mem_used = 0;
int64_t mem_free = 0;
int64_t mem_total = 0;
double freembytes = 0.0;
if (*status != SAI__OK) return mem_free;
# if HAVE_MACH_VM
{
mach_port_t host_port;
mach_msg_type_number_t host_size;
vm_size_t vmpagesize;
vm_statistics_data_t vm_stat;
host_port = mach_host_self();
host_size = sizeof(vm_statistics_data_t) / sizeof(integer_t);
host_page_size(host_port, &vmpagesize);
if (pagesize) *pagesize = vmpagesize;
if (host_statistics(host_port, HOST_VM_INFO, (host_info_t)&vm_stat, &host_size) == KERN_SUCCESS) {
/* Stats in bytes */
mem_used = (vm_stat.active_count +
vm_stat.inactive_count +
vm_stat.wire_count) * vmpagesize;
mem_free = vm_stat.free_count * vmpagesize;
mem_total = mem_used + mem_free;
}
}
#else
#ifdef _SC_AVPHYS_PAGES
/* Figure out available physical memory from sysconf */
{
size_t mypagesize;
mypagesize = sysconf(_SC_PAGE_SIZE);
mem_free = sysconf(_SC_AVPHYS_PAGES) * mypagesize;
mem_total = sysconf(_SC_PHYS_PAGES) * mypagesize;
mem_used = mem_total - mem_free;
if (pagesize) *pagesize = mypagesize;
}
#endif /* SC_AVPHYS_PAGES */
#endif /* MACH_VM */
if (mem_free > 0) {
freembytes = (double) mem_free / (double)SMF__MIB;
msgOutiff( MSG__DEBUG, "", "Free memory: %g MB Used Memory: %g MB Total Memory: %g MB", status,
freembytes, (double)mem_used/(double)SMF__MIB, (double)mem_total/(double)SMF__MIB
);
} else {
msgOutif( MSG__DEBUG,"", "Unable to determine free memory", status );
}
/* sort out return values */
if (mbytes) *mbytes = freembytes;
if (physsize) *physsize = mem_total;
return mem_free;
}
void smf_rebincube_ast( ThrWorkForce *wf, smfData *data, int first, int last,
int *ptime, dim_t nchan, dim_t ndet, dim_t nslice,
dim_t nel, dim_t nxy, dim_t nout, dim_t dim[3],
AstMapping *ssmap, AstSkyFrame *abskyfrm,
AstMapping *oskymap, Grp *detgrp, int moving,
int usewgt, int spread, const double params[],
int genvar, double tfac, double fcon,
float *data_array, float *var_array,
double *wgt_array, float *texp_array,
float *teff_array, int *good_tsys, int64_t *nused,
int *status ){
/* Local Variables */
AstCmpMap *detmap = NULL; /* Mapping from 1D det. index to 2D i/p "grid" coords */
AstMapping *dtotmap = NULL; /* 1D det index->o/p GRID Mapping */
AstMapping *fullmap = NULL; /* WCS->GRID LutMap from input WCS FrameSet */
AstMapping *lutmap = NULL; /* Mapping that identifies detectors to be used */
AstMapping *splut = NULL; /* Spatial LutMap */
AstMapping *sslut = NULL; /* Spectral LutMap */
AstMapping *totmap = NULL; /* WCS->GRID Mapping from input WCS FrameSet */
AstPermMap *pmap; /* Mapping to rearrange output axes */
const char *name = NULL; /* Pointer to current detector name */
const double *tsys = NULL; /* Pointer to Tsys value for first detector */
dim_t iv; /* Vector index into output 3D array */
double *detlut = NULL; /* Work space for detector mask */
double blk_bot[ 2*MAXTHREADS + 1 ]; /* First o/p channel no. in each block */
double con; /* Constant value */
double dtemp; /* Temporary value */
double tcon; /* Variance factor for whole time slice */
float *detwork = NULL; /* Work array for detector values */
float *tdata = NULL; /* Pointer to start of input time slice data */
float *varwork = NULL; /* Work array holding variances for 1 slice/channel */
float *vp = NULL; /* Pointer to next "varwork" element */
float invar; /* Input variance */
float rtsys; /* Tsys value */
float teff; /* Effective integration time */
float texp; /* Total time ( = ton + toff ) */
int *nexttime; /* Pointer to next time slice index to use */
int ast_flags; /* Basic flags to use with astRebinSeq */
int blk_size; /* Number of channels processed by a single thread */
int found; /* Was current detector name found in detgrp? */
int iblock; /* Index of current spectral block */
dim_t ichan; /* Index of current channel */
dim_t idet; /* detector index */
int ignore; /* Ignore this time slice? */
int inperm[ 3 ]; /* Input axis permutation array */
dim_t itime; /* Index of current time slice */
int64_t junk; /* Unused parameter */
int lbnd_in[ 2 ]; /* Lower input bounds on receptor axis */
int ldim[ 3 ]; /* Output array lower GRID bounds */
int maxthreads; /* Max no. of threads to use when re-binning */
int nblock; /* Number of spectral blocks */
int nthreads; /* Number of threads to use when re-binning */
int outperm[ 3 ]; /* Output axis permutation array */
int timeslice_size; /* Number of elements in a time slice */
int ubnd_in[ 2 ]; /* Upper input bounds on receptor axis */
int uddim[ 1 ]; /* Detector array upper GRID bounds */
int udim[ 3 ]; /* Output array upper GRID bounds */
smfHead *hdr = NULL; /* Pointer to data header for this time slice */
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Store a pointer to the input NDFs smfHead structure. */
hdr = data->hdr;
/* Fill an array with the lower grid index bounds of the output. */
ldim[ 0 ] = 1;
ldim[ 1 ] = 1;
ldim[ 2 ] = 1;
/* Integer upper grid index bounds of the output. */
udim[ 0 ] = dim[ 0 ];
udim[ 1 ] = dim[ 1 ];
udim[ 2 ] = dim[ 2 ];
/* Integer upper bounds of detector array. */
uddim[ 0 ] = ndet;
/* Store the size of an input time slice. */
timeslice_size = nel/nslice;
/* Create a LutMap that holds the output spectral axis GRID value at
the centre of each input spectral axis pixel. LutMaps are faster to
evaluate, and so astRebinSeq will go faster. We can use LutMaps without
loosing accuracy since astRebinSeq only ever transforms the GRID
values at input pixel centres (i.e. integer GRID values), and so the
LutMap will always return a tabulated value rather than an
interpolated value. */
atlTolut( (AstMapping *) ssmap, 1.0, (double) nchan, 1.0, "LutInterp=1",
&sslut, status );
/* If this is the first pass through this file, initialise the arrays. */
if( first ) smf_rebincube_init( 0, nxy, nout, genvar, data_array, var_array,
wgt_array, texp_array, teff_array, &junk, status );
/* Initialisation the flags for astRebinSeq (we do not include flag
AST__REBININIT because the arrays have been initialised). */
ast_flags = AST__USEBAD;
if( usewgt ) ast_flags = ast_flags | AST__VARWGT;
if( genvar == 1 ) {
ast_flags = ast_flags | AST__GENVAR;
} else if( genvar == 2 ) {
ast_flags = ast_flags | AST__USEVAR;
}
/* If required, allocate a work array to hold all the input variances for a
single time slice. */
if( usewgt || genvar == 2 ) varwork = astMalloc( timeslice_size * sizeof( float ) );
/* Allocate a work array to hold the exposure time for each detector. */
detwork = astMalloc( ndet * sizeof( float ) );
/* Debug message */
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_ast: Using %zu detectors "
"from data file '%s'.", status, ndet, data->file->name );
}
/* If we are dealing with more than 1 detector, create a LutMap that holds
the input GRID index of every detector to be included in the output, and
AST__BAD for every detector that is not to be included in the output cube.
First allocate the work space for the LUT. */
if( ndet > 1 ) {
detlut = astMalloc( ndet*sizeof( double ) );
/* Initialise a string to point to the name of the first detector for which
data is available */
name = hdr->detname;
/* Loop round all detectors for which data is available. */
for( idet = 0; idet < ndet; idet++ ) {
/* Store the input GRID coord of this detector. GRID coords start at 1,
not 0. */
detlut[ idet ] = idet + 1.0;
/* If a group of detectors to be used was supplied, search the group for
the name of the current detector. If not found, set the GRID coord bad.
This will cause astRebinSeq to ignore data from the detector. */
if( detgrp ) {
found = grpIndex( name, detgrp, 1, status );
if( !found ) detlut[ idet ] = AST__BAD;
}
/* Move on to the next available detector name. */
name += strlen( name ) + 1;
}
/* Create the LutMap. */
lutmap = (AstMapping *) astLutMap( ndet, detlut, 1.0, 1.0,
"LutInterp=1" );
/* If we only have 1 detector, use a UnitMap instead of a LutMap (lutMaps
must have 2 or more table entries). */
} else {
lutmap = (AstMapping *) astUnitMap( 1, " " );
}
/* Combine the above LutMap with a 1-input, 2-output PermMap that copies its
input to create its first output, and assigns a constant value of 1.0 to
its second output. We need to do this because smf_tslice returns a 2D
GRID system (even though the second GRID axis is not actually used). */
inperm[ 0 ] = 1;
outperm[ 0 ] = 1;
outperm[ 1 ] = -1;
con = 1.0;
detmap = astCmpMap( lutmap, astPermMap( 1, inperm, 2, outperm, &con, " " ),
1, " " );
/* Store the bounds of a single time slice grid. */
lbnd_in[ 0 ] = 1;
ubnd_in[ 0 ] = nchan;
lbnd_in[ 1 ] = 1;
ubnd_in[ 1 ] = ndet;
/* Create a PermMap that can be used to re-order the output axes so that
channel number is axis 3. */
outperm[ 0 ] = 2;
outperm[ 1 ] = 3;
outperm[ 2 ] = 1;
inperm[ 0 ] = 3;
inperm[ 1 ] = 1;
inperm[ 2 ] = 2;
pmap = astPermMap( 3, inperm, 3, outperm, NULL, " " );
/* If we are using multiple threads to rebin spectral blocks in parallel,
calculate the number of channels that are processed by each thread,
and the number of threads to use. The whole output spectrum is divided
up into blocks. The number of blocks is two times the number of
threads, and each thread rebins two adjacent blocks. Alternate blocks
are re-binned simultanously. First, the odd numbered blocks are re-binned
(one by each thread). When all odd numbered blocks have been re-binned,
the even numbered blocks are re-binned. We ensure that the number of
threads used results in a block size that is larger than the spreading
width produced by the requested spreading scheme. This means that no
pair of simultanously executing threads will ever try to write to the
same channel of the output spectrum. */
maxthreads = wf ? wf->nworker : 1;
if( maxthreads > MAXTHREADS ) maxthreads = MAXTHREADS;
if( maxthreads > 1 ) {
/* Find the largest number of threads into which each output spectrum can
be split. The limit is imposes by the requirement that each block is
larger than the pixel spreading produced by the requested spreading
scheme. */
nthreads = ( ( dim[ 2 ] + 1 )/2 )/smf_spreadwidth( spread, params,
status );
/* If the spectral range is less than twice the spreading width, we
cannot use multiple threads. */
if( nthreads > 1 ) {
/* Restrict the number of threads to be no more than the number of workers
available in the work force. */
if( nthreads > maxthreads ) nthreads = maxthreads;
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_ast: Using %d threads "
"to process data file '%s'.", status, nthreads, data->file->name );
}
/* Find the number of output channels in each spectral block. */
blk_size = ( dim[ 2 ] - 1 )/( 2*nthreads ) + 1;
/* Set up the first output channel number within each block. */
nblock = 2*nthreads;
for( iblock = 0; iblock < nblock; iblock++ ) {
blk_bot[ iblock ] = (double) ( iblock*blk_size + 1 );
}
/* Add in the first channel number beyond the last block. */
blk_bot[ nblock ] = blk_bot[ nblock - 1 ] + blk_size;
/* If the output spectrum is too short to guarantee that there are any
independent blocks of output channels, we process the whole spectrum
in a single thread. */
} else {
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_ast: Using one thread "
"to process data file '%s'.", status, data->file->name );
}
nthreads = 1;
nblock = 1;
blk_bot[ 0 ] = 1.0;
blk_bot[ 1 ] = (double) ( dim[ 2 ] + 1 );
}
/* If multiple threads are not available, we process the whole spectrum
in a single thread. */
} else {
nthreads = 1;
nblock = 1;
blk_bot[ 0 ] = 1.0;
blk_bot[ 1 ] = (double) ( dim[ 2 ] + 1 );
}
/* Convert the block boundaries from output channel numbers into input
channel numbers. */
astTran1( ssmap, nblock + 1, blk_bot, 0, blk_bot );
/* Ensure they are in increasing order, and are not outside the bounds of
the input array. */
if( blk_bot[ 0 ] > blk_bot[ 1 ] ) {
for( iblock = 0; iblock < ( nblock + 1 )/2; iblock++ ) {
dtemp = blk_bot[ nblock - iblock ];
blk_bot[ nblock - iblock ] = blk_bot[ iblock ];
blk_bot[ iblock ] = dtemp;
}
}
for( iblock = 0; iblock <= nblock; iblock++ ) {
if( blk_bot[ iblock ] < 1 ) {
blk_bot[ iblock ] = 1.0;
} else if( blk_bot[ iblock ] > nchan ) {
blk_bot[ iblock ] = nchan;
}
}
/* Count the number of time slices to be processed. */
if( ptime ) {
itime = 0;
while( ptime[ itime ] != VAL__MAXI ) itime++;
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_ast: Selecting %d time "
"slices from data file '%s'.", status, (int) itime,
data->file->name );
}
} else {
itime = nslice;
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_ast: Using all %d time "
"slices from data file '%s'.", status, (int) itime,
data->file->name );
}
}
/* Initialise a pointer to the next time slice index to be used. */
nexttime = ptime;
/* Initialise the progress meter. */
smf_reportprogress( itime, status );
/* Loop round all time slices in the input NDF. */
for( itime = 0; itime < nslice && *status == SAI__OK; itime++ ) {
/* If this time slice is not being pasted into the output cube, pass on. */
if( nexttime ){
if( *nexttime != (int) itime ) continue;
nexttime++;
}
/* Store a pointer to the first input data value in this time slice. */
tdata = ( (float *) (data->pntr)[ 0 ] ) + itime*timeslice_size;
/* Begin an AST context. Having this context within the time slice loop
helps keep the number of AST objects in use to a minimum. */
astBegin;
/* Get a Mapping from the spatial GRID axes in the input the spatial
GRID axes in the output for the current time slice. Note this has
to be done first since it stores details of the current time slice
in the "smfHead" structure inside "data", and this is needed by
subsequent functions. */
totmap = smf_rebin_totmap( data, itime, abskyfrm, oskymap, moving,
NO_FTS, status );
if( !totmap ) {
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_ast: Cannot get "
"Mapping for slice %d from data file '%s'.", status,
(int) itime, data->file->name );
}
break;
}
/* Get the effective exposure time, the total exposure time, and the
Tsys->Variance onversion factor for this time slice. Also get a
pointer to the start of the Tsys array. */
tsys = smf_rebincube_tcon( hdr, itime, fcon, &texp, &teff, &tcon,
status );
/* So "totmap" is a 2-input, 2-output Mapping that transforms the input
spatial GRID coords into output spatial GRID coords. In order to speed
up astRebinSeq we represent this by a pair of parallel LutMaps. To do
this (using atlTolut) we need a Mapping which only has 1 input, so we
preceed "totmap" with "detmap" (which also has the effect of exluding
data from unrequired detectors). We then combine this Mapping in
parallel with the spectral LutMap to get a 2-input (channel number,
detector index) and 3-output (output grid coords) Mapping. We finally
add a PermMap to re-arrange the output axes so that channel number is
axis 3 in the output. */
dtotmap = (AstMapping *) astCmpMap( detmap, totmap, 1, " " );
if( ndet > 1 ) {
atlTolut( dtotmap, 1.0, (double) ndet, 1.0, "LutInterp=1", &splut,
status );
} else {
splut = astClone( dtotmap );
}
fullmap = astSimplify( astCmpMap( astCmpMap( sslut, splut, 0, " " ),
pmap, 1, " " ) );
/* If required calculate the variance associated with each value in the
current time slice. based on the input Tsys values. If they are
needed, but not available, ignored the time slice. */
ignore = 0;
if( varwork ) {
ignore = 1;
vp = varwork;
for( idet = 0; idet < ndet; idet++ ) {
invar = VAL__BADR;
rtsys = tsys ? (float) tsys[ idet ] : VAL__BADR;
if( rtsys <= 0.0 ) rtsys = VAL__BADR;
if( rtsys != VAL__BADR ) {
*good_tsys = 1;
if( tcon != VAL__BADD ) {
invar = tcon*rtsys*rtsys;
ignore = 0;
}
}
for( ichan = 0; ichan < nchan; ichan++ ) *(vp++) = invar;
}
}
/* Unless we are ignoring this time slice, paste it into the 3D output
cube. The smf_rebincube_seqf function is a wrapper for astRebinSeqF
that splits the total job up between "nthreads" threads running in
parallel. */
if( !ignore ) {
smf_rebincube_seqf( wf, nthreads, blk_bot, fullmap, 0.0, 2, lbnd_in,
ubnd_in, tdata, varwork, spread, params,
ast_flags, 0.0, 50, VAL__BADR, 3, ldim, udim,
lbnd_in, ubnd_in, data_array, var_array,
wgt_array, nused, status );
/* Now we update the total exposure time array. Scale the exposure time
of this time slice in order to reduce its influence on the output
expsoure times if it does not have much spectral overlap with the
output cube. then fill the 1D work array with this constant value and
paste it into the 2D texp_array using the spatial mapping. Note we
want the simple sum of the exposure times, with no normalisation. SO
we use the AST__NONORM flag which means we do not need to supply a
weights array. */
if( texp != VAL__BADR ) {
texp *= tfac;
for( iv = 0; iv < ndet; iv++ ) detwork[ iv ] = texp;
astRebinSeqF( splut, 0.0, 1, ldim, uddim, detwork, NULL,
spread, params, AST__NONORM, 0.0, 50,
VAL__BADR, 2, ldim, udim, ldim, uddim, texp_array,
NULL, NULL, NULL );
}
/* Now do the same with the effective exposure time. */
if( teff != VAL__BADR ) {
teff *= tfac;
for( iv = 0; iv < ndet; iv++ ) detwork[ iv ] = teff;
astRebinSeqF( splut, 0.0, 1, ldim, uddim, detwork, NULL,
spread, params, AST__NONORM, 0.0, 50, VAL__BADR, 2,
ldim, udim, ldim, uddim, teff_array, NULL, NULL,
NULL );
}
} else if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_ast: Time slice %d "
"is being ignored when processing data file '%s'.",
status, (int) itime, data->file->name );
}
/* Update the progress meter. */
smf_reportprogress( 0, status );
/* End the AST context. */
astEnd;
}
/* If this is the final pass through this function, normalise the returned
data and variance values. */
if( last ) {
/* Check some data was pasted into the output. */
if( *nused > 0 ) {
/* Create a dummy mapping that can be used with astRebinSeq (it is not
actually used for anything since we are not adding any more data into the
output arrays). */
fullmap = (AstMapping *) astPermMap( 2, NULL, 3, NULL, NULL, " " );
/* Normalise the data values. We do not normalise the exposure time arrays. */
astRebinSeqF( fullmap, 0.0, 2, lbnd_in,
ubnd_in, NULL, NULL, spread, params,
AST__REBINEND | ast_flags, 0.0, 50, VAL__BADR, 3,
ldim, udim, lbnd_in, ubnd_in, data_array, var_array,
wgt_array, nused );
fullmap = astAnnul(fullmap);
/* If no data was pasted into the output, fill it with bad values and
issue a warning. */
} else {
size_t nel = dim[0]*dim[1]*dim[2];
size_t iel;
float *p1 = data_array;
for( iel = 0; iel < nel; iel++ ) *(p1++) = VAL__BADR;
if( genvar ) {
p1 = var_array;
for( iel = 0; iel < nel; iel++ ) *(p1++) = VAL__BADR;
}
msgOut( "", "WARNING: No good values in output cube.", status );
}
}
/* Free resources. */
detlut = astFree( detlut );
detwork = astFree( detwork );
varwork = astFree( varwork );
}
void smfCalcMapcoordPar( void *job_data_ptr, int *status ) {
AstSkyFrame *abskyfrm=NULL;
smfData *data=NULL;
fts2Port fts_port;
int *lbnd_out=NULL;
int *lut=NULL;
int moving;
int *ubnd_out=NULL;
dim_t nbolo; /* number of bolometers */
dim_t ntslice; /* number of time slices */
smfCalcMapcoordData *pdata=NULL; /* Pointer to job data */
AstMapping *sky2map=NULL;
double *theta = NULL;
struct timeval tv1; /* Timer */
struct timeval tv2; /* Timer */
if( *status != SAI__OK ) return;
/* Pointer to the data that this thread will process */
pdata = job_data_ptr;
/* Check for valid inputs */
if( !pdata ) {
*status = SAI__ERROR;
errRep( "", "smfCalcMapcoordPar: No job data supplied", status );
return;
}
/* Extract values from pdata */
abskyfrm = pdata->abskyfrm;
data = pdata->data;
lut = pdata->lut;
theta = pdata->theta;
lbnd_out = pdata->lbnd_out;
moving = pdata->moving;
sky2map = pdata->sky2map;
ubnd_out = pdata->ubnd_out;
fts_port = pdata->fts_port;
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, NULL, NULL, status );
/* if t1 past end of the work, nothing to do so we return */
if( pdata->t1 >= ntslice ) {
msgOutif( SMF__TIMER_MSG, "",
"smfCalcMapcoordPar: nothing for thread to do, returning",
status);
return;
}
/* Debugging message indicating thread started work */
msgOutiff( SMF__TIMER_MSG, "",
"smfCalcMapcoordPar: thread starting on tslices %zu -- %zu",
status, pdata->t1, pdata->t2 );
smf_timerinit( &tv1, &tv2, status );
/* Lock the supplied AST object pointers for exclusive use by this
thread. The invoking thread should have unlocked them before
starting this job. */
astLock( abskyfrm, 0 );
astLock( sky2map, 0 );
smf_lock_data( data, 1, status );
/* Calculate and store the LUT values for the range of time slices
being processed by this thread. A generic algorithm is used for
moving targets, but a faster algorithm can be used for stationary
targets. */
smf_coords_lut( data, pdata->tstep, pdata->t1, pdata->t2,
abskyfrm, sky2map, moving, lbnd_out, ubnd_out, fts_port,
lut + pdata->t1*nbolo, theta + pdata->t1,
pdata->lon_ptr, pdata->lat_ptr, status );
/* Unlock the supplied AST object pointers so that other threads can use
them. */
smf_lock_data( data, 0, status );
astUnlock( abskyfrm, 1 );
astUnlock( sky2map, 1 );
msgOutiff( SMF__TIMER_MSG, "",
"smfCalcMapcoordPar: thread finishing tslices %zu -- "
"%zu (%.3f sec)", status, pdata->t1, pdata->t2,
smf_timerupdate(&tv1, &tv2, status) );
}
void smf_find_science(const Grp * ingrp, Grp **outgrp, int reverttodark,
Grp **darkgrp, Grp **flatgrp, int reducedark,
int calcflat, smf_dtype darktype, smfArray ** darks,
smfArray **fflats, AstKeyMap ** heateffmap,
double * meanstep, int * status ) {
smfSortInfo *alldarks; /* array of sort structs for darks */
smfSortInfo *allfflats; /* array of fast flat info */
Grp * dgrp = NULL; /* Internal dark group */
double duration_darks = 0.0; /* total duration of all darks */
double duration_sci = 0.0; /* Duration of all science observations */
size_t dkcount = 0; /* Dark counter */
size_t ffcount = 0; /* Fast flat counter */
Grp * fgrp = NULL; /* Fast flat group */
size_t i; /* loop counter */
smfData *infile = NULL; /* input file */
size_t insize; /* number of input files */
size_t nsteps_dark = 0; /* Total number of steps for darks */
size_t nsteps_sci = 0; /* Total number of steps for science */
AstKeyMap * heatermap = NULL; /* Heater efficiency map */
AstKeyMap * obsmap = NULL; /* Info from all observations */
AstKeyMap * objmap = NULL; /* All the object names used */
AstKeyMap * scimap = NULL; /* All non-flat obs indexed by unique key */
Grp *ogrp = NULL; /* local copy of output group */
size_t sccount = 0; /* Number of accepted science files */
struct timeval tv1; /* Timer */
struct timeval tv2; /* Timer */
if (meanstep) *meanstep = VAL__BADD;
if (outgrp) *outgrp = NULL;
if (darkgrp) *darkgrp = NULL;
if (darks) *darks = NULL;
if (fflats) *fflats = NULL;
if (heateffmap) *heateffmap = NULL;
if (*status != SAI__OK) return;
/* Sanity check to make sure we return some information */
if ( outgrp == NULL && darkgrp == NULL && darks == NULL && fflats == NULL) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": Must have some non-NULL arguments"
" (possible programming error)", status);
return;
}
/* Start a timer to see how long this takes */
smf_timerinit( &tv1, &tv2, status );
/* Create new group for output files */
ogrp = smf_grp_new( ingrp, "Science", status );
/* and a new group for darks */
dgrp = smf_grp_new( ingrp, "DarkFiles", status );
/* and for fast flats */
fgrp = smf_grp_new( ingrp, "FastFlats", status );
/* and also create a keymap for the observation description */
obsmap = astKeyMap( "KeyError=1" );
/* and an object map */
objmap = astKeyMap( "KeyError=1" );
/* This keymap contains the sequence counters for each related
subarray/obsidss/heater/shutter combination and is used to decide
if a bad flat is relevant */
scimap = astKeyMap( "KeyError=1,KeyCase=0" );
/* This keymap is used to contain relevant heater efficiency data */
heatermap = astKeyMap( "KeyError=1,KeyCase=0" );
/* Work out how many input files we have and allocate sufficient sorting
space */
insize = grpGrpsz( ingrp, status );
alldarks = astCalloc( insize, sizeof(*alldarks) );
allfflats = astCalloc( insize, sizeof(*allfflats) );
/* check each file in turn */
for (i = 1; i <= insize; i++) {
int seqcount = 0;
char keystr[100]; /* Key for scimap entry */
/* open the file but just to get the header */
smf_open_file( ingrp, i, "READ", SMF__NOCREATE_DATA, &infile, status );
if (*status != SAI__OK) break;
/* Fill in the keymap with observation details */
smf_obsmap_fill( infile, obsmap, objmap, status );
/* Find the heater efficiency map if required */
if (*status == SAI__OK && heateffmap) {
char arrayidstr[32];
smf_fits_getS( infile->hdr, "ARRAYID", arrayidstr, sizeof(arrayidstr),
status );
if (!astMapHasKey( heatermap, arrayidstr ) ) {
smfData * heateff = NULL;
dim_t nbolos = 0;
smf_flat_params( infile, "RESIST", NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, &heateff, status );
smf_get_dims( heateff, NULL, NULL, &nbolos, NULL, NULL, NULL, NULL,
status );
if (heateff) astMapPut0P( heatermap, arrayidstr, heateff, NULL );
}
}
/* Get the sequence counter for the file. We do not worry about
duplicate sequence counters (at the moment) */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* The key identifying this subarray/obsidss/heater/shutter combo */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
if (smf_isdark( infile, status )) {
/* Store the sorting information */
dkcount = smf__addto_sortinfo( infile, alldarks, i, dkcount, "Dark", status );
smf__addto_durations( infile, &duration_darks, &nsteps_dark, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
} else {
/* compare sequence type with observation type and drop it (for now)
if they differ */
if ( infile->hdr->obstype == infile->hdr->seqtype ) {
/* Sanity check the header for corruption. Compare RTS_NUM with SEQSTART
and SEQEND. The first RTS_NUM must either be SEQSTART or else between
SEQSTART and SEQEND (if someone has giving us a section) */
int seqstart = 0;
int seqend = 0;
int firstnum = 0;
JCMTState *tmpState = NULL;
smf_getfitsi( infile->hdr, "SEQSTART", &seqstart, status );
smf_getfitsi( infile->hdr, "SEQEND", &seqend, status );
tmpState = infile->hdr->allState;
if( tmpState ) {
firstnum = (tmpState[0]).rts_num;
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
if ( firstnum >= seqstart && firstnum <= seqend ) {
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif(MSG__DEBUG, " ", "Non-dark file: ^F",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
} else {
msgOutif( MSG__QUIET, "",
"File ^F has a corrupt FITS header. Ignoring it.",
status );
}
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif( MSG__DEBUG, " ",
"File ^F lacks JCMTState: assuming it is non-dark",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
}
} else if (infile->hdr->seqtype == SMF__TYP_FASTFLAT ) {
ffcount = smf__addto_sortinfo( infile, allfflats, i, ffcount, "Fast flat", status );
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
msgOutif(MSG__DEBUG, " ", "Sequence type mismatch with observation type: ^F",status);
}
}
/* close the file */
smf_close_file( &infile, status );
}
/* Store output group in return variable or else free it */
if (outgrp) {
*outgrp = ogrp;
} else {
grpDelet( &ogrp, status );
}
/* process flatfields if necessary */
if (ffcount > 0 && fflats ) {
smfArray * array = NULL;
/* sort flats into order */
qsort( allfflats, ffcount, sizeof(*allfflats), smf_sort_bydouble);
if (fflats) array = smf_create_smfArray( status );
/* now open the flats and store them if requested */
if (*status == SAI__OK && array && ffcount) {
size_t start_ffcount = ffcount;
AstKeyMap * flatmap = NULL;
if (calcflat) {
/* Use AgeUp so that we get the keys out in the sorted order
that allfflats used */
flatmap = astKeyMap( "KeyCase=0,KeyError=1,SortBy=AgeDown" );
}
/* Read each flatfield. Calculate a responsivity image and a flatfield
solution. Store these in a keymap along with related information
which is itself stored in a keymap indexed by a string made of
OBSIDSS, reference heater value, shutter and subarray.
*/
for (i = 0; i < start_ffcount; i++ ) {
size_t ori_index = (allfflats[i]).index;
smfData * outfile = NULL;
char keystr[100];
AstKeyMap * infomap = astKeyMap( "KeyError=1" );
int oplen = 0;
char thisfile[MSG__SZMSG];
int seqcount = 0;
/* read filename from group */
infile = NULL;
smf_open_file( ingrp, ori_index, "READ", 0, &infile, status );
if ( *status != SAI__OK ) {
/* This should not happen because we have already opened
the file. If it does happen we abort with error. */
if (infile) smf_close_file( &infile, status );
break;
}
/* Calculate the key for this observation */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
/* Get the file name for error messages */
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>" );
msgLoad( "", "^F", thisfile, sizeof(thisfile), &oplen, status );
/* And the sequence counter to link against science observations */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* Prefill infomap */
astMapPut0C( infomap, "FILENAME", thisfile, "");
astMapPut0I( infomap, "SEQCOUNT", seqcount, "");
/* Collapse it */
if (*status == SAI__OK) {
smf_flat_fastflat( infile, &outfile, status );
if (*status == SMF__BADFLAT) {
errFlush( status );
if (calcflat) {
/* Need to generate an outfile like smf_flat_fastflat
and one heater setting will force smf_flat_calcflat to fail */
smf_flat_malloc( 1, infile, NULL, &outfile, status );
} else {
if (outfile) smf_close_file( &outfile, status );
if (infile) smf_close_file( &infile, status );
infomap = astAnnul( infomap );
ffcount--;
continue;
}
}
}
if (outfile && *status == SAI__OK) {
smf_close_file( &infile, status );
infile = outfile;
if (calcflat) {
size_t ngood = 0;
smfData * curresp = NULL;
int utdate;
if (*status == SAI__OK) {
ngood = smf_flat_calcflat( MSG__VERB, NULL, "RESIST",
"FLATMETH", "FLATORDER", NULL, "RESPMASK",
"FLATSNR", NULL, infile, &curresp, status );
if (*status != SAI__OK) {
/* if we failed to calculate a flatfield we continue but force the
flatfield to be completely bad. This will force the science data associated
with the flatfield to be correctly blanked. We do not annul though
if we have a SUBPAR error telling us that we have failed to define
our parameters properly. */
if (*status != SUBPAR__NOPAR) errAnnul(status);
/* parameters of flatfield */
ngood = 0;
/* Generate a blank flatfield and blank responsivity image */
smf_flat_badflat( infile, &curresp, status );
}
/* Retrieve the UT date so we can decide whether to compare
flatfields */
smf_getfitsi( infile->hdr, "UTDATE", &utdate, status );
/* Store the responsivity data for later on and the processed
flatfield until we have vetted it */
astMapPut0P( infomap, "CALCFLAT", infile, "" );
astMapPut0P( infomap, "RESP", curresp, "" );
astMapPut0I( infomap, "UTDATE", utdate, "" );
astMapPut0I( infomap, "ISGOOD", 1, "" );
astMapPut0I( infomap, "NGOOD", ngood, "" );
astMapPut0I( infomap, "GRPINDEX", ori_index, "" );
astMapPut0I( infomap, "SMFTYP", infile->hdr->obstype, "" );
astMapPutElemA( flatmap, keystr, -1, infomap );
}
} else { /* if (calcflat) */
/* Store the collapsed flatfield - the processed flat is not stored here yet */
smf_addto_smfArray( array, infile, status );
/* Copy the group info */
ndgCpsup( ingrp, ori_index, fgrp, status );
}
} /* if (outfile) */
/* Annul the keymap (will be fine if it is has been stored in another keymap) */
infomap = astAnnul( infomap );
} /* End loop over flatfields */
/* Now we have to loop over the related flatfields to disable
bolometers that are not good and also decide whether we
need to set status to bad. */
if (*status == SAI__OK && calcflat ) {
size_t nkeys = astMapSize( flatmap );
for (i = 0; i < nkeys; i++ ) {
const char *key = astMapKey( flatmap, i );
int nf = 0;
AstKeyMap ** kmaps = NULL;
int nelem = astMapLength( flatmap, key );
kmaps = astMalloc( sizeof(*kmaps) * nelem );
astMapGet1A( flatmap, key, nelem, &nelem, kmaps );
for ( nf = 0; nf < nelem && *status == SAI__OK; nf++ ) {
AstKeyMap * infomap = kmaps[nf];
int isgood = 0;
astMapGet0I( infomap, "ISGOOD", &isgood );
if (isgood) {
/* The flatfield worked */
size_t ngood = 0;
int itemp;
int utdate = 0;
int ratioFlats = 0;
/* Get the UT date - we do not compare flatfields after
the time we enabled heater tracking at each sequence. */
astMapGet0I( infomap, "UTDATE", &utdate );
/* Get the number of good bolometers at this point */
astMapGet0I( infomap, "NGOOD", &itemp );
ngood = itemp;
/* Decide if we want to do the ratio test. We default to
not doing it between 20110901 and 20120827 which is
the period when we did mini-heater tracks before each
flat. ! indicates that we choose based on date. */
if (*status == SAI__OK) {
parGet0l( "FLATUSENEXT", &ratioFlats, status );
if ( *status == PAR__NULL ) {
errAnnul( status );
if (utdate >= 20110901 || utdate <= 20120827 ) {
ratioFlats = 0;
} else {
ratioFlats = 1;
}
}
}
/* Can we compare with the next flatfield? */
if (ngood < SMF__MINSTATSAMP || !ratioFlats ) {
/* no point doing all the ratio checking for this */
} else if ( nelem - nf >= 2 ) {
AstKeyMap * nextmap = kmaps[nf+1];
const char *nextfname = NULL;
const char *fname = NULL;
smfData * curresp = NULL;
smfData * nextresp = NULL;
smfData * curflat = NULL;
void *tmpvar = NULL;
size_t bol = 0;
smfData * ratio = NULL;
double *in1 = NULL;
double *in2 = NULL;
double mean = VAL__BADD;
size_t nbolo;
double *out = NULL;
double sigma = VAL__BADD;
float clips[] = { 5.0, 5.0 }; /* 5.0 sigma iterative clip */
size_t ngoodz = 0;
astMapGet0C( nextmap, "FILENAME", &nextfname );
astMapGet0C( infomap, "FILENAME", &fname );
/* Retrieve the responsivity images from the keymap */
astMapGet0P( infomap, "RESP", &tmpvar );
curresp = tmpvar;
astMapGet0P( nextmap, "RESP", &tmpvar );
nextresp = tmpvar;
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
nbolo = (curresp->dims)[0] * (curresp->dims)[1];
/* get some memory for the ratio if we have not already.
We could get some memory once assuming each flat has the
same number of bolometers... */
ratio = smf_deepcopy_smfData( curresp, 0, 0, 0, 0, status );
if( *status == SAI__OK ) {
/* divide: smf_divide_smfData ? */
in1 = (curresp->pntr)[0];
in2 = (nextresp->pntr)[0];
out = (ratio->pntr)[0];
for (bol=0; bol<nbolo;bol++) {
if ( in1[bol] != VAL__BADD && in1[bol] != 0.0 &&
in2[bol] != VAL__BADD && in2[bol] != 0.0 ) {
out[bol] = in1[bol] / in2[bol];
} else {
out[bol] = VAL__BADD;
}
}
}
/* find some statistics */
smf_clipped_stats1D( out, 2, clips, 1, nbolo, NULL, 0, 0, &mean,
&sigma, NULL, 0, &ngoodz, status );
if (*status == SMF__INSMP) {
errAnnul(status);
msgOutiff( MSG__QUIET, "",
"Flatfield ramp ratio of %s with %s had too few bolometers (%zu < %d).",
status, fname, nextfname, ngoodz, SMF__MINSTATSAMP );
ngood = ngoodz; /* Must be lower or equal to original ngood */
} else if (*status == SAI__OK && mean != VAL__BADD && sigma != VAL__BADD && curflat->da) {
/* Now flag the flatfield as bad for bolometers that have changed
more than n%. We expect the variation to be 1+/-a small bit */
const double pmrange = 0.10;
double thrlo = 1.0 - pmrange;
double thrhi = 1.0 + pmrange;
size_t nmasked = 0;
double *flatcal = curflat->da->flatcal;
msgOutiff( MSG__DEBUG, "", "Flatfield fast ramp ratio mean = %g +/- %g (%zu bolometers)",
status, mean, sigma, ngood);
/* we can just set the first slice of the flatcal to bad. That should
be enough to disable the entire bolometer. We have just read these
data so they should be in ICD order. */
for (bol=0; bol<nbolo;bol++) {
if ( out[bol] != VAL__BADD &&
(out[bol] < thrlo || out[bol] > thrhi ) ) {
flatcal[bol] = VAL__BADD;
nmasked++;
} else if ( in1[bol] != VAL__BADD && in2[bol] == VAL__BADD ) {
/* A bolometer is bad next time but good now so we must set it bad now */
flatcal[bol] = VAL__BADD;
nmasked++;
}
}
if ( nmasked > 0 ) {
msgOutiff( MSG__NORM, "", "Masked %zu bolometers in %s from unstable flatfield",
status, nmasked, fname );
/* update ngood to take into account the masking */
ngood -= nmasked;
}
}
smf_close_file( &ratio, status );
} /* End of flatfield responsivity comparison */
/* if we only have a few bolometers left we now consider this
a bad flat unless it is actually an engineering measurement
where expect some configurations to give zero bolometers */
if (ngood < SMF__MINSTATSAMP) {
const char *fname = NULL;
void * tmpvar = NULL;
int smftyp = 0;
smfData * curflat = NULL;
astMapGet0I( infomap, "SMFTYP", &smftyp );
astMapGet0C( infomap, "FILENAME", &fname );
if (smftyp != SMF__TYP_NEP) {
msgOutiff( MSG__QUIET, "",
"Flatfield %s has %zu good bolometer%s.%s",
status, fname, ngood, (ngood == 1 ? "" : "s"),
( ngood == 0 ? "" : " Keeping none.") );
isgood = 0;
/* Make sure that everything is blanked. */
if (ngood > 0) {
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
if (curflat && curflat->da) {
size_t bol;
size_t nbolo = (curflat->dims)[0] * (curflat->dims)[1];
double *flatcal = curflat->da->flatcal;
for (bol=0; bol<nbolo; bol++) {
/* Just need to set the first element to bad */
flatcal[bol] = VAL__BADD;
}
}
}
} else {
msgOutiff( MSG__NORM, "",
"Flatfield ramp file %s has %zu good bolometer%s. Eng mode.",
status, fname, ngood, (ngood == 1 ? "" : "s") );
}
}
/* We do not need the responsivity image again */
{
void *tmpvar = NULL;
smfData * resp = NULL;
astMapGet0P( infomap, "RESP", &tmpvar );
resp = tmpvar;
if (resp) smf_close_file( &resp, status );
astMapRemove( infomap, "RESP" );
}
} /* End of isgood comparison */
/* We are storing flats even if they failed. Let the downstream
software worry about it */
{
int ori_index;
smfData * flatfile = NULL;
void *tmpvar = NULL;
/* Store in the output group */
astMapGet0I( infomap, "GRPINDEX", &ori_index );
ndgCpsup( ingrp, ori_index, fgrp, status );
/* And store in the smfArray */
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
astMapRemove( infomap, "CALCFLAT" );
flatfile = tmpvar;
smf_addto_smfArray( array, flatfile, status );
}
/* Free the object as we go */
kmaps[nf] = astAnnul( kmaps[nf] );
} /* End of loop over this obsidss/subarray/heater */
kmaps = astFree( kmaps );
}
}
if (array->ndat) {
if (fflats) *fflats = array;
} else {
smf_close_related(&array, status );
if (fflats) *fflats = NULL;
}
}
}
/* no need to do any more if neither darks nor darkgrp are defined or we might
be wanting to revert to darks. */
if (dkcount > 0 && (darks || darkgrp || reverttodark ) ) {
smfArray * array = NULL;
/* sort darks into order */
qsort( alldarks, dkcount, sizeof(*alldarks), smf_sort_bydouble);
if (darks) array = smf_create_smfArray( status );
/* now open the darks and store them if requested */
if (*status == SAI__OK) {
for (i = 0; i < dkcount; i++ ) {
size_t ori_index = (alldarks[i]).index;
/* Store the entry in the output group */
ndgCpsup( ingrp, ori_index, dgrp, status );
if (darks) {
/* read the value from the new group */
smf_open_file( dgrp, i+1, "READ", 0, &infile, status );
/* do we have to process these darks? */
if (reducedark) {
smfData *outfile = NULL;
smf_reduce_dark( infile, darktype, &outfile, status );
if (outfile) {
smf_close_file( &infile, status );
infile = outfile;
}
}
smf_addto_smfArray( array, infile, status );
}
}
if (darks) *darks = array;
}
}
/* free memory */
alldarks = astFree( alldarks );
allfflats = astFree( allfflats );
if( reverttodark && outgrp && (grpGrpsz(*outgrp,status)==0) &&
(grpGrpsz(dgrp,status)>0) ) {
/* If outgrp requested but no science observations were found, and
dark observations were found, return darks in outgrp and set
flatgrp and darkgrp to NULL. This is to handle cases where we
want to process data taken in the dark like normal science
data. To activate this behaviour set reverttodark */
msgOutiff( MSG__NORM, "", "Treating the dark%s as science data",
status, ( dkcount > 1 ? "s" : "" ) );
*outgrp = dgrp;
if( darkgrp ){
*darkgrp = NULL;
}
if( flatgrp ) {
*flatgrp = NULL;
}
grpDelet( &ogrp, status);
grpDelet( &fgrp, status);
if (meanstep && nsteps_dark > 0) *meanstep = duration_darks / nsteps_dark;
/* Have to clear the darks smfArray as well */
if (darks) smf_close_related( darks, status );
} else {
/* Store the output groups in the return variable or free it */
if (darkgrp) {
*darkgrp = dgrp;
} else {
grpDelet( &dgrp, status);
}
if (flatgrp) {
*flatgrp = fgrp;
} else {
grpDelet( &fgrp, status);
}
if (meanstep && nsteps_sci > 0) *meanstep = duration_sci / nsteps_sci;
}
msgSeti( "ND", sccount );
msgSeti( "DK", dkcount );
msgSeti( "FF", ffcount );
msgSeti( "TOT", insize );
if ( insize == 1 ) {
if (dkcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a dark",
status);
} else if (ffcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a fast flatfield",
status);
} else if (sccount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was accepted (observation type same as sequence type)",
status);
} else {
msgOutif( MSG__VERB, " ", "Single input file was not accepted.",
status);
}
} else {
if (dkcount == 1) {
msgSetc( "DKTXT", "was a dark");
} else {
msgSetc( "DKTXT", "were darks");
}
if (ffcount == 1) {
msgSetc( "FFTXT", "was a fast flat");
} else {
msgSetc( "FFTXT", "were fast flats");
}
if (sccount == 1) {
msgSetc( "NDTXT", "was science");
} else {
msgSetc( "NDTXT", "were science");
}
/* This might be a useful message */
msgOutif( MSG__NORM, " ", "Out of ^TOT input files, ^DK ^DKTXT, ^FF ^FFTXT "
"and ^ND ^NDTXT", status );
}
if (meanstep && *meanstep != VAL__BADD) {
msgOutiff( MSG__VERB, "", "Mean step time for input files = %g sec",
status, *meanstep );
}
/* Store the heater efficiency map */
if (*status != SAI__OK) heatermap = smf_free_effmap( heatermap, status );
if (heateffmap) *heateffmap = heatermap;
/* Now report the details of the observation */
smf_obsmap_report( MSG__NORM, obsmap, objmap, status );
obsmap = astAnnul( obsmap );
objmap = astAnnul( objmap );
scimap = astAnnul( scimap );
msgOutiff( SMF__TIMER_MSG, "",
"Took %.3f s to find science observations",
status, smf_timerupdate( &tv1, &tv2, status ) );
return;
}
void smurf_calcqu( int *status ) {
/* Local Variables: */
AstFitsChan *fc; /* Holds FITS headers for output NDFs */
AstKeyMap *config; /* Holds all cleaning parameters */
AstKeyMap *dkpars; /* Holds dark squid cleaning parameters */
AstKeyMap *heateffmap = NULL; /* Heater efficiency data */
AstKeyMap *sub_instruments;/* Indicates which instrument is being used */
Grp *bgrp = NULL; /* Group of base names for each chunk */
Grp *igrp = NULL; /* Group of input files */
Grp *ogrp = NULL; /* Group of output files */
Grp *sgrp = NULL; /* Group of science files */
HDSLoc *loci = NULL; /* Locator for output I container file */
HDSLoc *locq = NULL; /* Locator for output Q container file */
HDSLoc *locu = NULL; /* Locator for output U container file */
NdgProvenance *oprov; /* Provenance to store in each output NDF */
ThrWorkForce *wf; /* Pointer to a pool of worker threads */
char headval[ 81 ]; /* FITS header value */
char ndfname[ 30 ]; /* Name of output Q or U NDF */
char polcrd[ 81 ]; /* FITS 'POL_CRD' header value */
char subarray[ 10 ]; /* Subarray name (e.g. "s4a", etc) */
double angrot; /* Angle from focal plane X axis to fixed analyser */
double paoff; /* WPLATE value corresponding to POL_ANG=0.0 */
float arcerror; /* Max acceptable error (arcsec) in one block */
int block_end; /* Index of last time slice in block */
int block_start; /* Index of first time slice in block */
int dkclean; /* Clean dark squids? */
int fix; /* Fix the POL-2 triggering issue? */
int iblock; /* Index of current block */
int iplace; /* NDF placeholder for current block's I image */
int ipolcrd; /* Reference direction for waveplate angles */
int maxsize; /* Max no. of time slices in a block */
int minsize; /* Min no. of time slices in a block */
int nc; /* Number of characters written to a string */
int pasign; /* +1 or -1 indicating sense of POL_ANG value */
int qplace; /* NDF placeholder for current block's Q image */
int submean; /* Subtract mean value from each time slice? */
int uplace; /* NDF placeholder for current block's U image */
size_t ichunk; /* Continuous chunk counter */
size_t idx; /* Subarray counter */
size_t igroup; /* Index for group of related input NDFs */
size_t inidx; /* Index into group of science input NDFs */
size_t nchunk; /* Number continuous chunks outside iter loop */
size_t ssize; /* Number of science files in input group */
smfArray *concat = NULL; /* Pointer to smfArray holding bolometer data */
smfArray *darks = NULL; /* dark frames */
smfArray *dkarray = NULL; /* Pointer to smfArray holding dark squid data */
smfArray *flatramps = NULL;/* Flatfield ramps */
smfData *data = NULL; /* Concatenated data for one subarray */
smfData *dkdata = NULL; /* Concatenated dark squid data for one subarray */
smfGroup *sgroup = NULL; /* smfGroup corresponding to sgrp */
/* Check inhereited status */
if( *status != SAI__OK ) return;
/* Start new AST and NDF contexts. */
astBegin;
ndfBegin();
/* Find the number of cores/processors available and create a work force
holding the same number of threads. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get a group of input files */
kpg1Rgndf( "IN", 0, 1, " Give more NDFs...", &igrp, &ssize, status );
/* Get a group containing just the files holding science data. */
smf_find_science( igrp, &sgrp, 0, NULL, NULL, 1, 1, SMF__NULL, &darks,
&flatramps, &heateffmap, NULL, status );
/* Check we have at least once science file. */
ssize = grpGrpsz( sgrp, status );
if( ssize == 0 ) {
msgOutif( MSG__NORM, " ", "All supplied input frames were DARK.",
status );
} else {
/* See if a correction should be made for the POL2 triggering issue. */
parGet0l( "FIX", &fix, status );
/* Create HDS container files to hold the output NDFs. */
datCreat( "OUTQ", "CALCQU", 0, 0, status );
datCreat( "OUTU", "CALCQU", 0, 0, status );
/* Associate the locators with the structures. */
datAssoc( "OUTQ", "WRITE", &locq, status );
datAssoc( "OUTU", "WRITE", &locu, status );
/* The I images are optional. */
if( *status == SAI__OK ) {
datCreat( "OUTI", "CALCQU", 0, 0, status );
datAssoc( "OUTI", "WRITE", &loci, status );
if( *status == PAR__NULL ) {
errAnnul( status );
loci = NULL;
}
}
/* Group the input files so that all files within a single group have the
same wavelength and belong to the same subscan of the same observation.
Also identify chunks of data that are contiguous in time, and
determine to which such chunk each group belongs. All this information
is returned in a smfGroup structure ("*sgroup"). */
smf_grp_related( sgrp, ssize, 1, 1, 0, NULL, NULL, NULL,
NULL, &sgroup, &bgrp, NULL, status );
/* Obtain the number of contiguous chunks. */
if( *status == SAI__OK ) {
nchunk = sgroup->chunk[ sgroup->ngroups - 1 ] + 1;
} else {
nchunk = 0;
}
/* Indicate we have not yet found a value for the ARCERROR parameter. */
arcerror = 0.0;
/* Loop over all contiguous chunks */
for( ichunk = 0; ichunk < nchunk && *status == SAI__OK; ichunk++ ) {
/* Display the chunk number. */
if( nchunk > 1 ) {
msgOutiff( MSG__VERB, "", " Doing chunk %d of %d.",
status, (int) ichunk + 1, (int) nchunk );
}
/* Concatenate the data within this contiguous chunk. This produces a
smfArray ("concat") containing a smfData for each subarray present in
the chunk. Each smfData holds the concatenated data for a single
subarray. */
smf_concat_smfGroup( wf, NULL, sgroup, darks, NULL, flatramps,
heateffmap, ichunk, 1, 1, NULL, 0, NULL, NULL,
0, 0, 0, &concat, NULL, status );
/* Get a KeyMap holding values for the configuration parameters. Since we
sorted by wavelength when calling smf_grp_related, we know that all
smfDatas in the current smfArray (i.e. chunk) will relate to the same
wavelength. Therefore we can use the same parameters for all smfDatas in
the current smfArray. */
sub_instruments = smf_subinst_keymap( SMF__SUBINST_NONE,
concat->sdata[ 0 ], NULL,
0, status );
config = kpg1Config( "CONFIG", "$SMURF_DIR/smurf_calcqu.def",
sub_instruments, status );
sub_instruments = astAnnul( sub_instruments );
/* Get the CALCQU specific parameters. */
if( !astMapGet0I( config, "PASIGN", &pasign ) ) pasign = 1;
msgOutiff( MSG__VERB, "", "PASIGN=%d", status, pasign );
if( !astMapGet0D( config, "PAOFF", &paoff ) ) paoff = 0.0;
msgOutiff( MSG__VERB, "", "PAOFF=%g", status, paoff );
if( !astMapGet0D( config, "ANGROT", &angrot ) ) angrot = 90.0;
msgOutiff( MSG__VERB, "", "ANGROT=%g", status, angrot );
if( !astMapGet0I( config, "SUBMEAN", &submean ) ) submean = 0;
msgOutiff( MSG__VERB, "", "SUBMEAN=%d", status, submean );
/* See if the dark squids should be cleaned. */
if( !astMapGet0I( config, "DKCLEAN", &dkclean ) ) dkclean = 0;
/* If required, clean the dark squids now since we might need to use them to
clean the bolometer data. */
if( dkclean ) {
/* Create a smfArray containing the dark squid data. For each one, store
a pointer to the main header so that smf_clean_smfArray can get at the
JCMTState information. */
dkarray = smf_create_smfArray( status );
for( idx = 0; idx < concat->ndat && *status == SAI__OK; idx++ ) {
data = concat->sdata[ idx ];
if( data && data->da && data->da->dksquid ) {
dkdata = data->da->dksquid;
dkdata->hdr = data->hdr;
smf_addto_smfArray( dkarray, dkdata, status );
}
}
/* Clean the smfArray containing the dark squid data. Use the "CLEANDK.*"
parameters. */
(void) astMapGet0A( config, "CLEANDK", &dkpars );
smf_clean_smfArray( wf, dkarray, NULL, NULL, NULL, dkpars, status );
dkpars = astAnnul( dkpars );
/* Nullify the header pointers so that we don't accidentally close any. */
if( dkarray ) {
for( idx = 0; idx < dkarray->ndat; idx++ ) {
dkdata = dkarray->sdata[ idx ];
dkdata->hdr = NULL;
}
/* Free the smfArray holding the dark squid data, but do not free the
individual smfDatas within it. */
dkarray->owndata = 0;
smf_close_related( &dkarray, status );
}
}
/* Now clean the bolometer data */
smf_clean_smfArray( wf, concat, NULL, NULL, NULL, config, status );
/* If required correct for the POL2 triggering issue. */
if( fix ) smf_fix_pol2( wf, concat, status );
/* Loop round each sub-array in the current contiguous chunk of data. */
for( idx = 0; idx < concat->ndat && *status == SAI__OK; idx++ ) {
data = concat->sdata[ idx ];
/* Find the name of the subarray that generated the data. */
smf_find_subarray( data->hdr, subarray, sizeof(subarray), NULL,
status );
/* Display the sub-array. */
if( concat->ndat > 1 ) {
msgOutiff( MSG__VERB, "", " Doing sub-array %s.",
status, subarray );
}
/* Create an empty provenance structure. Each input NDF that contributes
to the current chunk and array will be added as an ancestor to this
structure, which will later be stored in each output NDF created for
this chunk and array. */
oprov = ndgReadProv( NDF__NOID, "SMURF:CALCQU", status );
/* Indicate we do not yet have any FITS headers for the output NDFs */
fc = NULL;
/* Indicate we do not yet know the coordinate reference frame for the
half-waveplate angle. */
polcrd[ 0 ] = 0;
ipolcrd = 0;
/* Go through the smfGroup looking for groups of related input NDFs that
contribute to the current chunk. */
for( igroup = 0; igroup < sgroup->ngroups; igroup++ ) {
if( sgroup->chunk[ igroup ] == ichunk ) {
/* Get the integer index into the GRP group (sgrp) that holds the input NDFs.
This index identifies the input NDF that provides the data for the current
chunk and subarray. This assumes that the order in which smf_concat_smfGroup
stores arrays in the "concat" smfArray matches the order in which
smf_grp_related stores arrays within the sgroup->subgroups. */
inidx = sgroup->subgroups[ igroup ][ idx ];
/* Add this input NDF as an ancestor into the output provenance structure. */
smf_accumulate_prov( NULL, sgrp, inidx, NDF__NOID,
"SMURF:CALCQU", &oprov, status );
/* Merge the FITS headers from the current input NDF into the FitsChan
that holds headers for the output NDFs. The merging retains only those
headers which have the same value in all input NDFs. */
smf_fits_outhdr( data->hdr->fitshdr, &fc, status );
/* Get the polarimetry related FITS headers and check that all input NDFs
have usabie values. */
headval[ 0 ] = 0;
smf_getfitss( data->hdr, "POL_MODE", headval,
sizeof(headval), status );
if( strcmp( headval, "CONSTANT" ) && *status == SAI__OK ) {
*status = SAI__ERROR;
grpMsg( "N", sgrp, inidx );
errRep( " ", "Input NDF ^N does not contain "
"polarimetry data obtained with a spinning "
"half-waveplate.", status );
}
headval[ 0 ] = 0;
smf_getfitss( data->hdr, "POLWAVIN", headval,
sizeof(headval), status );
if( strcmp( headval, "Y" ) && *status == SAI__OK ) {
*status = SAI__ERROR;
grpMsg( "N", sgrp, inidx );
errRep( " ", "Half-waveplate was not in the beam for "
"input NDF ^N.", status );
}
headval[ 0 ] = 0;
smf_getfitss( data->hdr, "POLANLIN", headval,
sizeof(headval), status );
if( strcmp( headval, "Y" ) && *status == SAI__OK ) {
*status = SAI__ERROR;
grpMsg( "N", sgrp, inidx );
errRep( " ", "Analyser was not in the beam for input "
"NDF ^N.", status );
}
if( polcrd[ 0 ] ) {
headval[ 0 ] = 0;
smf_getfitss( data->hdr, "POL_CRD", headval,
sizeof(headval), status );
if( strcmp( headval, polcrd ) && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( " ", "Input NDFs have differing values for "
"FITS header 'POL_CRD'.", status );
}
} else {
smf_getfitss( data->hdr, "POL_CRD", polcrd,
sizeof(polcrd), status );
if( !strcmp( polcrd, "FPLANE" ) ) {
ipolcrd = 0;
} else if( !strcmp( polcrd, "AZEL" ) ) {
ipolcrd = 1;
} else if( !strcmp( polcrd, "TRACKING" ) ) {
ipolcrd = 2;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
msgSetc( "N", data->file->name );
msgSetc( "V", polcrd );
errRep( " ", "Input NDF ^N contains unknown value "
"'^V' for FITS header 'POL_CRD'.", status );
}
}
}
}
/* If not already done, get the maximum spatial drift (in arc-seconds) that
can be tolerated whilst creating a single I/Q/U image. The default value is
half the makemap default pixel size. Also get limits on the number of
time slices in any block. */
if( arcerror == 0.0 ) {
parDef0d( "ARCERROR", 0.5*smf_calc_telres( data->hdr->fitshdr,
status ), status );
parGet0r( "ARCERROR", &arcerror, status );
parGet0i( "MAXSIZE", &maxsize, status );
parGet0i( "MINSIZE", &minsize, status );
if( maxsize > 0 && maxsize < minsize && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", "Value of parameter MAXSIZE (%d) is less "
"than value of parameter MINSIZE (%d)", status,
maxsize, minsize );
}
}
/* The algorithm that calculates I, Q and U assumes that all samples for a
single bolometer measure flux from the same point on the sky. Due to
sky rotation, this will not be the case - each bolometer will drift
slowly across the sky. However, since the drift is (or should be)
slow we can apply the I/Q/U algorithm to blocks of contiguous data over
which the bolometers do not move significantly. We produce a separate
I, Q and U image for each such block. The first block starts at the first
time slice in the smfData. */
block_start = 0;
/* Find the time slice at which the corner bolometers have moved
a critical distance (given by parameter ARCERROR) from their
positions at the start of the block. Then back off some time slices
to ensure that the block holds an integral number of half-waveplate
rotations. */
block_end = smf_block_end( data, block_start, ipolcrd, arcerror,
maxsize, status );
/* Loop round creating I/Q/U images for each block. Count them. */
iblock = 0;
while( block_end >= 0 && *status == SAI__OK ) {
/* Skip very short blocks. */
if( block_end - block_start > minsize ) {
/* Display the start and end of the block. */
msgOutiff( MSG__VERB, "", " Doing time slice block %d "
"-> %d", status, (int) block_start,
(int) block_end );
/* Get the name for the Q NDF for this block. Start of with "Q" followed by
the block index. */
iblock++;
nc = sprintf( ndfname, "Q%d", iblock );
/* Append the subarray name to the NDF name. */
nc += sprintf( ndfname + nc, "_%s", subarray );
/* Append the chunk index to the NDF name. */
nc += sprintf( ndfname + nc, "_%d", (int) ichunk );
/* Get NDF placeholder for the Q NDF. The NDFs are created inside the
output container file. */
ndfPlace( locq, ndfname, &qplace, status );
/* The name of the U NDF is the same except the initial "Q" is changed to
"U". */
ndfname[ 0 ] = 'U';
ndfPlace( locu, ndfname, &uplace, status );
/* The name of the I NDF is the same except the initial "Q" is changed to
"I". */
if( loci ) {
ndfname[ 0 ] = 'I';
ndfPlace( loci, ndfname, &iplace, status );
} else {
iplace = NDF__NOPL;
}
/* Store the chunk and block numbers as FITS headers. */
atlPtfti( fc, "POLCHUNK", (int) ichunk, "Chunk index used by CALCQU", status );
atlPtfti( fc, "POLBLOCK", iblock, "Block index used by CALCQU", status );
/* Create the Q and U images for the current block of time slices from
the subarray given by "idx", storing them in the output container
file. */
smf_calc_iqu( wf, data, block_start, block_end, ipolcrd,
qplace, uplace, iplace, oprov, fc,
pasign, AST__DD2R*paoff, AST__DD2R*angrot,
submean, status );
/* Warn about short blocks. */
} else {
msgOutiff( MSG__VERB, "", " Skipping short block of %d "
"time slices (parameter MINSIZE=%d).", status,
block_end - block_start - 1, minsize );
}
/* The next block starts at the first time slice following the previous
block. */
block_start = block_end + 1;
/* Find the time slice at which the corner bolometers have moved
a critical distance (given by parameter ARCERROR) from their
positions at the start of the block. Then back off some time slices
to ensure that the block holds an integral number of half-waveplate
rotations. This returns -1 if all time slices have been used. */
block_end = smf_block_end( data, block_start, ipolcrd,
arcerror, maxsize, status );
}
/* Free resources */
oprov = ndgFreeProv( oprov, status );
fc = astAnnul( fc );
}
config = astAnnul( config );
/* Close the smfArray. */
smf_close_related( &concat, status );
}
/* Annul the locators for the output container files. */
datAnnul( &locq, status );
datAnnul( &locu, status );
if( loci ) datAnnul( &loci, status );
/* The parameter system hangs onto a primary locator for each container
file, so cancel the parameters to annul these locators. */
datCancl( "OUTQ", status );
datCancl( "OUTU", status );
datCancl( "OUTI", status );
}
/* Free resources. */
smf_close_related( &darks, status );
smf_close_related( &flatramps, status );
if( igrp ) grpDelet( &igrp, status);
if( sgrp ) grpDelet( &sgrp, status);
if( bgrp ) grpDelet( &bgrp, status );
if( ogrp ) grpDelet( &ogrp, status );
if( sgroup ) smf_close_smfGroup( &sgroup, status );
if (heateffmap) heateffmap = smf_free_effmap( heateffmap, status );
/* End the NDF and AST contexts. */
ndfEnd( status );
astEnd;
/* Issue a status indication.*/
if( *status == SAI__OK ) {
msgOutif( MSG__VERB, " ", "CALCQU succeeded.", status);
} else {
msgOutif( MSG__VERB, " ", "CALCQU failed.", status);
}
}
/* Main entry point. */
void smf_calcmodel_smo( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel,
int flags __attribute__((unused)),
int *status) {
/* Local Variables */
size_t bstride; /* bolo stride */
dim_t boxcar = 0; /* size of boxcar smooth window */
smf_filt_t filter_type; /* The type of smoothing to perform */
size_t i; /* Loop counter */
dim_t idx=0; /* Index within subgroup */
int iworker; /* Owkrer index */
smfCalcmodelSmoJobData *job_data=NULL; /* Pointer to all job data structures */
AstKeyMap *kmap=NULL; /* Pointer to PLN-specific keys */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
double *model_data_copy=NULL; /* Copy of model_data for one bolo */
dim_t nbolo=0; /* Number of bolometers */
dim_t ndata=0; /* Total number of data points */
int notfirst=0; /* flag for delaying until after 1st iter */
dim_t ntslice=0; /* Number of time slices */
int nworker; /* No. of worker threads in supplied Workforce */
smfCalcmodelSmoJobData *pdata=NULL; /* Pointer to current data structure */
smfArray *qua=NULL; /* Pointer to QUA at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to quality data */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
int step; /* Number of bolometers per thread */
size_t tstride; /* Time slice stride in data array */
const char * typestr = NULL; /* smo.type value */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
/* Obtain pointer to sub-keymap containing PLN parameters. Something will
always be available.*/
astMapGet0A( keymap, "SMO", &kmap );
/* Are we skipping the first iteration? */
astMapGet0I(kmap, "NOTFIRST", ¬first);
if( notfirst && (flags & SMF__DIMM_FIRSTITER) ) {
msgOutif( MSG__VERB, "", FUNC_NAME
": skipping SMO this iteration", status );
return;
}
/* Get the boxcar size */
if( kmap ) smf_get_nsamp( kmap, "BOXCAR", res->sdata[0], &boxcar, status );
/* Get the type of smoothing filter to use. Anthing that is not "MEDIAN" is mean */
filter_type = SMF__FILT_MEAN;
if (astMapGet0C( kmap, "TYPE", &typestr ) ) {
if (strncasecmp( typestr, "MED", 3 ) == 0 ) {
filter_type = SMF__FILT_MEDIAN;
}
}
/* Assert bolo-ordered data */
smf_model_dataOrder( wf, dat, allmodel, chunk, SMF__RES|SMF__QUA,
0, status );
smf_get_dims( res->sdata[0], NULL, NULL, NULL, &ntslice,
&ndata, NULL, NULL, status);
model = allmodel[chunk];
msgOutiff(MSG__VERB, "",
" Calculating smoothed model using boxcar of width %" DIM_T_FMT " time slices",
status, boxcar);
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
/* Loop over index in subgrp (subarray) and put the previous iteration
of the filtered component back into the residual before calculating
and removing the new filtered component */
for( idx=0; (*status==SAI__OK)&&(idx<res->ndat); idx++ ) {
/* Obtain dimensions of the data */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice,
&ndata, &bstride, &tstride, status);
/* Get pointers to data/quality/model */
res_data = (res->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Uncomment to aid debugging */
/*
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_in",
NULL, 0, 0, MSG__VERB, 0, status );
*/
if( *status == SAI__OK ) {
/* Place last iteration back into residual if this is a smoothable section of the time series */
for (i=0; i< ndata; i++) {
if ( !(qua_data[i]&SMF__Q_FIT) && res_data[i] != VAL__BADD && model_data[i] != VAL__BADD ) {
res_data[i] += model_data[i];
}
}
}
/* Uncomment to aid debugging */
/*
smf_write_smfData( model->sdata[idx], NULL, qua_data, "model_b4",
NULL, 0, 0, MSG__VERB, 0, status );
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_b4",
NULL, 0, 0, MSG__VERB, 0, status );
*/
/* Determine which bolometers are to be processed by which threads. */
step = nbolo/nworker;
if( step < 1 ) step = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->b1 = iworker*step;
pdata->b2 = pdata->b1 + step - 1;
}
/* Ensure that the last thread picks up any left-over bolometers */
pdata->b2 = nbolo - 1;
/* Store all the other info needed by the worker threads, and submit the
jobs to apply the smoothing. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->boxcar = boxcar;
pdata->bstride = bstride;
pdata->bstride = bstride;
pdata->filter_type = filter_type;
pdata->model_data = model_data;
pdata->nbolo = nbolo;
pdata->nbolo = nbolo;
pdata->ntslice = ntslice;
pdata->ntslice = ntslice;
pdata->qua_data = qua_data;
pdata->qua_data = qua_data;
pdata->res_data = res_data;
pdata->res_data = res_data;
pdata->tstride = tstride;
pdata->tstride = tstride;
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_calcmodel_smo_job,
0, NULL, status );
}
thrWait( wf, status );
/* Uncomment to aid debugging */
/*
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_af",
NULL, 0, 0, MSG__VERB, 0, status );
smf_write_smfData( model->sdata[idx], NULL, qua_data, "model_af",
NULL, 0, 0, MSG__VERB, 0, status );
*/
}
}
/* Free work space (astFree returns without action if a NULL pointer is
supplied). */
model_data_copy = astFree( model_data_copy );
job_data = astFree( job_data );
/* Annul AST Object pointers (astAnnul reports an error if a NULL pointer
is supplied). */
if( kmap ) kmap = astAnnul( kmap );
}
unsigned char *smf_get_mask( ThrWorkForce *wf, smf_modeltype mtype,
AstKeyMap *config, smfDIMMData *dat, int flags,
int *status ) {
/* Local Variables: */
AstCircle *circle; /* AST Region used to mask a circular area */
AstKeyMap *akm; /* KeyMap holding AST config values */
AstKeyMap *subkm; /* KeyMap holding model config values */
char refparam[ DAT__SZNAM ];/* Name for reference NDF parameter */
char words[100]; /* Buffer for variable message words */
const char *cval; /* The ZERO_MASK string value */
const char *modname; /* The name of the model being masked */
const char *skyrefis; /* Pointer to SkyRefIs attribute value */
dim_t i; /* Pixel index */
double *pd; /* Pointer to next element of map data */
double *predef; /* Pointer to mask defined by previous run */
double *ptr; /* Pointer to NDF Data array */
double *pv; /* Pointer to next element of map variance */
double centre[ 2 ]; /* Coords of circle centre in radians */
double meanhits; /* Mean hits in the map */
double radius[ 1 ]; /* Radius of circle in radians */
double zero_circle[ 3 ]; /* LON/LAT/Radius of circular mask */
double zero_lowhits; /* Fraction of mean hits at which to threshold */
double zero_snr; /* Higher SNR at which to threshold */
double zero_snrlo; /* Lower SNR at which to threshold */
int *ph; /* Pointer to next hits value */
int have_mask; /* Did a mask already exist on entry? */
int imask; /* Index of next mask type */
int indf1; /* Id. for supplied reference NDF */
int indf2; /* Id. for used section of reference NDF */
int isstatic; /* Are all used masks static? */
int lbnd_grid[ 2 ]; /* Lower bounds of map in GRID coords */
int mask_types[ NTYPE ]; /* Identifier for the types of mask to use */
int munion; /* Use union of supplied masks */
int nel; /* Number of mapped NDF pixels */
int nmask; /* The number of masks to be combined */
int nsource; /* No. of source pixels in final mask */
int skip; /* No. of iters for which AST is not subtracted */
int thresh; /* Absolute threshold on hits */
int ubnd_grid[ 2 ]; /* Upper bounds of map in GRID coords */
int zero_c_n; /* Number of zero circle parameters read */
int zero_mask; /* Use the reference NDF as a mask? */
int zero_niter; /* Only mask for the first "niter" iterations. */
int zero_notlast; /* Don't zero on last iteration? */
size_t ngood; /* Number good samples for stats */
smf_qual_t *pq; /* Pinter to map quality */
unsigned char **mask; /* Address of model's mask pointer */
unsigned char *newmask; /* Individual mask work space */
unsigned char *pm; /* Pointer to next returned mask pixel */
unsigned char *pn; /* Pointer to next new mask pixel */
unsigned char *result; /* Returned mask pointer */
/* Initialise returned values */
result = NULL;
/* Check inherited status. Also check that a map is being created. */
if( *status != SAI__OK || !dat || !dat->map ) return result;
/* Begin an AST context. */
astBegin;
/* Get the sub-keymap containing the configuration parameters for the
requested model. Also get a pointer to the mask array to use (there is
one for each maskable model)*/
if( mtype == SMF__COM ) {
modname = "COM";
mask = &(dat->com_mask);
} else if( mtype == SMF__AST ) {
modname = "AST";
mask = &(dat->ast_mask);
} else if( mtype == SMF__FLT ) {
modname = "FLT";
mask = &(dat->flt_mask);
} else {
modname = NULL;
mask = NULL;
*status = SAI__ERROR;
errRepf( " ", "smf_get_mask: Unsupported model type %d supplied - "
"must be COM, FLT or AST.", status, mtype );
}
subkm = NULL;
astMapGet0A( config, modname, &subkm );
/* Get the "ast.skip" value - when considering "zero_niter" and
"zero_freeze", we only count iterations for which the AST model
is subtracted (i.e. the ones following the initial "ast.skip"
iterations). */
astMapGet0A( config, "AST", &akm );
astMapGet0I( akm, "SKIP", &skip );
akm = astAnnul( akm );
/* Get the number of iterations over which the mask is to be applied. Zero
means all. Return with no mask if this number of iterations has
already been performed. */
zero_niter = 0;
astMapGet0I( subkm, "ZERO_NITER", &zero_niter );
if( zero_niter == 0 || dat->iter < zero_niter + skip ) {
/* Only return a mask if this is not the last iteration, or if ZERO_NOTLAST
is unset. */
zero_notlast = 0;
astMapGet0I( subkm, "ZERO_NOTLAST", &zero_notlast );
if( !( flags & SMF__DIMM_LASTITER ) || !zero_notlast ) {
/* Create a list of the mask types to be combined to get the final mask by
looking for non-default values for the corresponding configuration
parameters in the supplied KeyMap. Static masks (predefined, circles
or external NDFs) may be used on any iteration, but dynamic masks
(lowhits, snr) will only be avialable once the map has been determined
at the end of the first iteration. This means that when masking anything
but the AST model (which is determined after the map), the dynamic masks
cannot be used on the first iteration. Make a note if all masks being
used are static. */
isstatic = 1;
nmask = 0;
zero_lowhits = 0.0;
astMapGet0D( subkm, "ZERO_LOWHITS", &zero_lowhits );
if( zero_lowhits > 0.0 ) {
if( mtype == SMF__AST || !( flags & SMF__DIMM_FIRSTITER ) ) {
mask_types[ nmask++] = LOWHITS;
isstatic = 0;
}
} else if( zero_lowhits < 0.0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter %s.ZERO_LOWHITS (%g) - "
"it must not be negative.", status, modname, zero_lowhits );
}
if( astMapGet1D( subkm, "ZERO_CIRCLE", 3, &zero_c_n, zero_circle ) ) {
if( zero_c_n == 1 || zero_c_n == 3 ) {
mask_types[ nmask++] = CIRCLE;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad number of values (%d) for config parameter "
"%s.ZERO_CIRCLE - must be 1 or 3.", status, zero_c_n,
modname );
}
}
cval = NULL;
astMapGet0C( subkm, "ZERO_MASK", &cval );
if( cval ) {
if( !astChrMatch( cval, "REF" ) &&
!astChrMatch( cval, "MASK2" ) &&
!astChrMatch( cval, "MASK3" ) ) {
astMapGet0I( subkm, "ZERO_MASK", &zero_mask );
cval = ( zero_mask > 0 ) ? "REF" : NULL;
}
if( cval ) {
strcpy( refparam, cval );
astChrCase( NULL, refparam, 1, 0 );
mask_types[ nmask++] = REFNDF;
}
}
zero_snr = 0.0;
astMapGet0D( subkm, "ZERO_SNR", &zero_snr );
if( zero_snr > 0.0 ) {
if( mtype == SMF__AST || !( flags & SMF__DIMM_FIRSTITER ) ) {
mask_types[ nmask++] = SNR;
isstatic = 0;
}
} else if( zero_snr < 0.0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter %s.ZERO_SNR (%g) - "
"it must not be negative.", status, modname, zero_snr );
}
if( astMapHasKey( subkm, "ZERO_MASK_POINTER" ) ) {
astMapGet0P( subkm, "ZERO_MASK_POINTER", (void **) &predef );
if( predef ) mask_types[ nmask++] = PREDEFINED;
}
/* No need to create a mask if no masking was requested or possible. */
if( nmask > 0 ) {
/* Decide if we are using the union or intersection of the masks. */
astMapGet0I( subkm, "ZERO_UNION", &munion );
/* Note if a mask existed on entry. If not, create a mask now, and
initialise it to hold the mask defined by the initial sky map. */
if( *mask == NULL ) {
have_mask = 0;
if( dat->initqual ) {
*mask = astMalloc( dat->msize*sizeof( **mask ) );
if( *mask ) {
pm = *mask;
pq = dat->initqual;
for( i = 0; i < dat->msize; i++ ) {
*(pm++) = ( *(pq++) != 0 );
}
}
} else{
*mask = astCalloc( dat->msize, sizeof( **mask ) );
}
} else {
have_mask = 1;
}
/* If we are combining more than one mask, we need work space to hold
an individual mask independently of the total mask. If we are using
only one mask, then just use the main mask array. */
if( nmask > 1 ) {
newmask = astMalloc( dat->msize*sizeof( *newmask ) );
} else {
newmask = *mask;
}
/* Get the number of iterations after which the mask is to be frozen.
Zero means "never freeze the mask". */
int zero_freeze = 0;
astMapGet0I( subkm, "ZERO_FREEZE", &zero_freeze );
/* Loop round each type of mask to be used. */
for( imask = 0; imask < nmask && *status == SAI__OK; imask++ ){
/* If the mask is now frozen, we just return the existing mask. So leave the
loop. */
if( zero_freeze != 0 && dat->iter > zero_freeze + skip ) {
break;
/* Low hits masking... */
} else if( mask_types[ imask ] == LOWHITS ) {
/* Set hits pixels with 0 hits to VAL__BADI so that stats1 ignores them */
ph = dat->hitsmap;
for( i = 0; i < dat->msize; i++,ph++ ) {
if( *ph == 0 ) *ph = VAL__BADI;
}
/* Find the mean hits in the map */
smf_stats1I( dat->hitsmap, 1, dat->msize, NULL, 0, 0, &meanhits,
NULL, NULL, &ngood, status );
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: mean hits = %lf, ngood "
"= %zd", status, meanhits, ngood );
/* Create the mask */
thresh = meanhits*zero_lowhits;
ph = dat->hitsmap;
pn = newmask;
for( i = 0; i < dat->msize; i++,ph++ ) {
*(pn++) = ( *ph != VAL__BADI && *ph < thresh ) ? 1 : 0;
}
/* Report masking info. */
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: masking %s "
"model at hits = %d.", status, modname, thresh );
/* Circle masking... */
} else if( mask_types[ imask ] == CIRCLE ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the circle
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* If only one parameter supplied it is radius, assume reference
LON/LAT from the frameset to get the centre. If the SkyFrame
represents offsets from the reference position (i.e. the source is
moving), assume the circle is to be centred on the origin. */
if( zero_c_n == 1 ) {
zero_circle[ 2 ] = zero_circle[ 0 ];
skyrefis = astGetC( dat->outfset, "SkyRefIs" );
if( skyrefis && !strcmp( skyrefis, "Origin" ) ) {
zero_circle[ 0 ] = 0.0;
zero_circle[ 1 ] = 0.0;
} else {
zero_circle[ 0 ] = astGetD( dat->outfset, "SkyRef(1)" );
zero_circle[ 1 ] = astGetD( dat->outfset, "SkyRef(2)" );
}
zero_circle[ 0 ] *= AST__DR2D;
zero_circle[ 1 ] *= AST__DR2D;
}
/* The supplied bounds are for pixel coordinates... we need bounds for grid
coordinates which have an offset */
lbnd_grid[ 0 ] = 1;
lbnd_grid[ 1 ] = 1;
ubnd_grid[ 0 ] = dat->ubnd_out[ 0 ] - dat->lbnd_out[ 0 ] + 1;
ubnd_grid[ 1 ] = dat->ubnd_out[ 1 ] - dat->lbnd_out[ 1 ] + 1;
/* Coordinates & radius of the circular region converted from degrees
to radians */
centre[ 0 ] = zero_circle[ 0 ]*AST__DD2R;
centre[ 1 ] = zero_circle[ 1 ]*AST__DD2R;
radius[ 0 ] = zero_circle[ 2 ]*AST__DD2R;
/* Create the Circle, defined in the current Frame of the FrameSet (i.e.
the sky frame). */
circle = astCircle( astGetFrame( dat->outfset, AST__CURRENT), 1,
centre, radius, NULL, " " );
/* Fill the mask with zeros. */
memset( newmask, 0, sizeof( *newmask )*dat->msize );
/* Get the mapping from the sky frame (current) to the grid frame (base),
and then set the mask to 1 for all of the values outside this circle */
astMaskUB( circle, astGetMapping( dat->outfset, AST__CURRENT,
AST__BASE ),
0, 2, lbnd_grid, ubnd_grid, newmask, 1 );
/* Report masking info. */
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model will"
" be masked %s using a circle of "
"radius %g arc-secs, centred at %s=%s, %s=%s.",
status, modname, words, radius[0]*AST__DR2D*3600,
astGetC( dat->outfset, "Symbol(1)" ),
astFormat( dat->outfset, 1, centre[ 0 ] ),
astGetC( dat->outfset, "Symbol(2)" ),
astFormat( dat->outfset, 2, centre[ 1 ] ) );
}
/* Reference NDF masking... */
} else if( mask_types[ imask ] == REFNDF ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the NDF
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* Begin an NDF context. */
ndfBegin();
/* Get an identifier for the NDF using the associated ADAM parameter. */
ndfAssoc( refparam, "READ", &indf1, status );
/* Get a section from this NDF that matches the bounds of the map. */
ndfSect( indf1, 2, dat->lbnd_out, dat->ubnd_out, &indf2,
status );
/* Map the section. */
ndfMap( indf2, "DATA", "_DOUBLE", "READ", (void **) &ptr,
&nel, status );
/* Check we can use the pointer safely. */
if( *status == SAI__OK ) {
/* Find bad pixels in the NDF and set those pixels to 1 in the mask. */
pn = newmask;
for( i = 0; i < dat->msize; i++ ) {
*(pn++) = ( *(ptr++) == VAL__BADD ) ? 1 : 0;
}
/* Report masking info. */
ndfMsg( "N", indf2 );
msgSetc( "M", modname );
if( zero_niter == 0 ) {
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The ^M "
"model will be masked on each iteration "
"using the bad pixels in NDF '^N'.",
status );
} else {
msgSeti( "I", zero_niter );
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The ^M "
"model will be masked for ^I iterations "
"using the bad pixels in NDF '^N'.",
status );
}
}
/* End the NDF context. */
ndfEnd( status );
}
/* SNR masking... */
} else if( mask_types[ imask ] == SNR ) {
/* Get the lower SNR limit. */
zero_snrlo = 0.0;
astMapGet0D( subkm, "ZERO_SNRLO", &zero_snrlo );
if( zero_snrlo <= 0.0 ) {
zero_snrlo = zero_snr;
} else if( zero_snrlo > zero_snr && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter "
"%s.ZERO_SNRLO (%g) - it must not be higher "
"than %s.ZERO_SNR (%g).", status, modname,
zero_snrlo, modname, zero_snr );
}
/* If the higher and lower SNR limits are equal, just do a simple
threshold on the SNR values to get the mask. */
if( zero_snr == zero_snrlo ) {
pd = dat->map;
pv = dat->mapvar;
pn = newmask;
for( i = 0; i < dat->msize; i++,pd++,pv++ ) {
*(pn++) = ( *pd != VAL__BADD && *pv != VAL__BADD &&
*pv >= 0.0 && *pd < zero_snr*sqrt( *pv ) ) ? 1 : 0;
}
/* Report masking info. */
if( !have_mask ) {
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using an SNR limit of %g.",
status, modname, words, zero_snr );
}
/* If the higher and lower SNR limits are different, create an initial
mask by thresholding at the ZERO_SNR value, and then extend the source
areas within the mask down to an SNR limit of ZERO_SNRLO. */
} else {
smf_snrmask( wf, dat->map, dat->mapvar, dat->mdims,
zero_snr, zero_snrlo, newmask, status );
/* Report masking info. */
if( !have_mask ) {
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using an SNR limit of %g "
"extended down to %g.", status, modname,
words, zero_snr, zero_snrlo );
}
}
/* Predefined masking... */
} else if( mask_types[ imask ] == PREDEFINED ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* Find bad pixels in the predefined array and set those pixels to 1 in
the mask. */
pn = newmask;
for( i = 0; i < dat->msize; i++ ) {
*(pn++) = ( *(predef++) == VAL__BADD ) ? 1 : 0;
}
/* Report masking info. */
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using a smoothed form of "
"the final mask created with the previous map.",
status, modname, words );
}
}
/* If required, add the new mask into the returned mask. If this is the
first mask, we just copy the new mask to form the returned mask.
Otherwise, we combine it with the existing returned mask. */
if( ! have_mask || ! isstatic ) {
if( nmask > 1 ) {
if( imask == 0 ) {
memcpy( *mask, newmask, dat->msize*sizeof(*newmask));
} else {
pm = *mask;
pn = newmask;
if( munion ) {
for( i = 0; i < dat->msize; i++,pm++ ) {
if( *(pn++) == 0 ) *pm = 0;
}
} else {
for( i = 0; i < dat->msize; i++,pm++ ) {
if( *(pn++) == 1 ) *pm = 1;
}
}
}
}
}
}
/* Free the individual mask work array if it was used. */
if( nmask > 1 ) newmask = astFree( newmask );
/* Check that the mask has some source pixels (i.e. pixels that have non-bad data values -
we do not also check variance values since they are not available until the second
iteration). */
if( *status == SAI__OK ) {
nsource = 0;
pm = *mask;
pd = dat->map;
for( i = 0; i < dat->msize; i++,pd++,pv++,pm++ ) {
if( *pd != VAL__BADD && *pm == 0 ) nsource++;
}
if( nsource < 5 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", "The %s mask being used has fewer than 5 "
"source pixels.", status, modname );
if( zero_snr > 0.0 ) {
errRepf( "", "Maybe your zero_snr value (%g) is too high?",
status, zero_snr );
}
}
}
/* Return the mask pointer if all has gone well. */
if( *status == SAI__OK ) result = *mask;
}
}
}
/* End the AST context, annulling all AST Objects created in the context. */
astEnd;
/* Return the pointer to the boolean mask. */
return result;
}
void smf__calc_wvm_job( void *job_data, int *status ) {
struct timeval tv1;
struct timeval tv2;
smfData * curdata = NULL;
smfArray * thesedata;
double prevtime = VAL__BADD;
double prevtau = VAL__BADD;
double lastgoodtau = VAL__BADD; /* most recent good tau */
size_t lastgoodidx = SMF__BADSZT; /* index of most recent good value */
size_t nbadidx = 0; /* number of time slices in the current gap */
size_t maxgap;
dim_t t1;
dim_t t2;
size_t nrelated;
double * taudata = NULL;
size_t ngood = 0;
dim_t i;
smfCalcWvmJobData *pdata;
double amprev;
AstKeyMap * extpars;
if (*status != SAI__OK) return;
pdata = (smfCalcWvmJobData *)job_data;
t1 = pdata->t1;
t2 = pdata->t2;
amprev = pdata->airmass;
thesedata = pdata->thesedata;
taudata = pdata->taudata;
nrelated = thesedata->ndat;
extpars = pdata->extpars;
maxgap = pdata->maxgap;
/* Lock the AST pointers to this thread */
astLock( extpars, 0 );
for (i=0;i<thesedata->ndat;i++) {
smf_lock_data( (thesedata->sdata)[i], 1, status );
}
/* Debugging message indicating thread started work */
msgOutiff( SMF__TIMER_MSG, "", "smfCalcSmoothedWVM: thread starting on slices %" DIM_T_FMT
" -- %" DIM_T_FMT,
status, t1, t2 );
smf_timerinit( &tv1, &tv2, status);
for (i=t1; i<=t2; i++) {
if (!curdata) {
SELECT_DATA( thesedata, curdata, VAL__BADD, wvm_time, i );
}
if (curdata) smf_tslice_ast( curdata, i, 0, NO_FTS, status );
if ( !curdata || curdata->hdr->state->wvm_time == VAL__BADD ) {
/* Try the other datas */
SELECT_DATA( thesedata, curdata, VAL__BADD, wvm_time, i );
}
if (*status != SAI__OK) break;
/* if we have no good data we store a bad value */
if (!curdata) {
prevtau = VAL__BADD;
} else {
const JCMTState * state = NULL;
state = curdata->hdr->state;
/* if we have old values from the WVM or no value we don't trust them */
if ( state->wvm_time != VAL__BADD &&
(fabs(state->wvm_time - state->rts_end) * SPD) < 60.0 ) {
/* Only calculate a tau when we have new values */
if ( prevtime != state->wvm_time ) {
double thistau = VAL__BADD;
double airmass = VAL__BADD;
prevtime = state->wvm_time;
airmass = state->tcs_airmass;
if (airmass == VAL__BADD) {
airmass = amprev;
} else {
amprev = airmass;
}
thistau = smf_calc_wvm( curdata->hdr, airmass, extpars, status );
/* Check status and/or value of tau */
if ( thistau == VAL__BADD ) {
if ( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf("", "Error calculating tau from WVM temperatures at time slice %" DIM_T_FMT,
status, i);
}
} else if ( thistau < 0.0 ) {
msgOutiff( MSG__QUIET, "", "WARNING: Negative WVM tau calculated (%g). Ignoring.",
status, thistau );
prevtau = VAL__BADD;
} else {
prevtau = thistau;
}
} else {
/* We use the previous tau since we should have calculated it earlier */
}
} else {
/* No good reading so tau is bad */
prevtau = VAL__BADD;
}
}
/* Prevtau is the tau that should be assigned to the current position */
/* see about gaps */
if (prevtau == VAL__BADD) {
nbadidx++;
} else {
/* we have a good value so we now have to see if there is a gap to fill */
if (i > 0 && lastgoodidx != (i-1) ) {
/* the previous value was bad so we may have to patch up if small */
if ( nbadidx < maxgap ) {
size_t j;
if (lastgoodidx == SMF__BADSZT) {
/* gap is at the start so fill with current value */
for (j=t1; j<i;j++) {
taudata[j] = prevtau;
ngood++;
}
} else {
/* replace with mean value */
double meantau = (lastgoodtau + prevtau) / 2.0;
for (j=lastgoodidx+1; j<i; j++) {
taudata[j] = meantau;
ngood++;
}
}
}
}
/* we know this index was good */
lastgoodidx = i;
lastgoodtau = prevtau;
nbadidx = 0;
ngood++;
}
/* Store the current tau value */
taudata[i] = prevtau;
}
/* if the last value in the time series was bad we need to see about
filling with the last good value */
if (*status == SAI__OK && nbadidx > 0 && nbadidx < maxgap) {
for (i=lastgoodidx+1; i<=t2; i++) {
taudata[i] = lastgoodtau;
ngood++;
}
}
/* Report the time taken in this thread. */
msgOutiff( SMF__TIMER_MSG, "",
"smfCalcSmoothedWVM: thread finishing slices %" DIM_T_FMT
" -- %" DIM_T_FMT " (%zu good) (%.3f sec)",
status, t1, t2, ngood, smf_timerupdate( &tv1, &tv2, status ) );
/* Store number of good values */
pdata->ngood = ngood;
/* Unlock the AST pointers from this thread */
astUnlock( extpars, 1 );
for (i=0;i<thesedata->ndat;i++) {
smf_lock_data( (thesedata->sdata)[i], 0, status );
}
}
void smf_mask_noisy( ThrWorkForce *wf, smfData *data, smfData **noise,
double sigcliphigh, double sigcliplow, int cliplog,
int zeropad, int * status ) {
size_t i;
dim_t nbolo = 0; /* Number of bolometers */
double *noisedata = NULL; /* Pointer to the noise data array */
smfData * noisemap = NULL; /* Somewhere to receive the result */
double *work = NULL; /* Temporary work array */
if (*status != SAI__OK) return;
if( (sigcliphigh <= 0.0) && (sigcliplow <= 0.0) ) return;
/* Work out how many bolometers we have */
smf_get_dims( data, NULL, NULL, &nbolo, NULL, NULL,
NULL, NULL, status );
/* Create some space for the result */
smf_create_bolfile( wf, NULL, 1, data, "Noise", "blahs s**0.5",
SMF__MAP_VAR, &noisemap, status );
if (noisemap) noisedata = (noisemap->pntr)[0];
/* Calculate the noise on each bolometer */
smf_bolonoise( wf, data, -1.0, 0, 0.5, SMF__F_WHITELO,
SMF__F_WHITEHI, 0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT,
(noisemap->pntr)[0], NULL, NULL, status );
/* Now need to convert this to noise by square rooting */
if (*status == SAI__OK) {
for (i=0; i<nbolo; i++) {
if (noisedata[i] != VAL__BADD) {
noisedata[i] = sqrt( noisedata[i] );
}
}
}
/* Now create a mask and then mask the quality. We only mask bolometers
that are too noisy. We also do not mask bolometers that were already
masked. We want the statistics to reflect what we actually masked
and not any previous masking. This will miss any bolometers masked
out by smf_bolonoise because they had zero power. */
msgOutif( MSG__VERB, "", FUNC_NAME
": Checking for bolometers with outlier noise values. ", status );
msgOutiff( MSG__VERB, "", FUNC_NAME
": Units are (%s s**0.5): ", status, data->hdr->units );
work = astCalloc( nbolo, sizeof(*work) );
if( *status == SAI__OK ) memcpy( work, noisedata, nbolo*sizeof(*work) );
smf_clipnoise( noisedata, nbolo, cliplog, sigcliplow, sigcliphigh, NULL,
status );
/* The only bad values that should appear in noisedata are new bolometers
that were clipped by smf_clipnoise, not bolometers that were bad before
for other reasons. */
if( *status == SAI__OK ) {
for( i=0; i<nbolo; i++ ) {
if( (noisedata[i]==VAL__BADD) && (work[i] == VAL__BADD) ) {
noisedata[i] = 0;
}
}
}
if( work ) work = astFree( work );
/* The mask has to be inside a smfArray */
if (*status == SAI__OK) {
smfArray *masks = smf_create_smfArray( status );
if (masks) masks->owndata = 0; /* someone else owns the smfData */
smf_addto_smfArray( masks, noisemap, status );
smf_apply_mask( wf, data, masks, SMF__BBM_QUAL, SMF__Q_NOISE,
status );
smf_close_related( wf, &masks, status );
}
/* Give noisemap back to caller if requested, or close it */
if( noise ) {
*noise = noisemap;
} else {
smf_close_file( wf, &noisemap, status );
}
}
static void smfFlagSpikesPar( void *job_data_ptr, int *status ) {
size_t box; /* Box size */
size_t bstride; /* Vector stride between bolometer samples */
double *dat = NULL; /* Pointer to bolo data */
double dnew; /* Data value being added into the filter box */
double dold; /* Data value being removed from the filter box*/
int iadd; /* Index within box at which to store new value*/
dim_t ibolo; /* Bolometer index */
dim_t ibox; /* Index within box */
dim_t iold; /* Index of oldest value in "w2" */
dim_t inbox; /* Number of values in current filter box */
int inoise; /* Index within noisebox element to be removed */
int iremove; /* Index within box of element to be removed */
dim_t itime; /* Time-slice index */
dim_t lasttime; /* Last time-slice index to check for spikes */
double lmedian; /* Median value in previous filter box */
smf_qual_t mask; /* quality bit mask */
double median; /* Median value in current filter box */
dim_t nbolo; /* Number of bolometers */
size_t newstride; /* Vector stride to new time sample */
size_t nflag; /* Number of samples flagged */
int nn; /* Number of good values in noise box */
double noise; /* Local noise estimate */
double *noisebox = NULL; /* Pointer to array holding values for
calculating local noise */
double nsum; /* Sum of values in noise box */
double nsum2; /* Sum of squared values in noise box */
dim_t ntime; /* Number of time-slices */
double *pdat = NULL; /* Pointer to next bolo data value */
double *pdat0 = NULL; /* Pointer to first bolo data value */
double *pdat1 = NULL; /* Pointer to last bolo data value */
smfFlagSpikesData *pdata=NULL;
double *pn = NULL; /* Pointer to next "noisebox" value */
smf_qual_t *pqua = NULL; /* Pointer to next quality flag */
smf_qual_t *pqua0 = NULL; /* Pointer to first bolo quality value */
smf_qual_t *pqua1 = NULL; /* Pointer to last bolo quality value */
double *pw1 = NULL; /* Pointer to next "w1" value */
double *pw2 = NULL; /* Pointer to next "w2" value */
smf_qual_t *qua = NULL; /* Pointer to quality flags */
int spike; /* Is current time slice a spike? */
double thresh; /* threshold for spikes */
size_t tstride; /* Vector stride between time samples */
double umedian; /* Lagged median value */
double *w1 = NULL; /* Array holding sorted data values */
double *w2 = NULL; /* Array holding un-sorted data values */
/* Retrieve job data */
pdata = job_data_ptr;
box = pdata->box;
bstride = pdata->bstride;
dat = pdata->dat;
mask = pdata->mask;
nbolo = pdata->nbolo;
ntime = pdata->ntime;
qua = pdata->qua;
thresh = pdata->thresh;
tstride = pdata->tstride;
/* Debugging message indicating thread started work */
msgOutiff( SMF__TIMER_MSG, "",
"smfFlagSpikesPar: thread starting on bolos %zu -- %zu",
status, pdata->b1, pdata->b2 );
/* Initialise the number of spikes found. */
nflag = 0;
/* Store the largest time slice index to be checked. */
lasttime = ntime - ( box + 1 )/2;
/* Store the offset from the central value in a filter box to the next
time slice value to enter the filter box (i.e. the time slice just
in front of the current filter box). */
newstride = ( ( box + 1 )/2 )*tstride;
/* Allocate work arrays. */
w1 = astMalloc( sizeof( *w1 )*box );
w2 = astMalloc( sizeof( *w2 )*box );
noisebox = astMalloc( sizeof( *noisebox )*box );
/* Initialise pointers to the first data value and quality value for the
first bolometer. */
pdat0 = dat + pdata->b1*bstride;
pqua0 = qua + pdata->b1*bstride;
/* Initialise pointers to the last data value and quality value for the first
bolometer. */
pdat1 = dat + ( ntime - 1 )*tstride + pdata->b1*bstride;
pqua1 = qua + ( ntime - 1 )*tstride + pdata->b1*bstride;
/* We process each bolometer in turn for this block. */
for( ibolo=pdata->b1; ibolo<=pdata->b2 && *status == SAI__OK; ibolo++ ) {
/* Ignore bad bolometers. */
if( !( *pqua0 & SMF__Q_BADB ) ) {
/* Initialise the running sums used to calculate the standard deviation
of the values in the initial noise box. */
nsum = 0.0;
nsum2 = 0.0;
nn = 0;
/* The highest element in the noise box is the element just below the
central element of the filter box (so the central filter box element
does not affect the noise level estimate). Initially, the lower half
of the noise box is off the edge of the array and so we fill it with bad
values. */
pn = noisebox;
for( ibox = 0; ibox < ( box + 1 )/2; ibox++ ) *(pn++) = VAL__BADD;
/* Initialise the filter box to contain the first "box" values from the
current bolometer time-series. Do not store bad or flagged values in
the filter box. The good values are stored at the start of the "w1"
array, with no gaps. The "w2" array holds all values in the box, good
or bad, in the order they occur in the time-series (i.e. un-sorted). */
pdat = pdat0;
pqua = pqua0;
pw1 = w1;
pw2 = w2;
for( ibox = 0; ibox < box; ibox++ ) {
if( !( *pqua & mask ) && *pdat != VAL__BADD ) {
*(pw2++) = *(pw1++) = *pdat;
} else {
*(pw2++) = VAL__BADD;
}
/* Also store values these data values in the second half of the noise
box. Check we have not reached the end of the noise box. */
if( pn - noisebox < (int) box ) {
/* Store the value in the next element of the noise box, and if the value is
good, update the running sums used for calculating the noise level. */
if( ( *(pn++) = pw2[-1] ) != VAL__BADD ) {
nsum += *pdat;
nsum2 += (*pdat)*(*pdat);
nn++;
}
}
/* Get pointers to the next data and quality values for the current
bolometer. */
pdat += tstride;
pqua += tstride;
}
/* Calculate the standard deviation of the values in the initial noise box.
We require at least three values. */
if( nn > 3 ) {
noise = ( nsum2 - (nsum*nsum)/nn )/( nn - 1 );
if( noise > 0 ) {
noise = sqrt( noise );
} else {
noise = VAL__BADD;
}
} else {
noise = VAL__BADD;
}
/* Store the index within the noise box at which to store the next value
(the new value will over-write the oldest value in the noise box -
i.e. the first element). */
inoise = 0;
/* Initialise the index at which to store the next bolometer data value in the
"w2" array. The first new value added to the box will over-write element
zero - the oldest value in the box. */
iold = 0;
/* Note the number of good values stored in the filter box. */
inbox = (int)( pw1 - w1 );
/* If there are any bad data values, pad out the w1 array with bad
values. */
for( ibox = inbox; ibox < box; ibox++ ) w1[ ibox ] = VAL__BADD;
/* If any good values are stored in the filter box, we now sort them. */
if( inbox > 0 ) gsl_sort( w1, 1, inbox );
/* Get pointers to the data value and quality value at the centre of the
first filter box. */
pdat = pdat0 + (box/2)*tstride;
pqua = pqua0 + (box/2)*tstride;
/* Initialise the median value in the previous box to "unknown". */
lmedian = VAL__BADD;
/* Loop round all time slices, except for the first and last "box/2"
slices, which are left unchanged. "itime" is the index of the time
slice at the centre of the filter box. */
for( itime = box/2; itime <= lasttime; itime++ ) {
/* Assume this time slice is not a spike. */
spike = 0;
/* If the current sorted filter box contains an odd number of good values,
use the central good value as the median value. If the box contains an
even number of good values, use the mean of the two central values as
the median value. If the box is empty use VAL__BADD. */
if( inbox == 0 ) {
median = VAL__BADD;
} else if( inbox % 2 == 1 ) {
median = w1[ inbox/2 ];
} else {
ibox = inbox/2;
median = 0.5*( w1[ ibox ] + w1[ ibox - 1 ] );
}
/* If we have all the values we need, we can check for a spike. */
if( !( *pqua & mask ) && *pdat != VAL__BADD &&
median != VAL__BADD && noise != VAL__BADD ) {
/* The median is a robust but intrinsically noisey estimator of the expected
value. So we smooth the median values a bit by using the mean of the
current median and the previous median. */
if( lmedian != VAL__BADD ) {
umedian = 0.5*( median + lmedian );
} else {
umedian = median;
}
/* If the central value in the filter box deviates from the median by more than
"thresh" times the current noise estimate, flag it as a spike. Note, we can
modify the quality array directly because we will not be re-reading the value
being modified (all the reading is done at the leading edge of the filter
box). */
if( fabs( *pdat - umedian ) > thresh*noise ) {
*pqua |= SMF__Q_SPIKE;
nflag++;
spike = 1;
}
}
/* Save the median in the current box so that we can use it to smooth the
next box. */
lmedian = median;
/* Now move the filter box on by one time sample so that we are ready for
the next "itime" value. Do not do this if we have just done the last
itime value. */
if( itime < lasttime ) {
/* Get the data value for the time slice that is about to enter the filter
box. Set it bad if it is flagged in the quality array. */
dnew = pdat[ newstride ];
if( pqua[ newstride ] & mask ) dnew = VAL__BADD;
/* Get the data value for the time slice that is about to leave the filter
box. */
dold = w2[ iold ];
/* Store the new value in "w2" in place of the old value, and then
increment the index of the next "w2" value to be removed, wrapping back
to the start when the end of the array is reached. */
w2[ iold++ ] = dnew;
if( iold == box ) iold = 0;
/* If the new value to be added into the box is good... */
if( dnew != VAL__BADD ) {
/* Find the index (iadd) within the w1 box at which to store the new
value so as to maintain the ordering of the values in the box. Could do
a binary chop here, but the box size is presumably going to be very
small and so it's probably not worth it. At the same time, look for
the value that is leaving the box (if it is not bad). */
iremove = -1;
iadd = -1;
if( dold != VAL__BADD ) {
for( ibox = 0; ibox < inbox; ibox++ ) {
if( iremove == -1 && w1[ ibox ] == dold ) {
iremove = ibox;
if( iadd != -1 ) break;
}
if( iadd == -1 && w1[ ibox ] >= dnew ) {
iadd = ibox;
if( iremove != -1 ) break;
}
}
} else {
for( iadd = 0; iadd < (int) inbox; iadd++ ) {
if( w1[ iadd ] >= dnew ) break;
}
}
/* If the new value is larger than any value currently in w1, we add it
to the end. */
if( iadd == -1 ) iadd = inbox;
/* If the value being removed is bad, shuffle all the good values greater
than the new value up one element, and increment the number of good
values for this bolometer box. */
if( iremove == -1 ) {
for( ibox = inbox; (int) ibox > iadd; ibox-- ) {
w1[ ibox ] = w1[ ibox - 1 ];
}
inbox++;
/* If the value being removed is good and the value being added is greater
than the value being removed, shuffle all the intermediate values down
one element within the box. */
} else if( (int) iadd > iremove ) {
for( ibox = iremove; (int) ibox < iadd - 1; ibox++ ) {
w1[ ibox ] = w1[ ibox + 1 ];
}
iadd--;
/* If the value being removed is good and the value being added is less
than or equal to the value being removed, shuffle all the intermediate
values up one element within the box. */
} else {
for( ibox = iremove; (int) ibox > iadd; ibox-- ) {
w1[ ibox ] = w1[ ibox - 1 ];
}
}
/* Store the new value in the box. */
w1[ iadd ] = dnew;
/* If the value being added is bad but the value being removed is good... */
} else if( dold != VAL__BADD ){
/* Find the index (iremove) within the w1 box at which is stored the value
that is leaving the box. */
iremove = -1;
for( iremove = 0; iremove < (int) inbox; iremove++ ) {
if( w1[ iremove ] == dold ) break;
}
/* Move all the larger values down one element in "w1" to fill the gap
left by the removal. */
for( iremove++; iremove < (int) inbox; iremove++ ) {
w1[ iremove - 1 ] = w1[ iremove ];
}
/* Over-write the un-used last element with a bad value, and decrement
the number of values in "w1". */
w1[ iremove - 1 ] = VAL__BADD;
inbox--;
}
/* Get the new value (the central value in the filter box) to add to the
noise box. If the central value was found to be a spike, do not
include it in the noise box as it would upset the estimate of the
local noise. */
if( spike ) {
dnew = VAL__BADD;
} else {
dnew = *pdat;
if( *pqua & mask ) dnew = VAL__BADD;
}
/* Get the old value to remove from the noise box. */
dold = noisebox[ inoise ];
/* If the value being added is good, add it into the running sums used to
calculate the standard deviation of the values in the local noise box. */
if( dnew != VAL__BADD ) {
nsum += dnew;
nsum2 += dnew*dnew;
nn++;
}
/* If the value being removed is good, remove it from the running sums used
to calculate the standard deviation of the values in the local noise box. */
if( dold != VAL__BADD ) {
nsum -= dold;
nsum2 -= dold*dold;
nn--;
}
/* Calculate the standard deviation of the values in the new noise box.
We require at least three values. */
if( nn > 3 ) {
noise = ( nsum2 - (nsum*nsum)/nn )/( nn - 1 );
if( noise > 0 ) {
noise = sqrt( noise );
} else {
noise = VAL__BADD;
}
} else {
noise = VAL__BADD;
}
/* Store the new value in the noise box, over-writing the oldest value,
and increment the index at which to store the next value. If the index
reaches the end of the array, wrap around to the start of the array. */
noisebox[ inoise++ ] = dnew;
if( inoise == (int) box ) inoise = 0;
}
/* Get pointers to the next central data and quality value for the current
bolometer. */
pdat += tstride;
pqua += tstride;
}
}
/* Get pointers to the first data value and quality value for the next
bolometer. */
pdat0 += bstride;
pqua0 += bstride;
/* Get pointers to the last data value and quality value for the next
bolometer. */
pdat1 += bstride;
pqua1 += bstride;
}
/* Free resources. */
w1 = astFree( w1 );
w2 = astFree( w2 );
noisebox = astFree( noisebox );
/* Store number of flagged samples */
pdata->nflag = nflag;
/* Debugging message indicating thread finished work */
msgOutiff( SMF__TIMER_MSG, "",
"smfFlagSpikesPar: thread finishing bolos %zu -- %zu",
status, pdata->b1, pdata->b2 );
}
static void smf1_calcmodel_smo_job( void *job_data, int *status ) {
/*
* Name:
* smf1_calcmodel_smo_job
* Purpose:
* Smooth each bolometer.
* Invocation:
* void smf1_calcmodel_smo_job( void *job_data, int *status )
* Arguments:
* job_data = void * (Given)
* Pointer to the data needed by the job. Should be a pointer to a
* smfCalcmodelSmoJobData structure.
* status = int * (Given and Returned)
* Pointer to global status.
* Description:
* This routine runs in a worker thread to smooth the specified bolometer
* time streams using the specified filter.
*/
/* Local Variables: */
dim_t b1;
dim_t b2;
dim_t i;
dim_t nbolo;
dim_t ntslice;
double *boldata = NULL;
double *model_data;
double *res_data;
double *w1 = NULL;
int *w3 = NULL;
smf_filt_t filter_type;
size_t boxcar;
size_t bstride;
size_t tstride;
size_t *w2 = NULL;
smfCalcmodelSmoJobData *pdata;
smf_qual_t *bolqua = NULL;
smf_qual_t *qua_data;
struct timeval tv1;
struct timeval tv2;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Get a pointer to the job data, and then extract its contents into a
set of local variables. */
pdata = (smfCalcmodelSmoJobData *) job_data;
b1 = pdata->b1;
b2 = pdata->b2;
boxcar = pdata->boxcar;
bstride = pdata->bstride;
filter_type = pdata->filter_type;
model_data = pdata->model_data;
nbolo = pdata->nbolo;
ntslice = pdata->ntslice;
qua_data = pdata->qua_data;
res_data = pdata->res_data;
tstride = pdata->tstride;
/* Check we have something to do. */
if( b1 < nbolo ) {
/* Debugging message indicating thread started work */
msgOutiff( SMF__TIMER_MSG, "", "smf_calcmodel_smo: thread starting on bolos %" DIM_T_FMT
" -- %" DIM_T_FMT,
status, b1, b2 );
smf_timerinit( &tv1, &tv2, status);
/* Allocate work space */
boldata = astMalloc( ntslice*sizeof( *boldata ) );
bolqua = astMalloc( ntslice*sizeof( *bolqua ) );
if( filter_type == SMF__FILT_MEDIAN ) {
w1 = astMalloc( boxcar*sizeof( *w1 ) );
w2 = astMalloc( boxcar*sizeof( *w2 ) );
w3 = astMalloc( boxcar*sizeof( *w3 ) );
}
/* Loop round all the bolometers to be processed by this thread. */
for( i = b1; i <= b2 && *status == SAI__OK; i++ ) {
/* Smooth the data and subtract it from res */
size_t j;
size_t boloff = i*bstride;
/* Copy the data for this bolometer into a temporary buffer */
for (j=0; j<ntslice; j++) {
size_t thisidx = boloff+j*tstride;
boldata[j] = res_data[thisidx];
bolqua[j] = qua_data[thisidx];
}
/* Smooth that bolometer time series */
if( filter_type == SMF__FILT_MEDIAN ) {
smf_median_smooth( (dim_t) boxcar, SMF__FILT_MEDIAN, 0.0, ntslice,
res_data+boloff, qua_data+boloff, 1, SMF__Q_FIT,
boldata, w1, w2, w3, status );
} else {
smf_tophat1D( boldata, ntslice, boxcar, bolqua, SMF__Q_FIT,
0.0, status );
}
/* Remove this model from the residual data and copy the data to the model */
for (j=0; j<ntslice; j++) {
size_t thisidx = boloff+j*tstride;
/* Modify RES if this section has smoothable quality */
if (! (qua_data[thisidx] & SMF__Q_FIT) ) {
if (boldata[j] == VAL__BADD) {
/* Need to set to bad quality but since SMO can not
introduce bad values we shouldn't get here. For now
we use the COM quality bit just in case. */
qua_data[thisidx] |= SMF__Q_COM;
} else if (res_data[thisidx] != VAL__BADD) {
res_data[thisidx] -= boldata[j];
}
/* Store the model */
model_data[thisidx] = boldata[j];
} else {
/* store bad value in the model */
model_data[thisidx] = VAL__BADD;
}
}
}
/* Free work space. */
w1 = astFree( w1 );
w2 = astFree( w2 );
w3 = astFree( w3 );
boldata = astFree( boldata );
bolqua = astFree( bolqua );
/* Report the time taken in this thread. */
msgOutiff( SMF__TIMER_MSG, "",
"smf_calcmodel_smo: thread finishing bolos %" DIM_T_FMT
" -- %" DIM_T_FMT "(%.3f sec)",
status, b1, b2, smf_timerupdate( &tv1, &tv2, status ) );
}
}
void smf_calc_smoothedwvm ( ThrWorkForce *wf, const smfArray * alldata,
const smfData * adata, AstKeyMap* extpars, double **wvmtau,
size_t *nelems, size_t *ngoodvals, int * status ) {
size_t i;
size_t nrelated = 0; /* Number of entries in smfArray */
size_t nframes = 0; /* Number of timeslices */
size_t ngood = 0; /* Number of elements with good tau */
double *taudata = NULL; /* Local version of WVM tau */
const smfArray * thesedata = NULL; /* Collection of smfDatas to analyse */
smfArray * tmpthesedata = NULL; /* Local version of adata in a smfArray */
if (*status != SAI__OK) return;
if (alldata && adata) {
*status = SAI__ERROR;
errRep("", "smf_calc_smoothedwvm can not be given non-NULL alldata and non-NULL adata arguments"
" (possible programming error)", status );
return;
}
if (!alldata && !adata) {
*status = SAI__ERROR;
errRep("", "smf_calc_smoothedwvm: One of alldata or adata must be non-NULL",
status);
return;
}
if (!wvmtau) {
*status = SAI__ERROR;
errRep("", "Must supply a non-NULL pointer for wvmtau argument"
" (possible programming error)", status );
return;
}
/* if we have a single smfData put it in a smfArray */
if (alldata) {
if (alldata->ndat == 0 ) {
*status = SAI__ERROR;
errRep("", "No smfDatas present in supplied smfArray for WVM smoothing"
" (possible programming error)", status );
return;
}
thesedata = alldata;
} else {
tmpthesedata = smf_create_smfArray( status );
if (tmpthesedata) {
tmpthesedata->owndata = 0; /*not owned by the smfArray */
/* we know that the smfData here will not be touched in this
function so we do the BAD thing of casting const to non-const */
smf_addto_smfArray( tmpthesedata, (smfData *)adata, status );
}
thesedata = tmpthesedata;
}
/* Check that we have headers and that the smfData are the same length */
nrelated = thesedata->ndat;
for (i = 0; i < nrelated; i++ ) {
smfData * data = (thesedata->sdata)[i];
smfHead * hdr = data->hdr;
dim_t thisframes = 0;
if ( !hdr) {
*status = SAI__ERROR;
errRepf( "", "smfData %zu has no header. Aborting WVM smoothing",
status, i );
return;
}
smf_get_dims( data, NULL, NULL, NULL, &thisframes, NULL, NULL, NULL, status );
if (!nframes) nframes = thisframes;
if (thisframes != nframes) {
*status = SAI__ERROR;
errRepf( "", "smfData %zu has different length. Aborting WVM smoothing",
status, i );
return;
}
}
/* We will need the earliest and last airmass value in order
to calculate a zenith tau */
/* As a first step, just fill the time series with calculated WVM
tau values even though we know there are about 240 fewer tau
readings in reality. This initial approach will make it easier to
use the smoothed data directly rather than having to interpolate
from the 1.2 second data back into the 200 Hz data. */
taudata = astCalloc( nframes, sizeof(*taudata) );
if (*status == SAI__OK) {
double amprev = VAL__BADD;
double steptime;
size_t maxgap;
struct timeval tv1;
struct timeval tv2;
smfCalcWvmJobData *job_data = NULL;
int nworker;
/* We need to know the steptime so we can define the max good gap
in seconds and convert it to steps*/
steptime = (thesedata->sdata)[0]->hdr->steptime;
maxgap = (size_t)( 5.0 / steptime ); /* 5 seconds is just larger than 2 WVM readings */
/* Assume all files have the same airmass information */
smf_find_airmass_interval( (thesedata->sdata)[0]->hdr, &rev, NULL, NULL, NULL, status );
smf_timerinit( &tv1, &tv2, status );
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
if (*status == SAI__OK) {
dim_t tstep;
int iworker;
smfCalcWvmJobData *pdata = NULL;
/* Get the number of time slices to process in each thread. */
if( nworker > (int) nframes ) {
tstep = 1;
} else {
tstep = nframes/nworker;
}
/* to return the same values for one thread and multiple threads
we need to break the threads on wvm sample boundaries wherever
possible. We make an initial estimate of the number of WVM measurements
by assuming one every two seconds. */
{
smfData * curdata = NULL;
size_t nwvm = 0;
double prevtime = VAL__BADD;
double curtime;
size_t *boundaries = astGrow(NULL, nframes*(size_t)(steptime/2.0), sizeof(*boundaries));
for (i=0; i<nframes; i++) {
if (!curdata) {
SELECT_DATA( thesedata, curdata, VAL__BADD, wvm_time, i );
}
if (curdata) smf_tslice_ast( curdata, i, 0, NO_FTS, status );
if ( !curdata || curdata->hdr->state->wvm_time == VAL__BADD ) {
/* Try the other datas */
SELECT_DATA( thesedata, curdata, VAL__BADD, wvm_time, i );
}
if (*status != SAI__OK) break;
if (!curdata) {
curtime = VAL__BADD;
} else {
curtime = curdata->hdr->state->wvm_time;
}
if (curtime != prevtime || nwvm == 0 ) {
/* Store the index in the boundaries array */
nwvm++;
boundaries = astGrow(boundaries, nwvm, sizeof(*boundaries));
if (!boundaries) { /* this is serious */
if (*status == SAI__OK) *status = SAI__ERROR;
errRep("", "Error allocating temporary memory for WVM calculation\n",
status );
break;
}
boundaries[nwvm-1] = i;
prevtime = curtime;
}
}
/* No point using too many threads */
if (*status == SAI__OK) {
if (nworker >= (int)nwvm) {
nworker = nwvm;
/* Allocate a measurement per thread */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->t1 = boundaries[iworker];
if (iworker+1 < nworker) pdata->t2 = boundaries[iworker+1]-1;
}
/* Ensure that the last thread picks up any left-over time slices */
pdata->t2 = nframes - 1;
} else {
/* Allocate the workers to slices of approximate size tstep */
size_t prevend = 0; /* End of previous slice */
size_t prevbnd = 0; /* Index into previous boundaries[] array selection */
for( iworker = 0; iworker < nworker; iworker++ ) {
size_t belowidx = prevend+1;
size_t aboveidx = nframes;
size_t lbnd;
size_t ubnd;
size_t j;
size_t guess;
pdata = job_data + iworker;
if (iworker == 0) { /* always start at the beginning */
pdata->t1 = 0;
} else { /* Start one after the previous block */
pdata->t1 = prevend + 1;
}
/* Now we have to find the end of this slice */
guess = (iworker*tstep) + tstep - 1;
if (guess <= pdata->t1) guess = pdata->t1 + tstep;
/* find nearest boundaries */
for (j=prevbnd; j<nwvm; j++) {
if ( boundaries[j] > guess ) {
aboveidx = boundaries[j];
ubnd = j;
if (j>0) {
belowidx = boundaries[j-1];
lbnd = j -1 ;
} else {
lbnd = 0;
}
break;
}
}
/* Choose the closest, making sure that we are not choosing t1 */
if ( (guess - belowidx < aboveidx - guess) && belowidx > pdata->t1 ) {
pdata->t2 = belowidx - 1;
prevbnd = lbnd;
} else {
pdata->t2 = aboveidx - 1;
prevbnd = ubnd;
}
prevend = pdata->t2;
if (prevend == nframes - 1 && iworker < nworker-1 ) {
/* we have run out of slices so just use fewer workers */
nworker = iworker + 1;
break;
}
}
/* Ensure that the last thread picks up any left-over time slices */
pdata->t2 = nframes - 1;
}
/* Tidy up */
boundaries = astFree( boundaries );
}
}
/* Store all the other info needed by the worker threads, and submit the
jobs to fix the steps in each bolo, and then wait for them to complete. */
for( iworker = 0; iworker < nworker; iworker++ ) {
smfArray *thrdata = NULL;
pdata = job_data + iworker;
pdata->nframes = nframes;
pdata->airmass = amprev; /* really need to get it from the start of each chunk */
pdata->taudata = taudata;
pdata->maxgap = maxgap;
/* Need to copy the smfDatas and create a new smfArray for each
thread */
thrdata = smf_create_smfArray( status );
for (i=0;i<nrelated;i++) {
smfData *tmpdata = NULL;
tmpdata = smf_deepcopy_smfData( wf, (thesedata->sdata)[i], 0, SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS |
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY, 0, 0,
status );
smf_lock_data( tmpdata, 0, status );
smf_addto_smfArray( thrdata, tmpdata, status );
}
pdata->thesedata = thrdata;
/* Need to do a deep copy of ast data and unlock them */
pdata->extpars = astCopy(extpars);
astUnlock( pdata->extpars, 1 );
/* Pass the job to the workforce for execution. */
thrAddJob( wf, THR__REPORT_JOB, pdata, smf__calc_wvm_job, 0, NULL,
status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
/* Now free the resources we allocated during job creation
and calculate the number of good values */
for( iworker = 0; iworker < nworker; iworker++ ) {
smfArray * thrdata;
pdata = job_data + iworker;
astLock( pdata->extpars, 0 );
pdata->extpars = astAnnul( pdata->extpars );
thrdata = pdata->thesedata;
for (i=0;i<thrdata->ndat;i++) {
smf_lock_data( (thrdata->sdata)[i], 1, status );
}
smf_close_related( wf, &thrdata, status );
ngood += pdata->ngood;
}
}
job_data = astFree( job_data );
msgOutiff( MSG__NORM, "", FUNC_NAME ": %f s to calculate unsmoothed WVM tau values",
status, smf_timerupdate(&tv1,&tv2,status) );
}
if (*status == SAI__OK && extpars) {
/* Read extpars to see if we need to smooth */
double smoothtime = VAL__BADD;
if (astMapGet0D( extpars, "SMOOTHWVM", &smoothtime ) ) {
if (smoothtime != VAL__BADD && smoothtime > 0.0) {
smfData * data = (thesedata->sdata)[0];
double steptime = data->hdr->steptime;
dim_t boxcar = (dim_t)( smoothtime / steptime );
msgOutiff( MSG__VERB, "",
"Smoothing WVM data with %f s tophat function",
status, smoothtime );
smf_tophat1D( taudata, nframes, boxcar, NULL, 0, 0.0, status );
/* The tophat smoothing puts a bad value at the start and end of
the time series so we replace that with the adjacent value since
the step time is much smaller than WVM readout time. If more than
one value is bad we do not try to find the good value. */
taudata[0] = taudata[1];
taudata[nframes-1] = taudata[nframes-2];
}
}
}
/* Use this to get the raw WVM output for debugging */
/*
if (*status == SAI__OK) {
smfData *data = (thesedata->sdata)[0];
smfHead *hdr = data->hdr;
printf("# IDX TAU RTS_NUM RTS_END WVM_TIME\n");
for (i=0; i<nframes;i++) {
JCMTState * state;
state = &(hdr->allState)[i];
printf("%zu %.*g %d %.*g %.*g\n", i, DBL_DIG, taudata[i], state->rts_num,
DBL_DIG, state->rts_end, DBL_DIG, state->wvm_time);
}
} */
/* Free resources */
if (tmpthesedata) smf_close_related( wf, &tmpthesedata, status );
if (*status != SAI__OK) {
if (taudata) taudata = astFree( taudata );
*nelems = 0;
*ngoodvals = 0;
} else {
*wvmtau = taudata;
*nelems = nframes;
*ngoodvals = ngood;
}
}
