void fts2_validatemirrorpositions(double* positions, int count, int* ni, int* nf, int* status)
{
if(*status != SAI__OK) { return; }
const double EPSILON = 0.001;
int i = 0;
// CREATE SHIFTED MIRROR POSITIONS
double* shifted = (double*) astCalloc(count, sizeof(double));
for(i = 1; i < count; i++) {
shifted[i] = positions[i - 1];
}
shifted[0] = positions[count - 1];
// COMPUTE DELTA MIRROR POSITIONS
double* delta = (double*) astCalloc(count, sizeof(double));
for(i = 0; i < count; i++) {
delta[i] = positions[i] - shifted[i];
}
// FIND THE START INDEX
for(i = 0; i < count; i++) {
if(fabs(delta[i]) >= EPSILON) {
*ni = i;
break;
}
}
// FIND THE END INDEX
for(i = count - 1; i > -1; i--) {
if(fabs(delta[i]) >= EPSILON) {
*nf = i;
break;
}
}
// FIND THE END INDEX
for(i = 0; i < count - 1; i++) {
if(positions[i] > 325.0) {
*nf = i - 1;
break;
}
}
/*
// CHECK TO SEE IF THE POSITIONS HAVE REPEATING VALUES IN BETWEEN
for(i = *ni + 1; i < *nf; i++) {
if(fabs(delta[i]) <= EPSILON) {
*status = SAI__ERROR;
errRep(FUNC_NAME, "Repeating mirror position values found!", status);
}
}
*/
}
void * smf_map_or_malloc (size_t nelem, smf_dtype type, int zero, int indf,
const char * comp, int *status ) {
void *pntr[3]; /* ndfMap pointers */
int nout = 0; /* number of elements mapped */
if (*status != SAI__OK) return NULL;
/* just malloc if we do not have a file */
if ( indf == NDF__NOID) {
if( zero ) {
return astCalloc( nelem, smf_dtype_sz(type, status) );
} else {
return astMalloc( nelem*smf_dtype_sz(type, status) );
}
}
ndfMap( indf, comp, smf_dtype_str(type, status),
(zero ? "WRITE/ZERO" : "WRITE"), pntr, &nout, status);
if (nelem != (size_t)nout && *status == SAI__OK) {
ndfUnmap( indf, comp, status );
*status = SAI__ERROR;
msgSetc( "COMP", comp );
msgSeti( "ORI", nelem );
msgSeti( "NOUT", nout );
errRep(" ", "Mapping ^COMP in NDF but size differs from that listed in smfData attributes (^ORI != ^NOUT)", status);
pntr[0] = NULL;
}
return pntr[0];
}
void sc2sim_getcurvepong
(
double angle, /* angle of pattern relative to telescope
axes in radians anticlockwise (given) */
double width, /* minimum width of PONG pattern in arcsec (given) */
double height, /* minimum height of PONG pattern in arcsec (given) */
double spacing, /* grid spacing in arcsec (given) */
double accel[2], /* telescope accelerations (arcsec) (given) */
double vmax[2], /* telescope maximum velocities (arcsec/sec) (given) */
double samptime, /* sample interval in sec (given) */
int nmaps, /* number of cycles of the pattern */
int *pongcount, /* number of positions in pattern (returned) */
double **posptr, /* list of positions (returned) */
int *status /* global status (given and returned) */
)
{
/* Local variables */
double amp_x; /* amplitude of x(t) (arcsec) */
double amp_y; /* amplitude of y(t) (arcsec) */
double cend[2]; /* ending coordinates in arcsec */
double cstart[2]; /* starting coordinates in arcsec */
int curroff; /* index of next free slot in position list */
double grid[1024][2]; /* array of vertex coordinates */
int i; /* loop counter */
double *mapptr; /* list of positions for one map */
int mcount; /* number of positions in one map */
int numvertices; /* number of vertices (including start & end,
which are the same) */
double period; /* total time of scan (seconds) */
double peri_x; /* period of x(t) (seconds) */
double peri_y; /* period of y(t) (seconds) */
double t_count; /* time counter (seconds) */
double tx; /* temporary x value (arcsec) */
double ty; /* temporary y value (arcsec) */
double vert_spacing; /* spacing along the vertices (arcsec) */
int x_numvert; /* number of vertices along x axis */
int y_numvert; /* number of vertices along y axis */
/* Check status */
if ( !StatusOkP(status) ) return;
/* Calculate how many vertices there must be in each direction,
and how far apart they are */
sc2sim_getpongvert ( width, height, spacing, &vert_spacing,
&x_numvert, &y_numvert, status );
/* KLUDGE : Calculate the approximate periods (assuming a PONG scan with
no "rounding" at the corners. Use the code from the Straight PONG
solution to determine the periods */
sc2sim_getpongends ( width, height, spacing, grid, &numvertices, status );
/* Rotate the grid coordinates */
for ( i=0; i< numvertices; i++ ) {
tx = grid[i][0] * cos(angle) - grid[i][1] * sin(angle);
ty = grid[i][0] * sin(angle) + grid[i][1] * cos(angle);
grid[i][0] = tx;
grid[i][1] = ty;
}
mcount = 0;
for ( i=0; i<numvertices - 1; i++ ) {
cstart[0] = grid[i][0];
cstart[1] = grid[i][1];
cend[0] = grid[i+1][0];
cend[1] = grid[i+1][1];
sc2sim_getscansegsize ( samptime, cstart, cend, accel, vmax, &curroff,
status );
mcount += curroff;
}
period = mcount * samptime;
peri_x = period / y_numvert;
peri_y = period / x_numvert;
/* Determine the number of positions required for the pattern
and allocate memory */
mapptr = astCalloc( mcount*2, sizeof(*mapptr) );
/* Calculate the amplitudes of x(t) and y(t) */
amp_x = ((double)(x_numvert) * vert_spacing) / 2.0;
amp_y = ((double)(y_numvert) * vert_spacing) / 2.0;
/* Get the positions using a triangle wave approximation for both
x(t) and y(t), applying a Fourier expansion with the first five
terms (1, 3, 5, 7, 9) so as to "round off" the corners */
t_count = 0.0;
for ( i = 0; ( i < mcount * 2 ); i++ ) {
tx = ( ( 8.0 * amp_x ) / ( M_PI * M_PI ) ) *
( sin ( 2.0 * M_PI * t_count / peri_x ) -
( 1.0/9.0 * sin ( 6.0 * M_PI * t_count / peri_x ) ) +
( 1.0/25.0 * sin ( 10.0 * M_PI * t_count / peri_x ) ) -
( 1.0/49.0 * sin ( 14.0 * M_PI * t_count / peri_x ) ) +
( 1.0/81.0 * sin ( 18.0 * M_PI * t_count / peri_x ) ) );
ty = ( ( 8.0 * amp_y ) / ( M_PI * M_PI ) ) *
( sin ( 2.0 * M_PI * t_count / peri_y ) -
( 1.0/9.0 * sin ( 6.0 * M_PI * t_count / peri_y ) ) +
( 1.0/25.0 * sin ( 10.0 * M_PI * t_count / peri_y ) ) -
( 1.0/49.0 * sin ( 14.0 * M_PI * t_count / peri_y ) ) +
( 1.0/81.0 * sin ( 18.0 * M_PI * t_count / peri_y ) ) );
/* Apply the rotation angle and record the coordinates */
mapptr[i] = tx * cos ( angle ) - ty * sin ( angle );
i++;
mapptr[i] = tx * sin ( angle ) + ty * cos ( angle );
t_count += samptime;
}
/* Allocate memory for all n cycles of the pattern */
*pongcount = mcount * nmaps;
*posptr = astCalloc( (*pongcount) * 2, sizeof(**posptr) );
/* Copy the required number of cycles into the
list of positions */
for ( i = 0; i < *pongcount * 2; i++ ) {
(*posptr)[i] = mapptr[i % (mcount * 2)];
}
mapptr = astFree( mapptr );
}
void smf_mask_noisy( ThrWorkForce *wf, smfData *data, smfData **noise,
double sigcliphigh, double sigcliplow, int cliplog,
int zeropad, int * status ) {
size_t i;
dim_t nbolo = 0; /* Number of bolometers */
double *noisedata = NULL; /* Pointer to the noise data array */
smfData * noisemap = NULL; /* Somewhere to receive the result */
double *work = NULL; /* Temporary work array */
if (*status != SAI__OK) return;
if( (sigcliphigh <= 0.0) && (sigcliplow <= 0.0) ) return;
/* Work out how many bolometers we have */
smf_get_dims( data, NULL, NULL, &nbolo, NULL, NULL,
NULL, NULL, status );
/* Create some space for the result */
smf_create_bolfile( wf, NULL, 1, data, "Noise", "blahs s**0.5",
SMF__MAP_VAR, &noisemap, status );
if (noisemap) noisedata = (noisemap->pntr)[0];
/* Calculate the noise on each bolometer */
smf_bolonoise( wf, data, -1.0, 0, 0.5, SMF__F_WHITELO,
SMF__F_WHITEHI, 0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT,
(noisemap->pntr)[0], NULL, NULL, status );
/* Now need to convert this to noise by square rooting */
if (*status == SAI__OK) {
for (i=0; i<nbolo; i++) {
if (noisedata[i] != VAL__BADD) {
noisedata[i] = sqrt( noisedata[i] );
}
}
}
/* Now create a mask and then mask the quality. We only mask bolometers
that are too noisy. We also do not mask bolometers that were already
masked. We want the statistics to reflect what we actually masked
and not any previous masking. This will miss any bolometers masked
out by smf_bolonoise because they had zero power. */
msgOutif( MSG__VERB, "", FUNC_NAME
": Checking for bolometers with outlier noise values. ", status );
msgOutiff( MSG__VERB, "", FUNC_NAME
": Units are (%s s**0.5): ", status, data->hdr->units );
work = astCalloc( nbolo, sizeof(*work) );
if( *status == SAI__OK ) memcpy( work, noisedata, nbolo*sizeof(*work) );
smf_clipnoise( noisedata, nbolo, cliplog, sigcliplow, sigcliphigh, NULL,
status );
/* The only bad values that should appear in noisedata are new bolometers
that were clipped by smf_clipnoise, not bolometers that were bad before
for other reasons. */
if( *status == SAI__OK ) {
for( i=0; i<nbolo; i++ ) {
if( (noisedata[i]==VAL__BADD) && (work[i] == VAL__BADD) ) {
noisedata[i] = 0;
}
}
}
if( work ) work = astFree( work );
/* The mask has to be inside a smfArray */
if (*status == SAI__OK) {
smfArray *masks = smf_create_smfArray( status );
if (masks) masks->owndata = 0; /* someone else owns the smfData */
smf_addto_smfArray( masks, noisemap, status );
smf_apply_mask( wf, data, masks, SMF__BBM_QUAL, SMF__Q_NOISE,
status );
smf_close_related( wf, &masks, status );
}
/* Give noisemap back to caller if requested, or close it */
if( noise ) {
*noise = noisemap;
} else {
smf_close_file( wf, &noisemap, status );
}
}
int *smf_jsatiles_data( Grp *igrp, size_t size, int *ntile, int *status ){
/* Local Variables */
AstFrame *frm = NULL;
AstFrameSet *fs;
JCMTState *state;
const char *trsys;
const char *trsys_last;
dim_t *hits = NULL;
dim_t *ph;
dim_t iframe;
double *gx = NULL;
double *gy = NULL;
double *p1;
double *p2;
double *px;
double *py;
double *trac1 = NULL;
double *trac2 = NULL;
double fov;
double point1[ 2 ];
double point2[ 2 ];
double search;
int *tiles = NULL;
int dim[ 2 ];
int i;
int ix;
int iy;
int lbnd[ 2 ];
int ubnd[ 2 ];
size_t ifile;
smfData *data = NULL;
smfHead *hdr = NULL;
smfJSATiling skytiling;
smf_inst_t inst = SMF__INST_NONE;
smf_inst_t instrument;
smf_subinst_t subinst;
/* Initialise */
*ntile = 0;
/* Check inherited status */
if( *status != SAI__OK ) return tiles;
/* Start an AST context so that all AST objects created in this function
are annulled automatically. */
astBegin;
/* Loop round all the input NDFs. */
trsys_last = "";
for( ifile = 1; ifile <= size && *status == SAI__OK; ifile++ ) {
/* Obtain information about the current input NDF. */
smf_open_file( igrp, ifile, "READ", SMF__NOCREATE_DATA, &data,
status );
/* Get a pointer to the header. */
hdr = data->hdr;
/* Get the instrument. */
if( hdr->instrument == INST__SCUBA2 ) {
subinst = smf_calc_subinst( hdr, status );
if( subinst == SMF__SUBINST_850 ) {
inst = SMF__INST_SCUBA_2_850;
} else {
inst = SMF__INST_SCUBA_2_450;
}
} else if( hdr->instrument == INST__ACSIS ) {
inst = SMF__INST_HARP;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
errRep( "", "No tiles are yet defined for the instrument that "
"created ^FILE.", status );
}
/* If this is the first file, it defines the instrument in use. */
if( ifile == 1 ) {
instrument = inst;
/* Get the parameters that define the layout of sky tiles for the
instrument. */
smf_jsatiling( instrument, &skytiling, status );
/* Create a FrameSet describing the whole sky in which each pixel
corresponds to a single tile. The current Frame is ICRS (RA,Dec) and
the base Frame is grid coords in which each grid pixel corresponds to
a single tile. */
smf_jsatile( 0, &skytiling, 0, NULL, &fs, NULL, lbnd, ubnd, status );
/* Allocate an image with one pixel for each tile, and fill it with
zeros. */
dim[ 0 ] = ubnd[ 0 ] - lbnd[ 0 ] + 1;
dim[ 1 ] = ubnd[ 1 ] - lbnd[ 1 ] + 1;
hits = astCalloc( dim[0]*dim[1], sizeof( *hits ) );
/* Get the radius of the field of view in radians. */
fov = 0.5*(skytiling.fov*AST__DD2R)/3600.0;
/* If this is not the first file, report an error if the instrument has
changed... */
} else if( instrument != inst && *status == SAI__OK ) {
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
errRep( "", "The file ^FILE was created by a different instrument "
"to the previous files.", status );
}
/* Re-map the current Frame of the hits map WCS FrameSet to be the tracking
system used by the current file (if it has changed). */
trsys = sc2ast_convert_system( (hdr->allState)[0].tcs_tr_sys,
status );
if( *status == SAI__OK && strcmp( trsys, trsys_last ) ) {
astSetC( fs, "System", trsys );
trsys_last = trsys;
frm = astGetFrame( fs, AST__CURRENT );
}
/* Get the radius of the search circle. */
search = fov + sqrt( hdr->instap[ 0 ]*hdr->instap[ 0 ] +
hdr->instap[ 1 ]*hdr->instap[ 1 ] );
/* Ensure our work arrays are big enough to hold the current file. */
trac1 = astGrow( trac1, 4*hdr->nframes, sizeof( *trac1 ) );
trac2 = astGrow( trac2, 4*hdr->nframes, sizeof( *trac2 ) );
gx = astGrow( gx, 4*hdr->nframes, sizeof( *gx ) );
gy = astGrow( gy, 4*hdr->nframes, sizeof( *gy ) );
/* Check pointers can be used safely. */
if( *status == SAI__OK ) {
/* Loop round all time slices, getting the tracking coords at the corners
of a box that enclose the field of view. */
p1 = trac1;
p2 = trac2;
state = hdr->allState;
for( iframe = 0; iframe < hdr->nframes; iframe++,state++ ) {
point1[ 0 ] = state->tcs_tr_ac1;
point1[ 1 ] = state->tcs_tr_ac2;
for( i = 0; i < 4; i++ ) {
astOffset2( frm, point1, i*AST__DPIBY2, search, point2 );
*(p1++) = point2[ 0 ];
*(p2++) = point2[ 1 ];
}
}
/* Convert them to grid coords in the hits map. */
astTran2( fs, 4*hdr->nframes, trac1, trac2, 0, gx, gy );
/* Loop round them all again. Update the hits map to indicate how many
points fall in each tile. */
px = gx;
py = gy;
for( iframe = 0; iframe < 4*hdr->nframes; iframe++,px++,py++ ) {
ix = (int)( *px + 0.5 ) - 1;
iy = (int)( *py + 0.5 ) - 1;
hits[ ix + iy*dim[ 0 ] ]++;
}
}
/* Close the current input data file. */
smf_close_file( &data, status);
data = NULL;
}
/* Form a list of all the tiles that receive any data. */
ph = hits;
for( iy = 0; iy < dim[ 1 ]; iy++ ) {
for( ix = 0; ix < dim[ 0 ]; ix++,ph++ ) {
if( *ph > 0 ) {
tiles = astGrow( tiles, ++(*ntile), sizeof( *tiles ) );
if( *status == SAI__OK ) {
tiles[ *ntile - 1 ] = smf_jsatilexy2i( ix, iy, &skytiling,
status );
}
}
}
}
/* Free resources. */
hits = astFree( hits );
trac1 = astFree( trac1 );
trac2 = astFree( trac2 );
gx = astFree( gx );
gy = astFree( gy );
if( *status != SAI__OK ) {
tiles = astFree( tiles );
*ntile = 0;
}
astEnd;
return tiles;
}
void smf_calc_mapcoord( ThrWorkForce *wf, AstKeyMap *config, smfData *data,
AstFrameSet *outfset, int moving, int *lbnd_out,
int *ubnd_out, fts2Port fts_port, int flags,
int *status ) {
/* Local Variables */
AstSkyFrame *abskyfrm = NULL;/* Output SkyFrame (always absolute) */
AstMapping *bolo2map=NULL; /* Combined mapping bolo->map coordinates */
int bndndf=NDF__NOID; /* NDF identifier for map bounds */
void *data_pntr[1]; /* Array of pointers to mapped arrays in ndf */
int *data_index; /* Mapped DATA_ARRAY part of NDF */
int docalc=1; /* If set calculate the LUT */
int doextension=0; /* Try to write LUT to MAPCOORD extension */
smfFile *file=NULL; /* smfFile pointer */
AstObject *fstemp = NULL; /* AstObject version of outfset */
int ii; /* loop counter */
int indf_lat = NDF__NOID; /* Identifier for NDF to receive lat values */
int indf_lon = NDF__NOID; /* Identifier for NDF to receive lon values */
smfCalcMapcoordData *job_data=NULL; /* Array of job */
int lbnd[1]; /* Pixel bounds for 1d pointing array */
int lbnd_old[2]; /* Pixel bounds for existing LUT */
int lbnd_temp[1]; /* Bounds for bounds NDF component */
int lutndf=NDF__NOID; /* NDF identifier for coordinates */
AstMapping *map2sky_old=NULL;/* Existing mapping map->celestial coord. */
HDSLoc *mapcoordloc=NULL; /* HDS locator to the MAPCOORD extension */
int nw; /* Number of worker threads */
AstFrameSet *oldfset=NULL; /* Pointer to existing WCS info */
AstSkyFrame *oskyfrm = NULL; /* SkyFrame from the output WCS Frameset */
smfCalcMapcoordData *pdata=NULL; /* Pointer to job data */
double *lat_ptr = NULL; /* Pointer to array to receive lat values */
double *lon_ptr = NULL; /* Pointer to array to receive lon values */
int ubnd[1]; /* Pixel bounds for 1d pointing array */
int ubnd_old[2]; /* Pixel bounds for existing LUT */
int ubnd_temp[1]; /* Bounds for bounds NDF component */
int *lut = NULL; /* The lookup table */
dim_t nbolo=0; /* Number of bolometers */
dim_t ntslice=0; /* Number of time slices */
int nmap; /* Number of mapped elements */
AstMapping *sky2map=NULL; /* Mapping celestial->map coordinates */
size_t step; /* step size for dividing up work */
AstCmpMap *testcmpmap=NULL; /* Combined forward/inverse mapping */
AstMapping *testsimpmap=NULL;/* Simplified testcmpmap */
double *theta = NULL; /* Scan direction at each time slice */
int tstep; /* Time slices between full Mapping calculations */
int exportlonlat; /* Dump longitude and latitude values? */
/* Main routine */
if (*status != SAI__OK) return;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Initialize bounds to avoid compiler warnings */
lbnd_old[0] = 0;
lbnd_old[1] = 0;
ubnd_old[0] = 0;
ubnd_old[1] = 0;
/* Check for pre-existing LUT and de-allocate it. This will only waste
time if the MAPCOORD extension is found to be valid and it has
to be re-loaded from disk. */
smf_close_mapcoord( data, status );
/* Assert ICD data order */
smf_dataOrder( data, 1, status );
/* Get the data dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, NULL, NULL, status );
/* If SMF__NOCREATE_FILE is not set, and file associated with an NDF,
map a new MAPCOORD extension (or verify an existing one) */
if( !(flags & SMF__NOCREATE_FILE) && data->file ) {
doextension = 1;
} else {
doextension = 0;
docalc = 1;
}
/* Create / check for existing MAPCOORD extension */
if( doextension ) {
file = data->file;
/* Check type of file before proceeding */
if( file->isSc2store ) {
*status = SAI__ERROR;
errRep(FUNC_NAME,
"File was opened by sc2store library (raw data?)",
status);
}
if( !file->isTstream ) {
*status = SAI__ERROR;
errRep(FUNC_NAME, "File does not contain time stream data",status);
}
/* Get HDS locator to the MAPCOORD extension */
mapcoordloc = smf_get_xloc( data, "MAPCOORD", "MAP_PROJECTION", "UPDATE",
0, 0, status );
/* Obtain NDF identifier/placeholder for LUT in MAPCOORD extension*/
lbnd[0] = 0;
ubnd[0] = nbolo*ntslice-1;
lutndf = smf_get_ndfid( mapcoordloc, "LUT", "UPDATE", "UNKNOWN",
"_INTEGER", 1, lbnd, ubnd, status );
if( *status == SAI__OK ) {
/* store the NDF identifier */
file->mapcoordid = lutndf;
/* Create sky to output grid mapping using the base coordinates to
get the coordinates of the tangent point if it hasn't been done
yet. */
sky2map = astGetMapping( outfset, AST__CURRENT, AST__BASE );
}
/* Before mapping the LUT, first check for existing WCS information
and LBND/UBND for the output map. If they are already correct don't
bother re-calculating the LUT! */
if( *status == SAI__OK ) {
/* Try reading in the WCS information */
kpg1Wread( mapcoordloc, "WCS", &fstemp, status );
oldfset = (AstFrameSet*)fstemp;
if( *status == SAI__OK ) {
/* Check that the old and new mappings are the same by
checking that combining one with the inverse of the other
reduces to a UnitMap. */
map2sky_old = astGetMapping( oldfset, AST__BASE, AST__CURRENT );
testcmpmap = astCmpMap( map2sky_old, sky2map, 1, " " );
testsimpmap = astSimplify( testcmpmap );
if( astIsAUnitMap( testsimpmap ) ) {
/* The mappings are the same, now just check the pixel
bounds in the output map */
lbnd_temp[0] = 1;
ubnd_temp[0] = 2;
bndndf = smf_get_ndfid( mapcoordloc, "LBND", "READ", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp,
status );
if( *status == SAI__OK ) {
ndfMap( bndndf, "DATA", "_INTEGER", "READ", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
lbnd_old[0] = data_index[0];
lbnd_old[1] = data_index[1];
}
ndfAnnul( &bndndf, status );
}
bndndf = smf_get_ndfid( mapcoordloc, "UBND", "READ", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp,
status );
if( *status == SAI__OK ) {
ndfMap( bndndf, "DATA", "_INTEGER", "READ", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
ubnd_old[0] = data_index[0];
ubnd_old[1] = data_index[1];
}
ndfAnnul( &bndndf, status );
}
if( *status == SAI__OK ) {
/* If we get this far finally do the bounds check! */
if( (lbnd_old[0] == lbnd_out[0]) &&
(lbnd_old[1] == lbnd_out[1]) &&
(ubnd_old[0] == ubnd_out[0]) &&
(ubnd_old[1] == ubnd_out[1]) ) {
docalc = 0; /* We don't have to re-calculate the LUT */
msgOutif(MSG__VERB," ",FUNC_NAME ": Existing LUT OK",
status);
}
}
}
/* Bad status / AST errors at this point due to problems with
MAPCOORD. Annul and continue calculating new MAPCOORD extension. */
astClearStatus;
errAnnul(status);
} else {
/* Bad status due to non-existence of MAPCOORD. Annul and continue */
errAnnul(status);
}
}
}
/* If we need to calculate the LUT do it here */
if( docalc && (*status == SAI__OK) ) {
msgOutif(MSG__VERB," ", FUNC_NAME ": Calculate new LUT",
status);
/* Get the increment in time slices between full Mapping calculations.
The Mapping for intermediate time slices will be approximated. */
dim_t dimval;
smf_get_nsamp( config, "TSTEP", data, &dimval, status );
tstep = dimval;
/* Get space for the LUT */
if( doextension ) {
/* Map the LUT array */
ndfMap( lutndf, "DATA", "_INTEGER", "WRITE", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
lut = data_index;
} else {
errRep( FUNC_NAME, "Unable to map LUT in MAPCOORD extension",
status);
}
} else {
/* alloc the LUT and THETA arrays */
lut = astMalloc( (nbolo*ntslice)*sizeof(*(data->lut)) );
theta = astMalloc( ntslice*sizeof(*(data->theta)) );
}
/* Retrieve the sky2map mapping from the output frameset (actually
map2sky) */
oskyfrm = astGetFrame( outfset, AST__CURRENT );
sky2map = astGetMapping( outfset, AST__BASE, AST__CURRENT );
/* If the longitude and latitude is being dumped, create new NDFs to
hold them, and map them. */
if( config ) {
astMapGet0I( config, "EXPORTLONLAT", &exportlonlat );
if( exportlonlat ) {
lon_ptr = smf1_calc_mapcoord1( data, nbolo, ntslice, oskyfrm,
&indf_lon, 1, status );
lat_ptr = smf1_calc_mapcoord1( data, nbolo, ntslice, oskyfrm,
&indf_lat, 2, status );
}
}
/* Invert the mapping to get Output SKY to output map coordinates */
astInvert( sky2map );
/* Create a SkyFrame in absolute coordinates */
abskyfrm = astCopy( oskyfrm );
astClear( abskyfrm, "SkyRefIs" );
astClear( abskyfrm, "SkyRef(1)" );
astClear( abskyfrm, "SkyRef(2)" );
if( *status == SAI__OK ) {
/* --- Begin parellelized portion ------------------------------------ */
/* Start a new job context. Each call to thrWait within this
context will wait until all jobs created within the context have
completed. Jobs created in higher contexts are ignored by thrWait. */
thrBeginJobContext( wf, status );
/* Allocate job data for threads */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
/* Set up job data, and start calculating pointing for blocks of
time slices in different threads */
if( nw > (int) ntslice ) {
step = 1;
} else {
step = ntslice/nw;
}
for( ii=0; (*status==SAI__OK)&&(ii<nw); ii++ ) {
pdata = job_data + ii;
/* Blocks of time slices */
pdata->t1 = ii*step;
pdata->t2 = (ii+1)*step-1;
/* Ensure that the last thread picks up any left-over tslices */
if( (ii==(nw-1)) && (pdata->t1<(ntslice-1)) ) {
pdata->t2=ntslice-1;
}
pdata->ijob = -1;
pdata->lut = lut;
pdata->theta = theta;
pdata->lbnd_out = lbnd_out;
pdata->moving = moving;
pdata->ubnd_out = ubnd_out;
pdata->tstep = tstep;
pdata->lat_ptr = lat_ptr;
pdata->lon_ptr = lon_ptr;
pdata->fts_port = fts_port;
/* Make deep copies of AST objects and unlock them so that each
thread can then lock them for their own exclusive use */
pdata->abskyfrm = astCopy( abskyfrm );
astUnlock( pdata->abskyfrm, 1 );
pdata->sky2map = astCopy( sky2map );
astUnlock( pdata->sky2map, 1 );
/* Similarly, make a copy of the smfData, including only the header
information which each thread will need in order to make calls to
smf_rebin_totmap */
pdata->data = smf_deepcopy_smfData( data, 0, SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS |
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY, 0, 0,
status );
smf_lock_data( pdata->data, 0, status );
}
for( ii=0; ii<nw; ii++ ) {
/* Submit the job */
pdata = job_data + ii;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfCalcMapcoordPar, 0, NULL, status );
}
/* Wait until all of the jobs submitted within the current job
context have completed */
thrWait( wf, status );
}
/* End the current job context. */
thrEndJobContext( wf, status );
/* --- End parellelized portion -------------------------------------- */
/* Set the lut pointer in data to the buffer */
data->lut = lut;
data->theta = theta;
/* Write the WCS for the projection to the extension */
if( doextension ) {
kpg1Wwrt( (AstObject*)outfset, "WCS", mapcoordloc, status );
/* Write the pixel bounds for the map to the extension */
lbnd_temp[0] = 1; /* Don't get confused! Bounds for NDF that will */
ubnd_temp[0] = 2; /* contain the bounds for the output 2d map! */
bndndf = smf_get_ndfid( mapcoordloc, "LBND", "UPDATE", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp, status );
ndfMap( bndndf, "DATA", "_INTEGER", "WRITE", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
data_index[0] = lbnd_out[0];
data_index[1] = lbnd_out[1];
} else {
errRep( FUNC_NAME, "Unable to map LBND in MAPCOORD extension",
status);
}
ndfAnnul( &bndndf, status );
bndndf = smf_get_ndfid( mapcoordloc, "UBND", "UPDATE", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp, status );
ndfMap( bndndf, "DATA", "_INTEGER", "WRITE", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
data_index[0] = ubnd_out[0];
data_index[1] = ubnd_out[1];
} else {
errRep( FUNC_NAME, "Unable to map UBND in MAPCOORD extension",
status);
}
ndfAnnul( &bndndf, status );
}
}
}
/* Clean Up */
if( testsimpmap ) testsimpmap = astAnnul( testsimpmap );
if( testcmpmap ) testcmpmap = astAnnul( testcmpmap );
if( map2sky_old ) map2sky_old = astAnnul( map2sky_old );
if( oldfset ) oldfset = astAnnul( oldfset );
if (sky2map) sky2map = astAnnul( sky2map );
if (bolo2map) bolo2map = astAnnul( bolo2map );
if( abskyfrm ) abskyfrm = astAnnul( abskyfrm );
if( oskyfrm ) oskyfrm = astAnnul( oskyfrm );
if( mapcoordloc ) datAnnul( &mapcoordloc, status );
if( indf_lat != NDF__NOID ) ndfAnnul( &indf_lat, status );
if( indf_lon != NDF__NOID ) ndfAnnul( &indf_lon, status );
/* If we get this far, docalc=0, and status is OK, there must be
a good LUT in there already. Map it so that it is accessible to
the caller; "UPDATE" so that the caller can modify it if desired. */
if( (*status == SAI__OK) && (docalc == 0) ) {
smf_open_mapcoord( data, "UPDATE", status );
}
/* Clean up job data */
if( job_data ) {
for( ii=0; (*status==SAI__OK)&&(ii<nw); ii++ ) {
pdata = job_data + ii;
if( pdata->data ) {
smf_lock_data( pdata->data, 1, status );
smf_close_file( &(pdata->data), status );
}
astLock( pdata->abskyfrm, 0 );
pdata->abskyfrm = astAnnul( pdata->abskyfrm );
astLock( pdata->sky2map, 0 );
pdata->sky2map = astAnnul( pdata->sky2map );
}
job_data = astFree( job_data );
}
}
void smf_rebinmap1( ThrWorkForce *wf, smfData *data, smfData *variance, int *lut,
size_t tslice1, size_t tslice2, int trange,
int *whichmap, dim_t nmap, smf_qual_t mask, int sampvar,
int flags, double *map, double *mapweight,
double *mapweightsq, int *hitsmap,
double *mapvar, dim_t msize, double *scalevariance,
int *status ) {
/* Local Variables */
SmfRebinMap1Data *job_data = NULL;
SmfRebinMap1Data *pdata;
double *dat=NULL; /* Pointer to data array */
size_t dbstride; /* bolo stride of data */
size_t dtstride; /* tstride of data */
int iw; /* Thread index */
dim_t mbufsize; /* Size of full (multi-map) map buffers */
dim_t nbolo; /* number of bolos */
dim_t ntslice; /* number of time slices */
int nw; /* Number of worker threads */
size_t pixstep; /* Number of map pixels per thread */
dim_t bolostep; /* Number of bolos per thread */
smf_qual_t * qual = NULL; /* Quality pointer */
double scalevar; /* variance scale factor */
double scaleweight; /* weights for calculating scalevar */
size_t t1, t2; /* range of time slices to re-grid */
double *var=NULL; /* Pointer to variance array */
size_t vbstride; /* bolo stride of variance */
dim_t vnbolo; /* number of bolos in variance */
dim_t vntslice; /* number of bolos in variance */
size_t vtstride; /* tstride of variance */
/* Main routine */
if (*status != SAI__OK) return;
/* Check inputs */
if( !data || !map || !lut || !mapweight || !mapweightsq || !mapvar ||
!hitsmap ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": Null inputs", status );
return;
}
if( !data->pntr[0] ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": supplied data is empty", status );
return;
}
dat = data->pntr[0];
qual = smf_select_qualpntr( data, NULL, status );
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &dbstride,
&dtstride, status );
/* Size of full map buffers */
if( whichmap ) {
mbufsize = nmap * msize;
} else {
mbufsize = msize;
}
if( variance ) {
var = variance->pntr[0];
smf_get_dims( variance, NULL, NULL, &vnbolo, &vntslice, NULL, &vbstride,
&vtstride, status );
/* Check that the variance dimensions are compatible with data */
if( (*status==SAI__OK) &&
((vnbolo != nbolo) || ((vntslice>1)&&(vntslice!=ntslice))) ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": variance dimensions incompatible with data",
status );
return;
}
}
/* Range of time slices to regrid */
if( trange ) {
if( tslice2 >= ntslice ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": tslice2 (%zu) can't be >= ntslice (%zu)",
status, tslice2, ntslice );
return;
}
if( tslice1 > tslice2 ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": tslice1 (%zu) > tslice2 (%zu)",
status, tslice1, tslice2 );
return;
}
t1 = tslice1;
t2 = tslice2;
} else {
t1 = 0;
t2 = ntslice-1;
}
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* If this is the first data to be accumulated zero the arrays */
if( flags & AST__REBININIT ) {
memset( map, 0, nw*mbufsize*sizeof(*map) );
memset( mapweight, 0, nw*mbufsize*sizeof(*mapweight) );
memset( mapweightsq, 0, nw*mbufsize*sizeof(*mapweightsq) );
memset( mapvar, 0, nw*mbufsize*sizeof(*mapvar) );
memset( hitsmap, 0, nw*mbufsize*sizeof(*hitsmap) );
}
/* Find how many bolos to process in each worker thread. */
bolostep = nbolo/nw;
if ( bolostep == 0 ) bolostep = 1;
/* Allocate job data for threads, and store the range of bolos to be
processed by each one. Ensure that the last thread picks up any
left-over bolos. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->p1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->p2 = pdata->p1 + bolostep - 1;
} else {
pdata->p2 = nbolo - 1 ;
}
/* Store other values common to all jobs. */
pdata->msize = msize;
pdata->nbolo = nbolo;
pdata->t1 = t1;
pdata->t2 = t2;
pdata->vntslice = vntslice;
pdata->dat = dat;
pdata->map = map;
pdata->mapvar = mapvar;
pdata->mapweightsq = mapweightsq;
pdata->mapweight = mapweight;
pdata->var = var;
pdata->hitsmap = hitsmap;
pdata->lut = lut;
pdata->whichmap = whichmap;
pdata->dbstride = dbstride;
pdata->dtstride = dtstride;
pdata->vbstride = vbstride;
pdata->vtstride = vtstride;
pdata->mask = mask;
pdata->qual = qual;
pdata->mbufsize = mbufsize;
pdata->nw = nw; /* used for final summing/rescaling */
pdata->iw = iw; /* so the thread knows with chunk it's working on */
}
}
if( var ) {
/* Accumulate data and weights in the case that variances are given*/
if( sampvar ) {
/* Measure weighted sample variance for varmap */
if( qual ) { /* QUALITY checking version */
/* Set up jobs to add the previous estimate of COM back on to the
residuals, and then wait for the jobs to complete. These jobs also
clear any SMF__Q_COM flags set by previous iterations. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 1;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
} else { /* VAL__BADD checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 2;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
}
} else {
/* Otherwise use simple error propagation for varmap */
if( qual ) { /* QUALITY checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 3;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
} else { /* VAL__BADD checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 4;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
}
}
} else {
/* Accumulate data and weights when no variances are given. In this case
the variance map is always estimated from the sample variance */
if( qual ) { /* QUALITY checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 5;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
} else { /* VAL__BADD checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 6;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
}
}
/* If this is the last data to be accumulated re-normalize */
if( flags & AST__REBINEND ) {
/* Find how many buffer pixels to process in each worker thread. */
pixstep = mbufsize/nw;
if( pixstep == 0 ) pixstep = 1;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->p1 = iw*pixstep;
if( iw < nw - 1 ) {
pdata->p2 = pdata->p1 + pixstep - 1;
} else {
pdata->p2 = mbufsize - 1 ;
}
}
if( sampvar || !var ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 7;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
scaleweight=0;
scalevar=0;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
scaleweight += pdata->scaleweight;
scalevar += pdata->scalevar;
}
/* Re-normalize scalevar */
if( scaleweight ) {
scalevar /= scaleweight;
if( scalevariance ) {
*scalevariance = scalevar;
}
}
} else {
/* Re-normalization for error propagation case */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 8;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
}
}
job_data = astFree( job_data );
}
void smf_calc_smoothedwvm ( ThrWorkForce *wf, const smfArray * alldata,
const smfData * adata, AstKeyMap* extpars, double **wvmtau,
size_t *nelems, size_t *ngoodvals, int * status ) {
size_t i;
size_t nrelated = 0; /* Number of entries in smfArray */
size_t nframes = 0; /* Number of timeslices */
size_t ngood = 0; /* Number of elements with good tau */
double *taudata = NULL; /* Local version of WVM tau */
const smfArray * thesedata = NULL; /* Collection of smfDatas to analyse */
smfArray * tmpthesedata = NULL; /* Local version of adata in a smfArray */
if (*status != SAI__OK) return;
if (alldata && adata) {
*status = SAI__ERROR;
errRep("", "smf_calc_smoothedwvm can not be given non-NULL alldata and non-NULL adata arguments"
" (possible programming error)", status );
return;
}
if (!alldata && !adata) {
*status = SAI__ERROR;
errRep("", "smf_calc_smoothedwvm: One of alldata or adata must be non-NULL",
status);
return;
}
if (!wvmtau) {
*status = SAI__ERROR;
errRep("", "Must supply a non-NULL pointer for wvmtau argument"
" (possible programming error)", status );
return;
}
/* if we have a single smfData put it in a smfArray */
if (alldata) {
if (alldata->ndat == 0 ) {
*status = SAI__ERROR;
errRep("", "No smfDatas present in supplied smfArray for WVM smoothing"
" (possible programming error)", status );
return;
}
thesedata = alldata;
} else {
tmpthesedata = smf_create_smfArray( status );
if (tmpthesedata) {
tmpthesedata->owndata = 0; /*not owned by the smfArray */
/* we know that the smfData here will not be touched in this
function so we do the BAD thing of casting const to non-const */
smf_addto_smfArray( tmpthesedata, (smfData *)adata, status );
}
thesedata = tmpthesedata;
}
/* Check that we have headers and that the smfData are the same length */
nrelated = thesedata->ndat;
for (i = 0; i < nrelated; i++ ) {
smfData * data = (thesedata->sdata)[i];
smfHead * hdr = data->hdr;
dim_t thisframes = 0;
if ( !hdr) {
*status = SAI__ERROR;
errRepf( "", "smfData %zu has no header. Aborting WVM smoothing",
status, i );
return;
}
smf_get_dims( data, NULL, NULL, NULL, &thisframes, NULL, NULL, NULL, status );
if (!nframes) nframes = thisframes;
if (thisframes != nframes) {
*status = SAI__ERROR;
errRepf( "", "smfData %zu has different length. Aborting WVM smoothing",
status, i );
return;
}
}
/* We will need the earliest and last airmass value in order
to calculate a zenith tau */
/* As a first step, just fill the time series with calculated WVM
tau values even though we know there are about 240 fewer tau
readings in reality. This initial approach will make it easier to
use the smoothed data directly rather than having to interpolate
from the 1.2 second data back into the 200 Hz data. */
taudata = astCalloc( nframes, sizeof(*taudata) );
if (*status == SAI__OK) {
double amprev = VAL__BADD;
double steptime;
size_t maxgap;
struct timeval tv1;
struct timeval tv2;
smfCalcWvmJobData *job_data = NULL;
int nworker;
/* We need to know the steptime so we can define the max good gap
in seconds and convert it to steps*/
steptime = (thesedata->sdata)[0]->hdr->steptime;
maxgap = (size_t)( 5.0 / steptime ); /* 5 seconds is just larger than 2 WVM readings */
/* Assume all files have the same airmass information */
smf_find_airmass_interval( (thesedata->sdata)[0]->hdr, &rev, NULL, NULL, NULL, status );
smf_timerinit( &tv1, &tv2, status );
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
if (*status == SAI__OK) {
dim_t tstep;
int iworker;
smfCalcWvmJobData *pdata = NULL;
/* Get the number of time slices to process in each thread. */
if( nworker > (int) nframes ) {
tstep = 1;
} else {
tstep = nframes/nworker;
}
/* to return the same values for one thread and multiple threads
we need to break the threads on wvm sample boundaries wherever
possible. We make an initial estimate of the number of WVM measurements
by assuming one every two seconds. */
{
smfData * curdata = NULL;
size_t nwvm = 0;
double prevtime = VAL__BADD;
double curtime;
size_t *boundaries = astGrow(NULL, nframes*(size_t)(steptime/2.0), sizeof(*boundaries));
for (i=0; i<nframes; i++) {
if (!curdata) {
SELECT_DATA( thesedata, curdata, VAL__BADD, wvm_time, i );
}
if (curdata) smf_tslice_ast( curdata, i, 0, NO_FTS, status );
if ( !curdata || curdata->hdr->state->wvm_time == VAL__BADD ) {
/* Try the other datas */
SELECT_DATA( thesedata, curdata, VAL__BADD, wvm_time, i );
}
if (*status != SAI__OK) break;
if (!curdata) {
curtime = VAL__BADD;
} else {
curtime = curdata->hdr->state->wvm_time;
}
if (curtime != prevtime || nwvm == 0 ) {
/* Store the index in the boundaries array */
nwvm++;
boundaries = astGrow(boundaries, nwvm, sizeof(*boundaries));
if (!boundaries) { /* this is serious */
if (*status == SAI__OK) *status = SAI__ERROR;
errRep("", "Error allocating temporary memory for WVM calculation\n",
status );
break;
}
boundaries[nwvm-1] = i;
prevtime = curtime;
}
}
/* No point using too many threads */
if (*status == SAI__OK) {
if (nworker >= (int)nwvm) {
nworker = nwvm;
/* Allocate a measurement per thread */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->t1 = boundaries[iworker];
if (iworker+1 < nworker) pdata->t2 = boundaries[iworker+1]-1;
}
/* Ensure that the last thread picks up any left-over time slices */
pdata->t2 = nframes - 1;
} else {
/* Allocate the workers to slices of approximate size tstep */
size_t prevend = 0; /* End of previous slice */
size_t prevbnd = 0; /* Index into previous boundaries[] array selection */
for( iworker = 0; iworker < nworker; iworker++ ) {
size_t belowidx = prevend+1;
size_t aboveidx = nframes;
size_t lbnd;
size_t ubnd;
size_t j;
size_t guess;
pdata = job_data + iworker;
if (iworker == 0) { /* always start at the beginning */
pdata->t1 = 0;
} else { /* Start one after the previous block */
pdata->t1 = prevend + 1;
}
/* Now we have to find the end of this slice */
guess = (iworker*tstep) + tstep - 1;
if (guess <= pdata->t1) guess = pdata->t1 + tstep;
/* find nearest boundaries */
for (j=prevbnd; j<nwvm; j++) {
if ( boundaries[j] > guess ) {
aboveidx = boundaries[j];
ubnd = j;
if (j>0) {
belowidx = boundaries[j-1];
lbnd = j -1 ;
} else {
lbnd = 0;
}
break;
}
}
/* Choose the closest, making sure that we are not choosing t1 */
if ( (guess - belowidx < aboveidx - guess) && belowidx > pdata->t1 ) {
pdata->t2 = belowidx - 1;
prevbnd = lbnd;
} else {
pdata->t2 = aboveidx - 1;
prevbnd = ubnd;
}
prevend = pdata->t2;
if (prevend == nframes - 1 && iworker < nworker-1 ) {
/* we have run out of slices so just use fewer workers */
nworker = iworker + 1;
break;
}
}
/* Ensure that the last thread picks up any left-over time slices */
pdata->t2 = nframes - 1;
}
/* Tidy up */
boundaries = astFree( boundaries );
}
}
/* Store all the other info needed by the worker threads, and submit the
jobs to fix the steps in each bolo, and then wait for them to complete. */
for( iworker = 0; iworker < nworker; iworker++ ) {
smfArray *thrdata = NULL;
pdata = job_data + iworker;
pdata->nframes = nframes;
pdata->airmass = amprev; /* really need to get it from the start of each chunk */
pdata->taudata = taudata;
pdata->maxgap = maxgap;
/* Need to copy the smfDatas and create a new smfArray for each
thread */
thrdata = smf_create_smfArray( status );
for (i=0;i<nrelated;i++) {
smfData *tmpdata = NULL;
tmpdata = smf_deepcopy_smfData( wf, (thesedata->sdata)[i], 0, SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS |
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY, 0, 0,
status );
smf_lock_data( tmpdata, 0, status );
smf_addto_smfArray( thrdata, tmpdata, status );
}
pdata->thesedata = thrdata;
/* Need to do a deep copy of ast data and unlock them */
pdata->extpars = astCopy(extpars);
astUnlock( pdata->extpars, 1 );
/* Pass the job to the workforce for execution. */
thrAddJob( wf, THR__REPORT_JOB, pdata, smf__calc_wvm_job, 0, NULL,
status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
/* Now free the resources we allocated during job creation
and calculate the number of good values */
for( iworker = 0; iworker < nworker; iworker++ ) {
smfArray * thrdata;
pdata = job_data + iworker;
astLock( pdata->extpars, 0 );
pdata->extpars = astAnnul( pdata->extpars );
thrdata = pdata->thesedata;
for (i=0;i<thrdata->ndat;i++) {
smf_lock_data( (thrdata->sdata)[i], 1, status );
}
smf_close_related( wf, &thrdata, status );
ngood += pdata->ngood;
}
}
job_data = astFree( job_data );
msgOutiff( MSG__NORM, "", FUNC_NAME ": %f s to calculate unsmoothed WVM tau values",
status, smf_timerupdate(&tv1,&tv2,status) );
}
if (*status == SAI__OK && extpars) {
/* Read extpars to see if we need to smooth */
double smoothtime = VAL__BADD;
if (astMapGet0D( extpars, "SMOOTHWVM", &smoothtime ) ) {
if (smoothtime != VAL__BADD && smoothtime > 0.0) {
smfData * data = (thesedata->sdata)[0];
double steptime = data->hdr->steptime;
dim_t boxcar = (dim_t)( smoothtime / steptime );
msgOutiff( MSG__VERB, "",
"Smoothing WVM data with %f s tophat function",
status, smoothtime );
smf_tophat1D( taudata, nframes, boxcar, NULL, 0, 0.0, status );
/* The tophat smoothing puts a bad value at the start and end of
the time series so we replace that with the adjacent value since
the step time is much smaller than WVM readout time. If more than
one value is bad we do not try to find the good value. */
taudata[0] = taudata[1];
taudata[nframes-1] = taudata[nframes-2];
}
}
}
/* Use this to get the raw WVM output for debugging */
/*
if (*status == SAI__OK) {
smfData *data = (thesedata->sdata)[0];
smfHead *hdr = data->hdr;
printf("# IDX TAU RTS_NUM RTS_END WVM_TIME\n");
for (i=0; i<nframes;i++) {
JCMTState * state;
state = &(hdr->allState)[i];
printf("%zu %.*g %d %.*g %.*g\n", i, DBL_DIG, taudata[i], state->rts_num,
DBL_DIG, state->rts_end, DBL_DIG, state->wvm_time);
}
} */
/* Free resources */
if (tmpthesedata) smf_close_related( wf, &tmpthesedata, status );
if (*status != SAI__OK) {
if (taudata) taudata = astFree( taudata );
*nelems = 0;
*ngoodvals = 0;
} else {
*wvmtau = taudata;
*nelems = nframes;
*ngoodvals = ngood;
}
}
void smf_write_smfFilter( ThrWorkForce *wf, const smfFilter *filt, const char *filename,
const Grp * igrp, size_t grpindex, int *status ) {
double *d = NULL; /* Data array pointer */
smfData *data=NULL; /* smfData for output */
size_t i; /* Loop counter */
size_t nsamp; /* Number of samples in the filter */
if( *status != SAI__OK ) return;
if( !filt ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfFilter supplied.", status );
return;
}
/* In case we change the data type of smfFilters in the future try
to catch that here */
if( sizeof(filt->real) != smf_dtype_sz(SMF__DOUBLE,status) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": warning -- smfFilter seems to have changed type!",
status );
return;
}
/* We will pack the data into an array that has an extra dimension to
store the real/imaginary parts of the filter just like an FFT */
data = smf_create_smfData( 0, status );
if( *status == SAI__OK ) {
data->dtype = SMF__DOUBLE;
data->ndims = filt->ndims+1;
nsamp=1;
for( i=0; i<filt->ndims; i++ ) {
data->dims[i] = filt->fdims[i];
nsamp *= filt->fdims[i];
}
data->dims[data->ndims-1] = 2;
/* Copy the real and imaginary parts into our new array consecutively */
data->pntr[0] = astCalloc( nsamp*2, smf_dtype_sz(SMF__DOUBLE,status) );
if( *status == SAI__OK ) {
d = data->pntr[0];
memcpy( d, filt->real, nsamp*sizeof(d) );
if( filt->isComplex ) {
memcpy( d + nsamp, filt->imag, nsamp*sizeof(d) );
}
}
}
/* Write out the file */
smf_write_smfData( wf, data, NULL, filename, igrp, grpindex, 0, MSG__NORM,
0, NULL, NULL, status );
if( data ) smf_close_file( wf, &data, status );
}
void smf_flat_write( smf_flatmeth flatmeth, const char * flatname,
double refres, const smfData * bolval,
const smfData * powref, const smfData * bolref,
const smfData * polyfit, const Grp * prvgrp, int * status ) {
size_t colsize; /* number of columns */
double *dbuf = NULL; /* input double buffer for mean data */
double *dvar = NULL; /* input double buffer for variance of data */
char fitsrec[SC2STORE__MAXFITS*SZFITSCARD+1]; /* Store for FITS records */
int *ibuf = NULL; /* int buffer for mean data */
int indf = NDF__NOID; /* NDF identifier for output file */
size_t ncards; /* number of fits cards */
size_t numbols; /* number of bolometers */
double *outvar = NULL; /* buffer for variance of data */
int place = NDF__NOPL; /* Dummy placeholder for NDF */
size_t rowsize; /* number of rows */
JCMTState *state = NULL; /* State for this flatfield */
sc2ast_subarray_t subnum; /* subarray number */
AstFrameSet *result, *spacefset;
AstLutMap *heatmap;
AstFrame *heatfrm;
int *dksquid; /* pointer to dummy dark SQUID data */
size_t j; /* loop counter */
int jig_vert[1][2]; /* dummy jiggle vertices */
double jig_path[1][2]; /* dummy jiggle path */
size_t nframes = 0; /* Number of frames in bolval */
int npath = 0; /* size of jiggle path */
int nvert = 0; /* number of jiggle vertices */
char *xmlfile = NULL; /* dummy xmlfile name */
if (*status != SAI__OK) return;
if (!bolval->da) {
*status = SAI__ERROR;
errRep( "", "No flatfield solution provided for writing",
status );
return;
}
if (!bolval->da->heatval) {
*status = SAI__ERROR;
errRep( "", "Must provide heater values in DA struct to smf_flat_write"
" (possible programming error)", status );
return;
}
/* note that colsize is the number of rows and rowsize is the number of
columns */
colsize = (bolval->dims)[SC2STORE__ROW_INDEX];
rowsize = (bolval->dims)[SC2STORE__COL_INDEX];
numbols = colsize * rowsize;
nframes = (bolval->dims)[2];
/* Make sure we have a FLAT header that reflects this file
as the flatfield solution */
smf_fits_updateS( bolval->hdr, "FLAT", flatname, "Name of flat-field file",
status );
/* Create a FITS header for DA */
smf_fits_export2DA( bolval->hdr->fitshdr, &ncards, fitsrec, status );
/* Copy the data as integers so it can be written to data file. To
prevent overflow in the variance we store that as doubles */
ibuf = astMalloc( (numbols * nframes)*sizeof(*ibuf) );
outvar = astMalloc( (numbols * nframes)*sizeof(*outvar) );
dbuf = (bolval->pntr)[0];
dvar = (bolval->pntr)[1];
if (*status == SAI__OK) {
for (j = 0; j < (nframes * numbols); j++) {
/* These started off as integers so the mean value must fit in
an integer */
if ( dbuf[j] == VAL__BADD) {
ibuf[j] = VAL__BADI;
} else {
ibuf[j] = (int)dbuf[j];
}
/* Same data type so no need to convert bad values */
if (dvar) {
outvar[j] = dvar[j];
} else {
outvar[j] = VAL__BADD;
}
}
}
/* get subarray number */
smf_find_subarray( bolval->hdr, NULL, 0, &subnum, status );
/* Create dummy components for output file */
dksquid = astCalloc ( rowsize* nframes, sizeof(*dksquid) );
jig_vert[0][0] = 0;
jig_vert[0][1] = 0;
jig_path[0][0] = 0.0;
jig_path[0][1] = 0.0;
sc2store_setcompflag ( SC2STORE__NONE, status );
sc2store_wrtstream ( flatname, subnum, ncards,
fitsrec, colsize, rowsize, nframes,
(bolref->dims)[2], refres, 0, smf_flat_methstring( flatmeth, status ),
bolval->hdr->allState, NULL,
ibuf, dksquid, (bolref->pntr)[0], (powref->pntr)[0],
"FLATCAL", NULL, NULL, jig_vert,
nvert, jig_path, npath, xmlfile, status );
sc2store_free ( status );
/* To copy in the variance and modify fix up the WCS we need to reopen
the file */
ndfOpen( NULL, flatname, "UPDATE", "OLD", &indf, &place, status );
/* make sure that history is not written twice */
ndfHsmod( "SKIP", indf, status );
if (outvar) {
void *pntr[3];
int el;
ndfStype( "_DOUBLE", indf, "VARIANCE", status );
ndfMap( indf, "VARIANCE", "_DOUBLE", "WRITE", pntr, &el, status );
if (*status == SAI__OK) {
memcpy( pntr[0], outvar, sizeof(*outvar)*el );
}
}
/* For the WCS a time frame is less relevant than heater settings */
astBegin;
/* Create frame for focal plane coordinates */
sc2ast_createwcs( subnum, NULL, NULL, NULL, NO_FTS, &spacefset, status );
/* Copy it to make sure we do not mess with the cache */
result = astCopy( spacefset );
/* and switch to BOLO frame which is best for bolometer analysis */
{
int frnum = AST__NOFRAME;
kpg1Asffr( result, "BOLO", &frnum, status );
if (frnum != AST__NOFRAME) astSetI( result, "CURRENT", frnum );
}
/* Create a simple frame for heater settings */
heatfrm = astFrame( 1, "Domain=HEATER,Label(1)=Heater Setting" );
heatmap = astLutMap( nframes, bolval->da->heatval, 1.0, 1.0, " " );
/* Append the heater axis to the spatial frameset */
atlAddWcsAxis( result, (AstMapping *)heatmap, (AstFrame *) heatfrm,
NULL, NULL, status );
/* write it to the NDF */
ndfPtwcs( result, indf, status );
/* Write provenance information */
if (prvgrp) {
size_t size = grpGrpsz( prvgrp, status );
char prvname[ 2 * PAR__SZNAM + 1];
smf_get_taskname( NULL, prvname, status );
for (j=1; j<=size; j++) {
smf_accumulate_prov( NULL, prvgrp, j, indf, prvname, NULL, status );
}
}
/* Write the polynomial expansion into an extension */
if (polyfit) {
char fitfile[GRP__SZNAM+1];
int fndf = NDF__NOID;
place = NDF__NOPL;
one_strlcpy( fitfile, flatname, sizeof(fitfile), status );
one_strlcat( fitfile, ".MORE.SMURF.FLATFIT", sizeof(fitfile), status );
/* create the file */
smf_write_smfData( polyfit, NULL, fitfile, NULL, 0, NDF__NOID,
MSG__VERB, 0, status );
/* Same WCS as the main file */
ndfOpen( NULL, fitfile, "UPDATE", "OLD", &fndf, &place, status );
ndfPtwcs( result, fndf, status );
ndfAnnul( &fndf, status );
}
astEnd;
ndfAnnul( &indf, status);
if (ibuf) ibuf = astFree( ibuf );
if (outvar) outvar = astFree( outvar );
if (dksquid) dksquid = astFree( dksquid );
if (state) state = astFree( state );
}
void smurf_sc2threadtest( int *status ) {
/* Local Variables */
smfArray **res=NULL; /* array of smfArrays of test data */
smfData *data=NULL; /* Pointer to SCUBA2 data struct */
dim_t datalen; /* Number of data points */
smfFilter *filt=NULL; /* Frequency domain filter */
size_t i; /* Loop counter */
size_t j; /* Loop counter */
smfTimeChunkData *job_data=NULL; /* Array of pointers for job data */
size_t joblen; /* Number of chunks per job */
size_t k; /* Loop counter */
size_t nchunks; /* Number of chunks */
size_t nsub; /* Number of subarrays */
int nthread; /* Number of threads */
smfTimeChunkData *pdata=NULL; /* Pointer to data for single job */
int temp; /* Temporary integer */
size_t tsteps; /* How many time steps in chunk */
struct timeval tv1, tv2; /* Timers */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
double *dat=NULL;
dim_t nbolo;
dim_t ntslice;
dim_t ndata;
size_t bstride;
size_t tstride;
dim_t offset;
if (*status != SAI__OK) return;
/* Get input parameters */
parGdr0i( "NTHREAD", 1, 1, NUM__MAXI, 1, &nthread, status );
parGdr0i( "TSTEPS", 6000, 0, NUM__MAXI, 1, &temp, status );
tsteps = (size_t) temp;
parGdr0i( "NCHUNKS", 1, 1, NUM__MAXI, 1, &temp, status );
nchunks = (size_t) temp;
parGdr0i( "NSUB", 1, 1, 4, 1, &temp, status );
nsub = (size_t) temp;
msgSeti("N",nthread);
msgOut( "", TASK_NAME ": Running test with ^N threads", status );
/*** TIMER ***/
smf_timerinit( &tv1, &tv2, status );
/* Create some fake test data in the form of an array of smfArrays */
msgSeti("T",tsteps);
msgSeti("C",nchunks);
msgSeti("NS",nsub);
msgOut( "", TASK_NAME
": Creating ^NS subarrays of data with ^C chunks * ^T samples",
status );
res = astCalloc( nchunks, sizeof(*res) );
for( k=0; (*status==SAI__OK)&&(k<nchunks); k++ ) {
res[k] = smf_create_smfArray( status );
for( i=0; (*status==SAI__OK)&&(i<nsub); i++ ) {
/* Create individual smfDatas and add to array */
data = smf_create_smfData( SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS, status );
if( *status==SAI__OK ) {
data->dtype=SMF__DOUBLE;
data->ndims=3;
data->dims[0]=40;
data->dims[1]=32;
data->dims[2]=(dim_t) tsteps;
datalen=1;
data->isFFT=-1;
for( j=0; j<data->ndims; j++ ) datalen *= data->dims[j];
data->hdr->steptime = 0.005;
data->pntr[0] = astCalloc( datalen, smf_dtype_sz(data->dtype,status) );
data->qual = astCalloc( datalen, sizeof(*data->qual) );
}
smf_addto_smfArray( res[k], data, status );
}
}
/*** TIMER ***/
msgOutf( "", "** %f seconds generating data", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut( "", TASK_NAME
": Starting test 1 __parallel time: dataOrder__", status );
/* Create a pool of threads. */
wf = thrGetWorkforce( nthread, status );
/* Work out number of chunks per thread */
joblen = nchunks/nthread;
if( joblen == 0 ) joblen = 1; /* At least one chunk per thread */
/* The first test will process separate time chunks of data in
parallel, re-ordering each to bolo-ordered format. All subarrays
and an integer number of input file chunks all go into a single
thread. Start by allocating and initializing a number of
smfTimeChunkData's that hold the information required for each
thread */
job_data = astCalloc( nthread, sizeof(*job_data) );
for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) {
pdata = job_data + i;
pdata->type = 0; /* Start with a data re-order */
pdata->data = res; /* Pointer to main data array */
pdata->chunk1 = i*joblen; /* Index of first chunk for job */
pdata->nchunks = nchunks; /* Total number of time chunks in data */
pdata->ijob = -1; /* Flag job as available to do work */
/* The last thread has to pick up the remainder of chunks */
if( i==(size_t)(nthread-1) ) pdata->chunk2=nchunks-1;
else pdata->chunk2 = (i+1)*joblen-1; /* Index of last chunk for job */
/* Ensure a valid chunk range, or set to a length that we know to ignore */
if( pdata->chunk1 >= nchunks ) {
pdata->chunk1 = nchunks;
pdata->chunk2 = nchunks;
} else if( pdata->chunk2 >= nchunks ) {
pdata->chunk2 = nchunks-1;
}
if( pdata->chunk1 >= nchunks ) {
/* Nothing for this thread to do */
msgSeti( "W", i+1);
msgOutif( MSG__DEBUG, "",
"-- parallel time: skipping thread ^W, nothing to do",
status);
} else {
/* Since we know there is one job_data per thread, just submit jobs
immediately */
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Annul the bad status that we set in smfParallelTime */
if( *status == SMF__INSMP ) {
errAnnul( status );
msgOut( "", " *** Annulled SMF__INSMP set in smfParallelTime *** ",
status );
} else {
msgOut( "", " *** Flushing good status *** ", status );
errFlush( status );
}
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
/* The second test will boxcar smooth bolometers from time chunks in
parallel */
msgOut( "", TASK_NAME
": Starting test 2 __parallel time: boxcar smooth__", status );
for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) {
pdata = job_data + i;
pdata->type = 1; /* Boxcar smooth */
if( pdata->chunk1 >= nchunks ) {
/* Nothing for this thread to do */
msgSeti( "W", i+1);
msgOutif( MSG__DEBUG, "",
"-- parallel time: skipping thread ^W, nothing to do",
status);
} else {
/* Since we know there is one job_data per thread, just submit jobs
immediately */
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut( "", TASK_NAME
": *** Next 2 tests will be done twice due to FFTW planning *****",
status );
for( k=0; k<2; k++ ) {
/* The third test will FFT filter bolometers from time chunks in
parallel */
msgOut( "", TASK_NAME
": Starting test 3 __parallel time: FFT filter__", status );
for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) {
pdata = job_data + i;
pdata->type = 2; /* FFT filter */
if( pdata->chunk1 >= nchunks ) {
/* Nothing for this thread to do */
msgSeti( "W", i+1);
msgOutif( MSG__DEBUG, "",
"-- parallel time: skipping thread ^W, nothing to do",
status);
} else {
/* Since we know there is one job_data per thread, just submit jobs
immediately */
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut( "", TASK_NAME
": Starting test 4 __FFTW filter using internal threading__",
status );
for( i=0; (*status==SAI__OK)&&(i<nchunks); i++ ) {
filt = smf_create_smfFilter( res[i]->sdata[0], status );
smf_filter_ident( filt, 1, status );
for( j=0; (*status==SAI__OK)&&(j<nsub); j++ ) {
msgOutiff( MSG__DEBUG, "", " filter chunk %zu/%zu, bolo %zu/%zu",
status, i+1, nchunks, j+1, nsub );
smf_filter_execute( wf, res[i]->sdata[j], filt, 0, 0, status );
}
if( filt ) filt = smf_free_smfFilter( filt, status );
}
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
}
msgOut( "", TASK_NAME
": **************************************************************",
status );
/* Series of short single-thread array index tests */
data = res[0]->sdata[0];
dat = data->pntr[0];
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride,
&tstride, status );
msgOut("","Array index test #1: two multiplies in inner loop",status);
smf_timerinit( &tv1, &tv2, status );
for( i=0; i<nbolo; i++ ) {
for( j=0; j<ntslice; j++ ) {
dat[i*bstride + j*tstride] += 5;
}
}
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut("","Array index test #2: only index increments",status);
smf_timerinit( &tv1, &tv2, status );
for( i=0; i<nbolo*bstride; i+=bstride ) {
for( j=i; j<(i+ntslice*tstride); j+=tstride ) {
dat[j] += 5;
}
}
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut("","Array index test #3: one multiply in outer loop",status);
smf_timerinit( &tv1, &tv2, status );
offset = 0;
for( i=0; i<nbolo; i++ ) {
offset = i*bstride;
for( j=0; j<ntslice; j++ ) {
dat[offset] += 5;
offset += tstride;
}
}
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
/* Clean up */
if( res ) {
for( i=0; i<nchunks; i++ ) {
if( res[i] ) {
smf_close_related( &res[i], status );
}
}
res = astFree( res );
}
job_data = astFree( job_data );
/* Ensure that FFTW doesn't have any used memory kicking around */
fftw_cleanup();
}
void smf_clean_dksquid( smfData *indata, smf_qual_t mask, size_t window, smfData *model,
int calcdk, int nofit, int replacebad, int *status ) {
dim_t b; /* Bolometer index */
size_t bstride; /* Bolometer index stride */
double corr; /* Linear correlation coefficient */
double *corrbuf=NULL; /* Array of correlation coeffs all bolos this col */
int needDA=0; /* Do we need dksquids from the DA? */
int *dkgood=NULL; /* Flag for non-constant dark squid */
double *dksquid=NULL; /* Buffer for smoothed dark squid */
double *dkav=NULL; /* Buffer for average dark squid */
double firstdk; /* First value in dksquid signal */
double gain; /* Gain parameter from template fit */
double *gainbuf=NULL; /* Array of gains for all bolos in this col */
size_t i; /* Loop counter */
size_t jt1;
size_t jt2;
size_t jf1; /* Starting tslice that should be fit */
size_t jf2; /* Final tslice that should be fit */
size_t j; /* Loop counter */
size_t k; /* Loop counter */
size_t nbad=0; /* Number of new bad bolos due to bad dark squid */
dim_t nbolo; /* Number of bolometers */
dim_t ncol; /* Number of columns */
dim_t ndata; /* Number of data points */
size_t nfit; /* number of samples over good range to fit */
size_t ngood=0; /* number of good dark squids */
dim_t nrow; /* Number of rows */
size_t ntot;
dim_t ntslice; /* Number of time slices */
double offset; /* Offset parameter from template fit */
double *offsetbuf=NULL; /* Array of offsets for all bolos in this col */
int pass; /* two passes over data to get estimate of average */
smf_qual_t *qua=NULL;/* Pointer to quality array */
size_t tstride; /* Time slice index stride */
if (*status != SAI__OK) return;
/* Check for NULL smfData pointer */
if( !indata ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, smfData pointer is NULL", status );
return;
}
/* Decide if we need the DA extension or not */
if( (!model) || (model && calcdk) ) {
needDA = 1;
if( !indata->da) {
/* Check for NULL smfDA */
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, no smfDA struct in smfData",
status);
return;
} else if( !indata->da->dksquid) {
/* Check for NULL dksquid */
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, no dksquid array in smfData",
status);
return;
}
/* Assert the correct data order here */
smf_dataOrder( indata->da->dksquid, 1, status );
}
/* Check for 3-d data and get dimensions */
smf_get_dims( indata, &nrow, &ncol, &nbolo, &ntslice, &ndata, &bstride,
&tstride, status );
/* Identify the range of data that should be fit using SMF__Q_BOUND */
if( qua ) {
smf_get_goodrange( qua, ntslice, tstride, SMF__Q_BOUND, &jf1, &jf2,
status );
} else {
jf1 = 0;
jf2 = ntslice-1;
}
nfit = jf2-jf1+1;
/* Total total range only using SMF__Q_PAD */
if( qua ) {
smf_get_goodrange( qua, ntslice, tstride, SMF__Q_BOUND, &jt1, &jt2,
status );
} else {
jt1 = 0;
jt2 = ntslice-1;
}
ntot = jt2-jt1+1;
if( model ) {
/* Check for valid model dimensions if supplied */
if( model->dtype != SMF__DOUBLE ) {
msgSetc("DT", smf_dtype_str(model->dtype, status) );
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": Data type ^DT for model not supported.",
status );
return;
}
if( (model->ndims != 2) ||
(model->dims[0] != ntslice+nrow*3) ||
(model->dims[1] != ncol) ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": model has incorrect dimensions", status );
return;
}
} else {
/* Otherwise allocate space for local dksquid buffer */
dksquid = astCalloc( ntslice, sizeof(*dksquid) );
}
/* Pointer to quality */
qua = smf_select_qualpntr( indata, 0, status );
/* Two passes: in the first we calculate an average dark squid to use as
a surrogate for columns with dead dark squids. In the second we do the
actual cleaning etc. */
dkgood = astCalloc( ncol, sizeof(*dkgood) );
dkav = astCalloc( ntslice, sizeof(*dkav) );
for( pass=0; (*status==SAI__OK)&&(pass<2); pass++ ) {
/* Loop over columns */
for( i=0; (*status==SAI__OK)&&(i<ncol); i++ ) {
/* Point dksquid, gainbuf, offsetbuf and corrbuf to the right
place in model if supplied. */
if( model ) {
dksquid = model->pntr[0];
dksquid += i*(ntslice+nrow*3);
gainbuf = dksquid + ntslice;
offsetbuf = gainbuf + nrow;
corrbuf = offsetbuf + nrow;
}
/* First pass is just to copy the dark squid over to the model and replace
dead dark squids with the average */
if( pass==0 ) {
/* Copy dark squids from the DA extension into dksquid */
if( needDA && calcdk && model ) {
double *ptr = indata->da->dksquid->pntr[0];
for( j=0; j<ntslice; j++ ) {
dksquid[j] = ptr[i+ncol*j];
}
}
/* Check for a good dark squid by seeing if it ever changes */
firstdk = VAL__BADD;
for( j=jt1; j<=jt2; j++ ) {
if( dksquid[j] != VAL__BADD ) {
if( firstdk == VAL__BADD ) {
firstdk = dksquid[j];
} else if( dksquid[j] != firstdk ) {
dkgood[i] = 1;
ngood++;
/* Add good squid to average dksquid */
for( k=jt1; k<=jt2; k++ ) {
dkav[k] += dksquid[k];
}
break;
}
}
}
}
/* Second pass actually do the fitting / replace with average dksquid if
dksquid was dead */
if( pass==1 ) {
/* Do some dksquid initialization if requested */
if( (*status==SAI__OK) && needDA && calcdk && model && dkgood[i] ) {
/* Smooth the dark squid template */
smf_boxcar1D( &dksquid[jt1], ntot, window, NULL, 0, status );
}
/* Initialize fit coeffs to VAL__BADD */
if( (*status == SAI__OK) && model ) {
for( j=0; j<nrow; j++ ) {
gainbuf[j] = VAL__BADD;
offsetbuf[j] = VAL__BADD;
corrbuf[j] = VAL__BADD;
}
}
/* Loop over rows, removing the fitted dksquid template. */
for( j=0; (!nofit) && (*status==SAI__OK) && (j<nrow); j++ ) {
/* Calculate bolometer index from row/col counters */
if( SC2STORE__COL_INDEX ) {
b = i*nrow + j;
} else {
b = i + j*ncol;
}
/* If dark squid is bad, flag entire bolo as bad if it isn't already */
if( !dkgood[i] && qua && !(qua[b*bstride]&SMF__Q_BADB) ) {
nbad++;
for( k=0; k<ntslice; k++ ) {
qua[b*bstride+k*tstride] |= SMF__Q_BADB;
}
}
/* Try to fit if we think we have a good dark squid and bolo, and only
the goodrange of data (excluding padding etc.) */
if((!qua && dkgood[i]) ||
(qua && dkgood[i] && !(qua[b*bstride]&SMF__Q_BADB))) {
double *d_d;
int *d_i;
switch( indata->dtype ) {
case SMF__DOUBLE:
d_d = (double *) indata->pntr[0];
smf_templateFit1D( &d_d[b*bstride+jf1*tstride],
&qua[b*bstride+jf1*tstride], NULL, NULL,
mask, mask, nfit, tstride, &dksquid[jf1], 1,
1, &gain, &offset, &corr, status );
break;
case SMF__INTEGER:
d_i = (int *) indata->pntr[0];
smf_templateFit1I( &d_i[b*bstride+jf1*tstride],
&qua[b*bstride+jf1*tstride], NULL, NULL, mask,
mask, nfit, tstride, &dksquid[jf1], 1, 1,
&gain, &offset, &corr, status );
break;
default:
msgSetc( "DT", smf_dtype_string( indata, status ));
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": Unsupported data type for dksquid cleaning (^DT)",
status );
}
if( *status == SMF__INSMP || *status == SMF__DIVBZ ) {
int wasinsmp = (*status == SMF__INSMP ? 1 : 0 );
/* Annul SMF__INSMP as it was probably due to a bad bolometer */
errAnnul( status );
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": ROW,COL (%zu,%zu) %s", status, j, i,
(wasinsmp ? "insufficient good samples" : "division by zero" ));
/* Flag entire bolo as bad if it isn't already */
if( qua && !(qua[b*bstride]&SMF__Q_BADB) ) {
for( k=0; k<ntslice; k++ ) {
qua[b*bstride+k*tstride] |= SMF__Q_BADB;
}
}
} else {
/* Store gain and offset in model */
if( model ) {
gainbuf[j] = gain;
offsetbuf[j] = offset;
corrbuf[j] = corr;
}
if( msgFlevok( MSG__DEBUG1, status ) ) {
msgSeti( "COL", i );
msgSeti( "ROW", j );
msgSetd( "GAI", gain );
msgSetd( "OFF", offset );
msgSetd( "CORR", corr );
msgOutif( MSG__DEBUG1, "", FUNC_NAME
": ROW,COL (^ROW,^COL) GAIN,OFFSET,CORR "
"(^GAI,^OFF,^CORR)", status );
}
}
}
}
}
}
/* Re-normalize the average dark-squid here at the end of pass 0 */
if( (pass==0) && (ngood) && (*status==SAI__OK) ) {
for( j=jt1; j<=jt2; j++ ) {
dkav[j] /= ngood;
}
}
/* Replace bad bolos in model with the average? */
if( replacebad && calcdk && needDA && model && (*status==SAI__OK) ) {
for( i=0; i<ncol; i++ ) {
dksquid = model->pntr[0];
dksquid += i*(ntslice+nrow*3);
if( !dkgood[i] ) {
memcpy( dksquid, dkav, sizeof(*dksquid)*ntslice );
dkgood[i] = 1;
}
}
}
}
/* Report number of new bad bolos that were flagged */
if( !replacebad && nbad ) {
msgOutiff( MSG__VERB, "", FUNC_NAME
": %zu new bolos flagged bad due to dead DKS", status, nbad );
}
/* Free dksquid only if it was a local buffer */
if( !model && dksquid ) dksquid = astFree( dksquid );
dkgood = astFree( dkgood );
dkav = astFree( dkav );
}
smf_qual_t * smf_qual_unmap( ThrWorkForce *wf, int indf, smf_qfam_t family,
smf_qual_t * qual, smf_qual_t mask, int * status ) {
int canwrite = 0; /* can we write to the file? */
size_t nqbits = 0; /* Number of quality bits in this family */
SmfQualUnmapData *job_data = NULL;
SmfQualUnmapData *pdata;
int nw;
size_t step;
int iw;
if (*status != SAI__OK) goto CLEANUP;
/* do nothing if there is no quality */
if (!qual) return NULL;
/* if we do not have an NDF identifier we just free the memory */
if (indf == NDF__NOID) goto CLEANUP;
/* See if we have WRITE access to the file */
ndfIsacc( indf, "WRITE", &canwrite, status );
/* if we have WRITE access and the data were not mapped we have
to copy to the file. Also check we have a non-NULL input pointer.
If the data were mapped we still have to make sure the quality names
are stored. */
if ( canwrite && qual ) {
int highbit = -1; /* highest bit used */
size_t i;
int itemp;
int lowbit = -1; /* Lowest bit used */
size_t nout;
int nqual = 0;
void *qpntr[1];
size_t qcount[SMF__NQBITS]; /* statically allocate the largest array */
IRQLocs *qlocs;
unsigned char * qmap;
int there;
ndfMsg( "FILE", indf );
msgOutif( MSG__DEBUG, "", "Finalising quality for file ^FILE", status);
if (family == SMF__QFAM_TCOMP ) {
/* note that TCOMP is not an allowed quality because SMURF should not be
using it anywhere in a permanent way. */
*status = SAI__ERROR;
ndfMsg( "NDF", indf );
errRepf( "", "Unsupported quality family '%s' for quality unmapping of "
"file ^NDF", status, smf_qfamily_str(family,status) );
goto CLEANUP;
} else if (family == SMF__QFAM_NULL) {
/* In this case we have to assume that we just cast the quality
to UBYTE and copy it without changing anything or naming the
entries. Use a simple type conversion. */
ndfMap( indf, "QUALITY", "_UBYTE", "WRITE", &qpntr[0], &itemp, status );
qmap = qpntr[0];
nout = itemp;
for (i = 0; i<nout; i++) {
qmap[i] = qual[i];
}
ndfUnmap( indf, "QUALITY", status );
/* Turn on all quality */
ndfSbb( 255, indf, status );
/* we are finished so jump to tidy up */
goto CLEANUP;
}
/* work out how many quality items are in this family */
nqbits = smf_qfamily_count( family, status );
/* initialize qcount */
for (i=0; i<SMF__NQBITS; i++) {
qcount[i] = 0;
}
/* how many pixels in NDF (assumed to be number in quality) */
ndfSize( indf, &itemp, status );
nout = itemp;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many elements to process in each worker thread. */
step = nout/nw;
if( step == 0 ) step = 1;
/* Allocate job data for threads, and store common values. Ensure that the
last thread picks up any left-over elements. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->i1 = iw*step;
if( iw < nw - 1 ) {
pdata->i2 = pdata->i1 + step - 1;
} else {
pdata->i2 = nout - 1 ;
}
pdata->nqbits = nqbits;
pdata->qual = qual;
pdata->nout = nout;
}
}
/* Work out which bits are actually used */
if (*status == SAI__OK) {
size_t k;
/* now we try to be a bit clever. It may be a mistake since we have to
do multiple passes through "qual". First determine how many quality
bits are actually set. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 1;
thrAddJob( wf, 0, pdata, smf1_qual_unmap, 0, NULL, status );
}
thrWait( wf, status );
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
for( k=0; k<nqbits; k++ ) {
qcount[k] += pdata->qcount[k];
}
}
/* Reset the counts to zero for any bits that are not required
(i.e. are not set in "mask"). */
for( k=0; k<nqbits; k++ ) {
if( ! (mask & (1<<k)) ) qcount[k] = 0;
}
/* see how many we got */
for (k=0; k<nqbits; k++) {
if ( qcount[k] ) {
nqual++;
highbit = k;
if (lowbit < 0) lowbit = k;
}
}
}
/* for IRQ we need to ensure the SMURF extension exists so open and annul it if it is missing.
We are completely rewriting any IRQ information so we have to delete any previously existing
IRQ extension. */
irqDelet( indf, status );
ndfXstat( indf, SMURF__EXTNAME, &there, status );
if (!there) {
HDSLoc * smurfloc = NULL;
/* Create SMURF extension if it does not already exist */
ndfXnew( indf, SMURF__EXTNAME, SMURF__EXTTYPE, 0, NULL, &smurfloc, status );
if (smurfloc) datAnnul( &smurfloc, status );
}
irqNew( indf, SMURF__EXTNAME, &qlocs, status );
/* malloced so we need to map and copy over the values. IRQ
names need to be set BEFORE we copy. */
/* Map the quality component with WRITE access */
ndfMap( indf, "QUALITY", "_UBYTE", "WRITE", &qpntr[0], &itemp, status );
qmap = qpntr[0];
/* we assume the number of elements in "qual" is the same as in "qmap" */
if (*status == SAI__OK) {
size_t k;
/* if we only have 8 or fewer bits active we can just compress
by mapping them to the lower 8 bits. This will work if we also
set the IRQ quality names in the NDF. */
if (nqual == 0 ) {
/* easy */
memset( qmap, 0, nout * smf_dtype_sz( SMF__UBYTE, status ) );
} else if ( nqual <= 8 ) {
size_t curbit = 0;
/* and the quality names. Start at lowbit and go to highbit
knowing that we have shifted them down so that lowbit in qual
is bit 0 in NDF. */
for (k=lowbit; k<=(size_t)highbit; k++) {
if (qcount[k]) {
int fixed = 0; /* is bit fixed? */
const char * qdesc = NULL; /* Description of quality */
const char * qstr = NULL; /* Quality string identifier */
curbit++;
qstr = smf_qual_str( family, 1, k, &qdesc, status );
irqAddqn( qlocs, qstr, 0, qdesc, status );
irqFxbit( qlocs, qstr, curbit, &fixed, status );
}
}
/* shift them down */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 2;
pdata->qmap = qmap;
pdata->highbit = highbit;
pdata->lowbit = lowbit;
for( k=0; k<nqbits; k++ ) {
pdata->qcount[k] = qcount[k];
}
thrAddJob( wf, 0, pdata, smf1_qual_unmap, 0, NULL, status );
}
thrWait( wf, status );
} else {
size_t curbit = 0;
/* Quality names are now needed and we have to write them
all out because we have not compressed the bits in the
output quality array we've only compressed the input.
To limit the number of active bits we'd have to copy the
compressed bits to the output and then set the quality
names but IRQ does not let you do that so you would need
to run through the entire array first counting which bits
were used. */
for (k=0; k<SMF__NQBITS_TCOMP; k++) {
int fixed = 0;
const char * qdesc = NULL; /* Description of quality */
const char * qstr = NULL; /* Quality string identifier */
qstr = smf_qual_str( SMF__QFAM_TCOMP, 1, k, &qdesc, status );
/* Set the quality name */
irqAddqn( qlocs, qstr, 0, qdesc, status );
curbit++;
irqFxbit( qlocs, qstr, curbit, &fixed, status );
}
/* compress them */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 3;
pdata->qmap = qmap;
thrAddJob( wf, 0, pdata, smf1_qual_unmap, 0, NULL, status );
}
thrWait( wf, status );
}
}
/* Unmap quality */
ndfUnmap( indf, "QUALITY", status );
/* Set the badbits mask to enable all quality by default.
Do not do this for MAP quality at the moment. */
if (family != SMF__QFAM_MAP) ndfSbb( 255, indf, status );
/* release IRQ resources */
irqRlse( &qlocs, status );
}
CLEANUP:
/* Tidy up */
qual = astFree( qual );
job_data = astFree( job_data );
return NULL;
}
void fts2_validatemirrorpositions(double* positions, int count, int* ni, int* nf, smfData* inData, int* status)
{
if(*status != SAI__OK) { return; }
// Compute EPSILON as a fraction of the expected mirror position step size (s)
// calculated from the SCANVEL (v) and the STEPTIME (t), where:
// s = vt
// and EPSILON should be reasonably large to ignore jitter, but small enough not to miss valid movement,
// let's say at least half way to the next expected mirror position, or:
// EPSILON = s/2
double s = 0.0;
double t = 0.0;
smf_fits_getD(inData->hdr, "STEPTIME", &t, status);
double v = 0.0;
smf_fits_getD(inData->hdr, "SCANVEL", &v, status);
s = v*t;
double EPSILON = s/2;
int i,j = 0;
double positive = 0.0;
double negative = 0.0;
double direction = 0.0;
// Determine scan direction
/* Mirror scans are supposed to be unidirectional (monotonically increasing or decreasing)
but can have slow starts or trailing ends where there is little to no significant change.
Determine the majority direction of this scan: positive or negative.
*/
for(i=0; i<count-1; i++) {
if(positions[i] < positions[i+1])
positive += (positions[i+1] - positions[i]);
else if(positions[i] > positions[i+1])
negative += (positions[i] - positions[i+1]);
}
direction = positive - negative;
// Invert negative scan
double* inverted = NULL;
if(direction < 0) {
inverted = (double*) astCalloc(count, sizeof(double));
for(i=0,j=count-1; i < count; i++,j--) {
inverted[i] = positions[j];
}
// Copy inverted values back to positions
for(i=0; i < count; i++) {
positions[i] = inverted[i];
}
}
/*
// CREATE SHIFTED MIRROR POSITIONS
double* shifted = (double*) astCalloc(count, sizeof(double));
for(i = 1; i < count; i++) {
shifted[i] = positions[i - 1];
}
shifted[0] = positions[count - 1];
*/
// COMPUTE DELTA MIRROR POSITIONS
double* delta = (double*) astCalloc(count, sizeof(double));
for(i = 0; i < count-1; i++) {
delta[i] = positions[i+1] - positions[i];
}
// FIND THE START INDEX
for(i = 0; i < count-1; i++) {
if(delta[i] >= EPSILON) {
*ni = i;
break;
}
}
// FIND THE END INDEX
for(i = count - 1; i > -1; i--) {
if(delta[i] >= EPSILON) {
*nf = i+1;
break;
}
}
/*
// CHECK TO SEE IF THE POSITIONS HAVE REPEATING VALUES IN BETWEEN
for(i = *ni + 1; i < *nf; i++) {
if(abs(delta[i]) <= EPSILON) {
*status = SAI__ERROR;
errRep(FUNC_NAME, "Repeating mirror position values found!", status);
}
}
*/
// CLEANUP:
if(delta) {astFree(delta); delta = NULL;}
//if(shifted) {astFree(shifted); shifted = NULL;}
if(direction < 0 && inverted) {astFree(inverted); inverted = NULL;}
}
smfData *smf_fft_avpspec( const smfData *pspec, smf_qual_t *quality,
size_t qstride, smf_qual_t mask, double *weights,
int *status ) {
dim_t fdims[2]; /* Lengths of frequency-space axes */
size_t i; /* Loop counter */
double *idptr=NULL; /* Pointer to input data */
double mean; /* Mean value at time slice */
dim_t nbolo=0; /* Number of detectors */
size_t ngood; /* Number of good samples */
size_t ndata; /* Total number of data points */
dim_t ntslice=0; /* Number of time slices */
dim_t nf=0; /* Number of frequencies in FFT */
double *odptr=NULL; /* Pointer to output data */
double *ovptr=NULL; /* Pointer to output variance */
dim_t rdims[2]; /* Lengths of real-space axes */
smfData *retdata=NULL; /* Returned power spectrum */
double sigma; /* RMS value at time slice */
if (*status != SAI__OK) return NULL;
/* Check for NULL pointer */
if( pspec == NULL ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData pointer is NULL", status );
return NULL;
}
/* Check that we have frequency-domain input */
if( !smf_isfft( pspec, rdims, &nbolo, fdims, NULL, NULL, status ) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Input data are not FFT!", status );
return NULL;
}
ntslice = rdims[0];
nf = fdims[0];
/* Check that we don't have a 1-d input power spectrum */
if( pspec->ndims != 4 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Input data are not 4-dimensional!", status );
return NULL;
}
idptr = pspec->pntr[0];
if( !idptr ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null input data", status );
return NULL;
}
/* Assume qstride = bolo stride of main array if not specified */
if( !qstride ) {
qstride = nf;
}
/* Create a new smfData, copying over everything except for the bolo
data itself */
retdata = smf_deepcopy_smfData( pspec, 0, SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY |
SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS, 0, 0, status );
if( *status == SAI__OK ) {
/* Allocate space for the averaged power spectrum */
retdata->ndims = 4;
retdata->dims[0] = nf;
retdata->dims[1] = 1;
retdata->dims[2] = 1;
retdata->dims[3] = 2;
retdata->dtype = SMF__DOUBLE;
ndata=1;
for( i=0; i<retdata->ndims; i++ ) {
ndata *= retdata->dims[i];
}
/* Allocate space for DATA and VARIANCE components */
retdata->pntr[0] = astCalloc( ndata, smf_dtype_sz(retdata->dtype,status) );
retdata->pntr[1] = astCalloc( ndata, smf_dtype_sz(retdata->dtype,status) );
/* Pointers to output data */
odptr = retdata->pntr[0];
ovptr = retdata->pntr[1];
/* Call the returned data tordered even though it is irrelevant */
retdata->isTordered=0;
/* Since we assumed we're in power/polar form just need to calculate
average and scatter of the amplitude coefficients. The phase
coefficients are left at their initialized values of 0. */
for( i=0; i<nf; i++ ) {
/* The bolometer stride is nf */
smf_weightstats1D( idptr+i, nf, nbolo, quality, qstride, mask,
weights, 1, &mean, &sigma, &ngood, status );
if( *status == SMF__INSMP ) {
/* If not enough samples just annul and set bad value */
errAnnul( status );
odptr[i] = VAL__BADD;
ovptr[i] = VAL__BADD;
} else {
odptr[i] = mean;
ovptr[i] = sigma*sigma;
}
}
}
return retdata;
}
void smurf_fts2_split(int* status)
{
if( *status != SAI__OK ) { return; }
const double STAGE_LENGTH = 450.0; /* mm */
int LR = 0; /* Treat as Low Resolution scan */
Grp* gIn = NULL; /* Input group */
Grp* gOut = NULL; /* Output group */
Grp* gTmp = NULL; /* Temporary group */
smfData* inData = NULL; /* Pointer to input data */
smfData* outData = NULL; /* Pointer to output data */
double* outData_pntr = NULL; /* Pointer to output data values array */
int nMirPos = 0; /* Number of frames where the mirror actually moves */
int nStart = 0; /* Frame index where the mirror starts moving */
int nStartNext = 0; /* Frame index where the mirror starts moving in the next scan */
int nStop = 0; /* Frame index where the mirror stops */
int lrStart = 0; /* Frame index where low resolution mirror limit starts */
int hrStop = 0; /* Frame index where high resolution mirror limit stops */
int hrStart = 0; /* Frame index where high resolution mirror limit starts */
int lrStop = 0; /* Frame index where low resolution mirror limit stops */
int lrCentre = 0; /* Frame index at centre of low resolution mirror positions */
int i = 0; /* Counter */
int j = 0; /* Counter */
int k = 0; /* Counter */
int n = 0; /* Counter */
double fNyquist = 0.0; /* Nyquist frequency */
double dz = 0.0; /* Step size in evenly spaced OPD grid */
double* MIRPOS = NULL; /* Mirror positions */
size_t nFiles = 0; /* Size of the input group */
size_t nOutFiles = 0; /* Size of the output group */
size_t fIndex = 0; /* File index */
size_t nWidth = 0; /* Data cube width */
size_t nHeight = 0; /* Data cube height */
size_t nFrames = 0; /* Data cube depth in input file */
size_t nFramesOut = 0; /* Data cube depth in output file */
size_t nFramesOutPrev = 0; /* Data cube depth in previous output file */
size_t hrFramesOut = 0; /* Data cube depth in high res output file */
size_t hrFramesOutPrev = 0; /* Data cube depth in previous high res output file */
size_t lrFramesOut = 0; /* Data cube depth in low res output file */
size_t lrFramesOutPrev = 0; /* Data cube depth in previous low res output file */
size_t nPixels = 0; /* Number of bolometers in the subarray */
char object[SZFITSTR];
char subarray[SZFITSTR];
char obsID[SZFITSTR];
char scanMode[SZFITSTR];
double scanVel = 0.0; /* Mirror speed in mm/sec */
double stepTime = 0.0; /* RTS step time, average sample rate */
double minOPD = 0; /* OPD minimum */
double maxOPD = 0; /* OPD maximum */
double ZPD = 0;
double lrmmBandPass = 0.0; /* low res mm +/- offset from centre */
int lrBandPassFrames = 0; /* Number of low res band pass frames from centre +/- length of lrmmBandPass */
int nTmp = 0;
int nMax = 0;
int nOPD = 0;
int bolIndex = 0;
int index = 0;
int indexIn = 0;
int indexOut = 0;
int badPixel = 0;
int k0 = 0;
int indexZPD = 0;
int done = 0; /* Track completion of extracting multiple scans */
int outDataCount = 0; /* The number of output data files being written */
double lenLeft,
lenRight,
minLenLeft,
minLenRight,
minLen,
minZPD,
maxZPD,
midZPD = 0.0; /* Mirror position half side measures */
int midZPDPos = 0; /* Middle ZPD position in mirror position array */
double EPSILON = 0.0;
char fileName[SMF_PATH_MAX+1];
char scanNumStr[5+1]; /* String form of scan number of the input file */
int scanNum = 0; /* Scan number of the input file */
int conNum = 0; /* Concatenation number of the input file (left shifted scanNum) */
int scanDir = 0; /* Scan direction: 1 -> back to front (positive), -1 -> front to back (negative) */
JCMTState *allState = NULL; /* Temporary buffer for reduced header allState array data */
/* Get Input, Output groups */
kpg1Rgndf("IN", 0, 1, "", &gIn, &nFiles, status);
kpg1Wgndf("OUT", gOut, nFiles, nFiles, "More output files expected!", &gOut, &nOutFiles, status);
/* Read in ADAM parameters */
parGet0d("BANDPASS", &lrmmBandPass, status); /* Low res mm band +/- offset from centre */
/* Treat as Low Resolution scan? */
if(lrmmBandPass > 0) {
LR = 1;
}
/* Eliminate the first record in the output group, since it will be replaced later */
gTmp = grpCopy(gOut, 1, 1, 1, status);
grpDelet(&gOut, status);
gOut = gTmp;
/* BEGIN NDF */
ndfBegin();
/* Loop through each input file */
for(fIndex = 1; fIndex <= nFiles; fIndex++) {
/* Open Observation file */
smf_open_file(gIn, fIndex, "READ", 0, &inData, status);
if(*status != SAI__OK) {
*status = SAI__ERROR;
errRep(FUNC_NAME, "Unable to open the source file!", status);
goto CLEANUP;
}
smf_fits_getS(inData->hdr, "OBJECT", object, sizeof(object), status);
smf_fits_getS(inData->hdr, "SUBARRAY", subarray, sizeof(subarray), status);
smf_fits_getS(inData->hdr, "OBSID", obsID, sizeof(obsID), status);
smf_fits_getS(inData->hdr, "FTS_MODE", scanMode, sizeof(scanMode), status);
smf_fits_getD(inData->hdr, "SCANVEL", &scanVel, status);
smf_fits_getD(inData->hdr, "STEPTIME", &stepTime, status);
/* Nyquist frequency */
fNyquist = 10.0 / (8.0 * scanVel * stepTime);
dz = 1.0 / (2.0 * fNyquist);
EPSILON = scanVel * stepTime / 2;
/* Extract the scan number from the input file to be incremented in the output files */
one_strlcpy(scanNumStr, &(inData->file->name[strlen(inData->file->name) - 8]),
astMIN(SMF_PATH_MAX + 1, 5), status);
if (*status == ONE__TRUNC) {
errRep(FUNC_NAME, "Error extracting scanNumStr!", status);
errAnnul(status);
}
/* Create a temporary base file name from input file name */
one_strlcpy(fileName, inData->file->name,
astMIN(SMF_PATH_MAX + 1, strlen(inData->file->name) - 7), status);
if (*status == ONE__TRUNC) {
errRep(FUNC_NAME, "Error extracting base fileName!", status);
errAnnul(status);
}
scanNum = (int) one_strtod(scanNumStr, status);
if (*status != SAI__OK) {
errRep(FUNC_NAME, "Error extracting scanNum!", status);
errAnnul(status);
}
/* Left shift scanNum to conNum as a prefix to make output scan number unique */
if(scanNum < 100) {
conNum = scanNum * 100;
} else if(scanNum < 1000) {
conNum = scanNum * 10;
}
/*printf("%s: Processing file: %s, having basename: %s and scanNumStr: %s, scanNum: %04d\n",
TASK_NAME, inData->file->name, fileName, scanNumStr, scanNum);*/
/* Data cube dimensions */
nWidth = inData->dims[0];
nHeight = inData->dims[1];
nFrames = inData->dims[2];
nPixels = nWidth * nHeight;
/* Mirror positions in mm */
nTmp = nFrames;
MIRPOS = astCalloc(nFrames, sizeof(*MIRPOS));
fts2_getmirrorpositions(inData, MIRPOS, &nTmp, status); // (mm)
if(*status != SAI__OK) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Unable to get the mirror positions!", status);
goto CLEANUP;
}
nStart = -1;
nStop = -1;
nStartNext = 0;
hrStart = -1;
hrStop = -1;
lrStart = -1;
lrStop = -1;
outDataCount = 0;
done = 0;
do {
/* Find the next range of single scan mirror positions for which to extract corresponding NDF data */
for(n=nStartNext; n<nFrames-1; n++){
if(hrStart < 0 && fabs(MIRPOS[n+1] - MIRPOS[n]) >= EPSILON) {
nStart = n;
hrStart = n;
/*printf("%s: Split nStart=%d\n", TASK_NAME, nStart);*/
}
if(hrStart >= 0 && hrStop < 0 && (fabs(MIRPOS[n+1] - MIRPOS[n]) < EPSILON || n+1 == nFrames-1)) {
hrStop = n+1;
hrFramesOutPrev = hrFramesOut;
hrFramesOut = abs(hrStop - hrStart) + 1;
outDataCount++;
nStop = hrStop;
nFramesOutPrev = hrFramesOutPrev;
nFramesOut = hrFramesOut;
/*printf("%s: Split: %d of %d frames found at hrStart=%d, hrStop=%d\n",
TASK_NAME, outDataCount, hrFramesOut, hrStart, hrStop);*/
break;
}
}
/* Determine scan direction */
if(MIRPOS[hrStart] < MIRPOS[hrStop]) {
scanDir = 1; /* Positive */
} else {
scanDir = -1; /* Negative */
}
/* Limit to specified mirror position range */
if(LR) {
/* Calculate how many frames correspond to the given +/- mm of LR bandpass */
lrBandPassFrames = lrmmBandPass / dz;
/* Find the centre of the current scan */
lrCentre = floor((abs(hrStop-hrStart)+1)/2);
/* Set low res start and stop values at corresponding frame offsets from centre */
lrStart = lrCentre - lrBandPassFrames;
lrStop = lrCentre + lrBandPassFrames;
lrFramesOutPrev = lrFramesOut;
lrFramesOut = abs(lrStop - lrStart) + 1;
nStart = lrStart;
nStop = lrStop;
nFramesOutPrev = lrFramesOutPrev;
nFramesOut = lrFramesOut;
/*printf("%s: LR Split: %d of %d frames found at lrStart=%d, lrStop=%d\n",
TASK_NAME, outDataCount, lrFramesOut, lrStart, lrStop);*/
}
/* Check for end of data condition */
if(hrStop < hrStart || hrStop >= nFrames-1) {
hrStop = nFrames-1;
done = 1;
}
/* Output scan if there is a start and stop position found,
and for the last scan if it's the only one
and if it's not too short (compared to the previous one) */
/*printf("%s: nStart=%d, nStop=%d, nFramesOutPrev=%d, nFramesOut=%d\n", TASK_NAME, nStart, nStop, nFramesOutPrev, nFramesOut);*/
if(nStart >=0 && nStop > 0 &&
(nFramesOutPrev == 0 ||
(nFramesOutPrev > 0 && nFramesOut > 0 && (double)hrFramesOut/(double)hrFramesOutPrev >= 0.5))) {
/* Copy single scan NDF data from input to output */
outData = smf_deepcopy_smfData(inData, 0, SMF__NOCREATE_DATA | SMF__NOCREATE_FTS, 0, 0, status);
outData->dtype = SMF__DOUBLE;
outData->ndims = 3;
outData->dims[0] = nWidth;
outData->dims[1] = nHeight;
outData->dims[2] = nFramesOut;
outData_pntr = (double*) astMalloc((nPixels * nFramesOut) * sizeof(*outData_pntr));
outData->pntr[0] = outData_pntr;
outData->hdr->nframes = nFramesOut;
for(i=0; i<nWidth; i++) {
for(j=0; j<nHeight; j++) {
bolIndex = i + j * nWidth;
for(k=nStart; k<=nStop; k++) {
indexIn = bolIndex + k * nPixels;
indexOut = bolIndex + (k-nStart) * nPixels;
*((double*)(outData->pntr[0]) + indexOut) = *((double*)(inData->pntr[0]) + indexIn);
}
}
}
/* Update the FITS headers */
outData->fts = smf_create_smfFts(status);
/* Update FITS component */
smf_fits_updateD(outData->hdr, "FNYQUIST", fNyquist, "Nyquist frequency (cm^-1)", status);
smf_fits_updateI(outData->hdr, "MIRSTART", 1, "Frame index in which the mirror starts moving", status);
smf_fits_updateI(outData->hdr, "MIRSTOP", nFramesOut, "Frame index in which the mirror stops moving", status);
smf_fits_updateI(outData->hdr, "SCANDIR", scanDir, "Scan direction", status);
smf_fits_updateD(outData->hdr, "OPDMIN", 0.0, "Minimum OPD", status);
smf_fits_updateD(outData->hdr, "OPDSTEP", 0.0, "OPD step size", status);
/* Update the JCMTSTATE header */
/* Reallocate outData header array memory to reduced size */
allState = (JCMTState*) astRealloc(outData->hdr->allState, nFramesOut * sizeof(*(outData->hdr->allState)));
if(*status == SAI__OK && allState) {
outData->hdr->allState = allState;
} else {
errRepf(TASK_NAME, "Error reallocating allState JCMTState header", status);
goto CLEANUP;
}
for(k=nStart; k<=nStop; k++) {
/* Copy over JCMTstate */
/*printf("%s: memcpy allState: %d to: %p from: %p size: %d\n",TASK_NAME, k,
(void *) &(outData->hdr->allState[k-nStart]), (void *) &(inData->hdr->allState[k]), sizeof(*(outData->hdr->allState)) );*/
memcpy( (void *) &(outData->hdr->allState[k-nStart]), (void *) &(inData->hdr->allState[k]), sizeof(*(outData->hdr->allState)) );
/*printf("%s: Scan: %d index: %d rts_num: %d\n", TASK_NAME, outDataCount, k-nStart, outData->hdr->allState[k-nStart].rts_num);*/
/*printf("%s: Scan: %d index: %d fts_pos: %f\n", TASK_NAME, outDataCount, k-nStart, outData->hdr->allState[k-nStart].fts_pos);*/
}
/* Write output */
/* Append unique suffix to fileName */
/* This must be modified by the concatenation file scan number to improve uniqueness */
n = one_snprintf(outData->file->name, SMF_PATH_MAX, "%s%04d_scn.sdf", status, fileName, conNum+outDataCount);
/*printf("%s: Writing outData->file->name: %s\n", TASK_NAME, outData->file->name);*/
if(n < 0 || n >= SMF_PATH_MAX) {
errRepf(TASK_NAME, "Error creating outData->file->name", status);
goto CLEANUP;
}
/* Update the list of output _scn file names */
grpPut1(gOut, outData->file->name, 0, status);
if(*status != SAI__OK) {
errRepf(TASK_NAME, "Error saving outData file name", status);
goto CLEANUP;
}
smf_write_smfData(outData, NULL, outData->file->name, gOut, fIndex, 0, MSG__VERB, 0, status);
if(*status != SAI__OK) {
errRepf(TASK_NAME, "Error writing outData file", status);
goto CLEANUP;
}
smf_close_file(&outData, status);
if(*status != SAI__OK) {
errRepf(TASK_NAME, "Error closing outData file", status);
goto CLEANUP;
}
if(*status != SAI__OK) {
errRepf(TASK_NAME, "Error closing outData file", status);
goto CLEANUP;
}
}/* else {
if(!(nStart >=0 && nStop)) printf("%s: Output scan condition failed: nStart(%d) >= nStop(%d) is FALSE\n",TASK_NAME, nStart, nStop);
if(!(nFramesOutPrev == 0 ||
(nFramesOutPrev > 0 && nFramesOut > 0 && (double)nFramesOut/(double)nFramesOutPrev >= 0.5))) printf("%s: Output scan condition failed: nFramesOutPrev(%d) == 0 || (nFramesOutPrev(%d) > 0 && nFramesOut(%d) > 0 && nFramesOut/nFramesOutPrev (%f) >= 0.5) is FALSE\n", TASK_NAME, nFramesOutPrev, nFramesOutPrev, nFramesOut, (double)nFramesOut/(double)nFramesOutPrev);
}*/
/* Prepare for next iteration */
nStartNext = hrStop + 1;
hrStart = -1;
hrStop = -1;
} while (!done);
/* Deallocate memory used by arrays */
if(MIRPOS) { MIRPOS = astFree(MIRPOS); }
/* Close the file */
smf_close_file(&inData, status);
}
CLEANUP:
/* Deallocate memory used by arrays */
if(inData) { smf_close_file(&inData, status); }
if(outData) { smf_close_file(&outData, status); }
/* END NDF */
ndfEnd(status);
/* Write out the list of output NDF names, annulling the error if a null
parameter value is supplied. */
if( *status == SAI__OK && gOut ) {
grpList( "OUTFILES", 0, 0, NULL, gOut, status );
if( *status == PAR__NULL ) {
errRep(FUNC_NAME, "Error writing OUTFILES!", status);
errAnnul( status );
}
}
/* Delete groups */
if(gIn) { grpDelet(&gIn, status); }
if(gOut) { grpDelet(&gOut, status); }
}
void smf_find_science(const Grp * ingrp, Grp **outgrp, int reverttodark,
Grp **darkgrp, Grp **flatgrp, int reducedark,
int calcflat, smf_dtype darktype, smfArray ** darks,
smfArray **fflats, AstKeyMap ** heateffmap,
double * meanstep, int * status ) {
smfSortInfo *alldarks; /* array of sort structs for darks */
smfSortInfo *allfflats; /* array of fast flat info */
Grp * dgrp = NULL; /* Internal dark group */
double duration_darks = 0.0; /* total duration of all darks */
double duration_sci = 0.0; /* Duration of all science observations */
size_t dkcount = 0; /* Dark counter */
size_t ffcount = 0; /* Fast flat counter */
Grp * fgrp = NULL; /* Fast flat group */
size_t i; /* loop counter */
smfData *infile = NULL; /* input file */
size_t insize; /* number of input files */
size_t nsteps_dark = 0; /* Total number of steps for darks */
size_t nsteps_sci = 0; /* Total number of steps for science */
AstKeyMap * heatermap = NULL; /* Heater efficiency map */
AstKeyMap * obsmap = NULL; /* Info from all observations */
AstKeyMap * objmap = NULL; /* All the object names used */
AstKeyMap * scimap = NULL; /* All non-flat obs indexed by unique key */
Grp *ogrp = NULL; /* local copy of output group */
size_t sccount = 0; /* Number of accepted science files */
struct timeval tv1; /* Timer */
struct timeval tv2; /* Timer */
if (meanstep) *meanstep = VAL__BADD;
if (outgrp) *outgrp = NULL;
if (darkgrp) *darkgrp = NULL;
if (darks) *darks = NULL;
if (fflats) *fflats = NULL;
if (heateffmap) *heateffmap = NULL;
if (*status != SAI__OK) return;
/* Sanity check to make sure we return some information */
if ( outgrp == NULL && darkgrp == NULL && darks == NULL && fflats == NULL) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": Must have some non-NULL arguments"
" (possible programming error)", status);
return;
}
/* Start a timer to see how long this takes */
smf_timerinit( &tv1, &tv2, status );
/* Create new group for output files */
ogrp = smf_grp_new( ingrp, "Science", status );
/* and a new group for darks */
dgrp = smf_grp_new( ingrp, "DarkFiles", status );
/* and for fast flats */
fgrp = smf_grp_new( ingrp, "FastFlats", status );
/* and also create a keymap for the observation description */
obsmap = astKeyMap( "KeyError=1" );
/* and an object map */
objmap = astKeyMap( "KeyError=1" );
/* This keymap contains the sequence counters for each related
subarray/obsidss/heater/shutter combination and is used to decide
if a bad flat is relevant */
scimap = astKeyMap( "KeyError=1,KeyCase=0" );
/* This keymap is used to contain relevant heater efficiency data */
heatermap = astKeyMap( "KeyError=1,KeyCase=0" );
/* Work out how many input files we have and allocate sufficient sorting
space */
insize = grpGrpsz( ingrp, status );
alldarks = astCalloc( insize, sizeof(*alldarks) );
allfflats = astCalloc( insize, sizeof(*allfflats) );
/* check each file in turn */
for (i = 1; i <= insize; i++) {
int seqcount = 0;
char keystr[100]; /* Key for scimap entry */
/* open the file but just to get the header */
smf_open_file( ingrp, i, "READ", SMF__NOCREATE_DATA, &infile, status );
if (*status != SAI__OK) break;
/* Fill in the keymap with observation details */
smf_obsmap_fill( infile, obsmap, objmap, status );
/* Find the heater efficiency map if required */
if (*status == SAI__OK && heateffmap) {
char arrayidstr[32];
smf_fits_getS( infile->hdr, "ARRAYID", arrayidstr, sizeof(arrayidstr),
status );
if (!astMapHasKey( heatermap, arrayidstr ) ) {
smfData * heateff = NULL;
dim_t nbolos = 0;
smf_flat_params( infile, "RESIST", NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, &heateff, status );
smf_get_dims( heateff, NULL, NULL, &nbolos, NULL, NULL, NULL, NULL,
status );
if (heateff) astMapPut0P( heatermap, arrayidstr, heateff, NULL );
}
}
/* Get the sequence counter for the file. We do not worry about
duplicate sequence counters (at the moment) */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* The key identifying this subarray/obsidss/heater/shutter combo */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
if (smf_isdark( infile, status )) {
/* Store the sorting information */
dkcount = smf__addto_sortinfo( infile, alldarks, i, dkcount, "Dark", status );
smf__addto_durations( infile, &duration_darks, &nsteps_dark, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
} else {
/* compare sequence type with observation type and drop it (for now)
if they differ */
if ( infile->hdr->obstype == infile->hdr->seqtype ) {
/* Sanity check the header for corruption. Compare RTS_NUM with SEQSTART
and SEQEND. The first RTS_NUM must either be SEQSTART or else between
SEQSTART and SEQEND (if someone has giving us a section) */
int seqstart = 0;
int seqend = 0;
int firstnum = 0;
JCMTState *tmpState = NULL;
smf_getfitsi( infile->hdr, "SEQSTART", &seqstart, status );
smf_getfitsi( infile->hdr, "SEQEND", &seqend, status );
tmpState = infile->hdr->allState;
if( tmpState ) {
firstnum = (tmpState[0]).rts_num;
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
if ( firstnum >= seqstart && firstnum <= seqend ) {
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif(MSG__DEBUG, " ", "Non-dark file: ^F",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
} else {
msgOutif( MSG__QUIET, "",
"File ^F has a corrupt FITS header. Ignoring it.",
status );
}
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif( MSG__DEBUG, " ",
"File ^F lacks JCMTState: assuming it is non-dark",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
}
} else if (infile->hdr->seqtype == SMF__TYP_FASTFLAT ) {
ffcount = smf__addto_sortinfo( infile, allfflats, i, ffcount, "Fast flat", status );
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
msgOutif(MSG__DEBUG, " ", "Sequence type mismatch with observation type: ^F",status);
}
}
/* close the file */
smf_close_file( &infile, status );
}
/* Store output group in return variable or else free it */
if (outgrp) {
*outgrp = ogrp;
} else {
grpDelet( &ogrp, status );
}
/* process flatfields if necessary */
if (ffcount > 0 && fflats ) {
smfArray * array = NULL;
/* sort flats into order */
qsort( allfflats, ffcount, sizeof(*allfflats), smf_sort_bydouble);
if (fflats) array = smf_create_smfArray( status );
/* now open the flats and store them if requested */
if (*status == SAI__OK && array && ffcount) {
size_t start_ffcount = ffcount;
AstKeyMap * flatmap = NULL;
if (calcflat) {
/* Use AgeUp so that we get the keys out in the sorted order
that allfflats used */
flatmap = astKeyMap( "KeyCase=0,KeyError=1,SortBy=AgeDown" );
}
/* Read each flatfield. Calculate a responsivity image and a flatfield
solution. Store these in a keymap along with related information
which is itself stored in a keymap indexed by a string made of
OBSIDSS, reference heater value, shutter and subarray.
*/
for (i = 0; i < start_ffcount; i++ ) {
size_t ori_index = (allfflats[i]).index;
smfData * outfile = NULL;
char keystr[100];
AstKeyMap * infomap = astKeyMap( "KeyError=1" );
int oplen = 0;
char thisfile[MSG__SZMSG];
int seqcount = 0;
/* read filename from group */
infile = NULL;
smf_open_file( ingrp, ori_index, "READ", 0, &infile, status );
if ( *status != SAI__OK ) {
/* This should not happen because we have already opened
the file. If it does happen we abort with error. */
if (infile) smf_close_file( &infile, status );
break;
}
/* Calculate the key for this observation */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
/* Get the file name for error messages */
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>" );
msgLoad( "", "^F", thisfile, sizeof(thisfile), &oplen, status );
/* And the sequence counter to link against science observations */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* Prefill infomap */
astMapPut0C( infomap, "FILENAME", thisfile, "");
astMapPut0I( infomap, "SEQCOUNT", seqcount, "");
/* Collapse it */
if (*status == SAI__OK) {
smf_flat_fastflat( infile, &outfile, status );
if (*status == SMF__BADFLAT) {
errFlush( status );
if (calcflat) {
/* Need to generate an outfile like smf_flat_fastflat
and one heater setting will force smf_flat_calcflat to fail */
smf_flat_malloc( 1, infile, NULL, &outfile, status );
} else {
if (outfile) smf_close_file( &outfile, status );
if (infile) smf_close_file( &infile, status );
infomap = astAnnul( infomap );
ffcount--;
continue;
}
}
}
if (outfile && *status == SAI__OK) {
smf_close_file( &infile, status );
infile = outfile;
if (calcflat) {
size_t ngood = 0;
smfData * curresp = NULL;
int utdate;
if (*status == SAI__OK) {
ngood = smf_flat_calcflat( MSG__VERB, NULL, "RESIST",
"FLATMETH", "FLATORDER", NULL, "RESPMASK",
"FLATSNR", NULL, infile, &curresp, status );
if (*status != SAI__OK) {
/* if we failed to calculate a flatfield we continue but force the
flatfield to be completely bad. This will force the science data associated
with the flatfield to be correctly blanked. We do not annul though
if we have a SUBPAR error telling us that we have failed to define
our parameters properly. */
if (*status != SUBPAR__NOPAR) errAnnul(status);
/* parameters of flatfield */
ngood = 0;
/* Generate a blank flatfield and blank responsivity image */
smf_flat_badflat( infile, &curresp, status );
}
/* Retrieve the UT date so we can decide whether to compare
flatfields */
smf_getfitsi( infile->hdr, "UTDATE", &utdate, status );
/* Store the responsivity data for later on and the processed
flatfield until we have vetted it */
astMapPut0P( infomap, "CALCFLAT", infile, "" );
astMapPut0P( infomap, "RESP", curresp, "" );
astMapPut0I( infomap, "UTDATE", utdate, "" );
astMapPut0I( infomap, "ISGOOD", 1, "" );
astMapPut0I( infomap, "NGOOD", ngood, "" );
astMapPut0I( infomap, "GRPINDEX", ori_index, "" );
astMapPut0I( infomap, "SMFTYP", infile->hdr->obstype, "" );
astMapPutElemA( flatmap, keystr, -1, infomap );
}
} else { /* if (calcflat) */
/* Store the collapsed flatfield - the processed flat is not stored here yet */
smf_addto_smfArray( array, infile, status );
/* Copy the group info */
ndgCpsup( ingrp, ori_index, fgrp, status );
}
} /* if (outfile) */
/* Annul the keymap (will be fine if it is has been stored in another keymap) */
infomap = astAnnul( infomap );
} /* End loop over flatfields */
/* Now we have to loop over the related flatfields to disable
bolometers that are not good and also decide whether we
need to set status to bad. */
if (*status == SAI__OK && calcflat ) {
size_t nkeys = astMapSize( flatmap );
for (i = 0; i < nkeys; i++ ) {
const char *key = astMapKey( flatmap, i );
int nf = 0;
AstKeyMap ** kmaps = NULL;
int nelem = astMapLength( flatmap, key );
kmaps = astMalloc( sizeof(*kmaps) * nelem );
astMapGet1A( flatmap, key, nelem, &nelem, kmaps );
for ( nf = 0; nf < nelem && *status == SAI__OK; nf++ ) {
AstKeyMap * infomap = kmaps[nf];
int isgood = 0;
astMapGet0I( infomap, "ISGOOD", &isgood );
if (isgood) {
/* The flatfield worked */
size_t ngood = 0;
int itemp;
int utdate = 0;
int ratioFlats = 0;
/* Get the UT date - we do not compare flatfields after
the time we enabled heater tracking at each sequence. */
astMapGet0I( infomap, "UTDATE", &utdate );
/* Get the number of good bolometers at this point */
astMapGet0I( infomap, "NGOOD", &itemp );
ngood = itemp;
/* Decide if we want to do the ratio test. We default to
not doing it between 20110901 and 20120827 which is
the period when we did mini-heater tracks before each
flat. ! indicates that we choose based on date. */
if (*status == SAI__OK) {
parGet0l( "FLATUSENEXT", &ratioFlats, status );
if ( *status == PAR__NULL ) {
errAnnul( status );
if (utdate >= 20110901 || utdate <= 20120827 ) {
ratioFlats = 0;
} else {
ratioFlats = 1;
}
}
}
/* Can we compare with the next flatfield? */
if (ngood < SMF__MINSTATSAMP || !ratioFlats ) {
/* no point doing all the ratio checking for this */
} else if ( nelem - nf >= 2 ) {
AstKeyMap * nextmap = kmaps[nf+1];
const char *nextfname = NULL;
const char *fname = NULL;
smfData * curresp = NULL;
smfData * nextresp = NULL;
smfData * curflat = NULL;
void *tmpvar = NULL;
size_t bol = 0;
smfData * ratio = NULL;
double *in1 = NULL;
double *in2 = NULL;
double mean = VAL__BADD;
size_t nbolo;
double *out = NULL;
double sigma = VAL__BADD;
float clips[] = { 5.0, 5.0 }; /* 5.0 sigma iterative clip */
size_t ngoodz = 0;
astMapGet0C( nextmap, "FILENAME", &nextfname );
astMapGet0C( infomap, "FILENAME", &fname );
/* Retrieve the responsivity images from the keymap */
astMapGet0P( infomap, "RESP", &tmpvar );
curresp = tmpvar;
astMapGet0P( nextmap, "RESP", &tmpvar );
nextresp = tmpvar;
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
nbolo = (curresp->dims)[0] * (curresp->dims)[1];
/* get some memory for the ratio if we have not already.
We could get some memory once assuming each flat has the
same number of bolometers... */
ratio = smf_deepcopy_smfData( curresp, 0, 0, 0, 0, status );
if( *status == SAI__OK ) {
/* divide: smf_divide_smfData ? */
in1 = (curresp->pntr)[0];
in2 = (nextresp->pntr)[0];
out = (ratio->pntr)[0];
for (bol=0; bol<nbolo;bol++) {
if ( in1[bol] != VAL__BADD && in1[bol] != 0.0 &&
in2[bol] != VAL__BADD && in2[bol] != 0.0 ) {
out[bol] = in1[bol] / in2[bol];
} else {
out[bol] = VAL__BADD;
}
}
}
/* find some statistics */
smf_clipped_stats1D( out, 2, clips, 1, nbolo, NULL, 0, 0, &mean,
&sigma, NULL, 0, &ngoodz, status );
if (*status == SMF__INSMP) {
errAnnul(status);
msgOutiff( MSG__QUIET, "",
"Flatfield ramp ratio of %s with %s had too few bolometers (%zu < %d).",
status, fname, nextfname, ngoodz, SMF__MINSTATSAMP );
ngood = ngoodz; /* Must be lower or equal to original ngood */
} else if (*status == SAI__OK && mean != VAL__BADD && sigma != VAL__BADD && curflat->da) {
/* Now flag the flatfield as bad for bolometers that have changed
more than n%. We expect the variation to be 1+/-a small bit */
const double pmrange = 0.10;
double thrlo = 1.0 - pmrange;
double thrhi = 1.0 + pmrange;
size_t nmasked = 0;
double *flatcal = curflat->da->flatcal;
msgOutiff( MSG__DEBUG, "", "Flatfield fast ramp ratio mean = %g +/- %g (%zu bolometers)",
status, mean, sigma, ngood);
/* we can just set the first slice of the flatcal to bad. That should
be enough to disable the entire bolometer. We have just read these
data so they should be in ICD order. */
for (bol=0; bol<nbolo;bol++) {
if ( out[bol] != VAL__BADD &&
(out[bol] < thrlo || out[bol] > thrhi ) ) {
flatcal[bol] = VAL__BADD;
nmasked++;
} else if ( in1[bol] != VAL__BADD && in2[bol] == VAL__BADD ) {
/* A bolometer is bad next time but good now so we must set it bad now */
flatcal[bol] = VAL__BADD;
nmasked++;
}
}
if ( nmasked > 0 ) {
msgOutiff( MSG__NORM, "", "Masked %zu bolometers in %s from unstable flatfield",
status, nmasked, fname );
/* update ngood to take into account the masking */
ngood -= nmasked;
}
}
smf_close_file( &ratio, status );
} /* End of flatfield responsivity comparison */
/* if we only have a few bolometers left we now consider this
a bad flat unless it is actually an engineering measurement
where expect some configurations to give zero bolometers */
if (ngood < SMF__MINSTATSAMP) {
const char *fname = NULL;
void * tmpvar = NULL;
int smftyp = 0;
smfData * curflat = NULL;
astMapGet0I( infomap, "SMFTYP", &smftyp );
astMapGet0C( infomap, "FILENAME", &fname );
if (smftyp != SMF__TYP_NEP) {
msgOutiff( MSG__QUIET, "",
"Flatfield %s has %zu good bolometer%s.%s",
status, fname, ngood, (ngood == 1 ? "" : "s"),
( ngood == 0 ? "" : " Keeping none.") );
isgood = 0;
/* Make sure that everything is blanked. */
if (ngood > 0) {
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
if (curflat && curflat->da) {
size_t bol;
size_t nbolo = (curflat->dims)[0] * (curflat->dims)[1];
double *flatcal = curflat->da->flatcal;
for (bol=0; bol<nbolo; bol++) {
/* Just need to set the first element to bad */
flatcal[bol] = VAL__BADD;
}
}
}
} else {
msgOutiff( MSG__NORM, "",
"Flatfield ramp file %s has %zu good bolometer%s. Eng mode.",
status, fname, ngood, (ngood == 1 ? "" : "s") );
}
}
/* We do not need the responsivity image again */
{
void *tmpvar = NULL;
smfData * resp = NULL;
astMapGet0P( infomap, "RESP", &tmpvar );
resp = tmpvar;
if (resp) smf_close_file( &resp, status );
astMapRemove( infomap, "RESP" );
}
} /* End of isgood comparison */
/* We are storing flats even if they failed. Let the downstream
software worry about it */
{
int ori_index;
smfData * flatfile = NULL;
void *tmpvar = NULL;
/* Store in the output group */
astMapGet0I( infomap, "GRPINDEX", &ori_index );
ndgCpsup( ingrp, ori_index, fgrp, status );
/* And store in the smfArray */
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
astMapRemove( infomap, "CALCFLAT" );
flatfile = tmpvar;
smf_addto_smfArray( array, flatfile, status );
}
/* Free the object as we go */
kmaps[nf] = astAnnul( kmaps[nf] );
} /* End of loop over this obsidss/subarray/heater */
kmaps = astFree( kmaps );
}
}
if (array->ndat) {
if (fflats) *fflats = array;
} else {
smf_close_related(&array, status );
if (fflats) *fflats = NULL;
}
}
}
/* no need to do any more if neither darks nor darkgrp are defined or we might
be wanting to revert to darks. */
if (dkcount > 0 && (darks || darkgrp || reverttodark ) ) {
smfArray * array = NULL;
/* sort darks into order */
qsort( alldarks, dkcount, sizeof(*alldarks), smf_sort_bydouble);
if (darks) array = smf_create_smfArray( status );
/* now open the darks and store them if requested */
if (*status == SAI__OK) {
for (i = 0; i < dkcount; i++ ) {
size_t ori_index = (alldarks[i]).index;
/* Store the entry in the output group */
ndgCpsup( ingrp, ori_index, dgrp, status );
if (darks) {
/* read the value from the new group */
smf_open_file( dgrp, i+1, "READ", 0, &infile, status );
/* do we have to process these darks? */
if (reducedark) {
smfData *outfile = NULL;
smf_reduce_dark( infile, darktype, &outfile, status );
if (outfile) {
smf_close_file( &infile, status );
infile = outfile;
}
}
smf_addto_smfArray( array, infile, status );
}
}
if (darks) *darks = array;
}
}
/* free memory */
alldarks = astFree( alldarks );
allfflats = astFree( allfflats );
if( reverttodark && outgrp && (grpGrpsz(*outgrp,status)==0) &&
(grpGrpsz(dgrp,status)>0) ) {
/* If outgrp requested but no science observations were found, and
dark observations were found, return darks in outgrp and set
flatgrp and darkgrp to NULL. This is to handle cases where we
want to process data taken in the dark like normal science
data. To activate this behaviour set reverttodark */
msgOutiff( MSG__NORM, "", "Treating the dark%s as science data",
status, ( dkcount > 1 ? "s" : "" ) );
*outgrp = dgrp;
if( darkgrp ){
*darkgrp = NULL;
}
if( flatgrp ) {
*flatgrp = NULL;
}
grpDelet( &ogrp, status);
grpDelet( &fgrp, status);
if (meanstep && nsteps_dark > 0) *meanstep = duration_darks / nsteps_dark;
/* Have to clear the darks smfArray as well */
if (darks) smf_close_related( darks, status );
} else {
/* Store the output groups in the return variable or free it */
if (darkgrp) {
*darkgrp = dgrp;
} else {
grpDelet( &dgrp, status);
}
if (flatgrp) {
*flatgrp = fgrp;
} else {
grpDelet( &fgrp, status);
}
if (meanstep && nsteps_sci > 0) *meanstep = duration_sci / nsteps_sci;
}
msgSeti( "ND", sccount );
msgSeti( "DK", dkcount );
msgSeti( "FF", ffcount );
msgSeti( "TOT", insize );
if ( insize == 1 ) {
if (dkcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a dark",
status);
} else if (ffcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a fast flatfield",
status);
} else if (sccount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was accepted (observation type same as sequence type)",
status);
} else {
msgOutif( MSG__VERB, " ", "Single input file was not accepted.",
status);
}
} else {
if (dkcount == 1) {
msgSetc( "DKTXT", "was a dark");
} else {
msgSetc( "DKTXT", "were darks");
}
if (ffcount == 1) {
msgSetc( "FFTXT", "was a fast flat");
} else {
msgSetc( "FFTXT", "were fast flats");
}
if (sccount == 1) {
msgSetc( "NDTXT", "was science");
} else {
msgSetc( "NDTXT", "were science");
}
/* This might be a useful message */
msgOutif( MSG__NORM, " ", "Out of ^TOT input files, ^DK ^DKTXT, ^FF ^FFTXT "
"and ^ND ^NDTXT", status );
}
if (meanstep && *meanstep != VAL__BADD) {
msgOutiff( MSG__VERB, "", "Mean step time for input files = %g sec",
status, *meanstep );
}
/* Store the heater efficiency map */
if (*status != SAI__OK) heatermap = smf_free_effmap( heatermap, status );
if (heateffmap) *heateffmap = heatermap;
/* Now report the details of the observation */
smf_obsmap_report( MSG__NORM, obsmap, objmap, status );
obsmap = astAnnul( obsmap );
objmap = astAnnul( objmap );
scimap = astAnnul( scimap );
msgOutiff( SMF__TIMER_MSG, "",
"Took %.3f s to find science observations",
status, smf_timerupdate( &tv1, &tv2, status ) );
return;
}
void smf_filter_mce( smfFilter *filt, int noinverse, int *status ) {
/* Filter parameters */
double B_1_1;
double B_1_2;
double B_2_1;
double B_2_2;
double CLOCK_PERIOD;
double ROW_DWELL;
double NUM_ROWS=41;
double DELTA_TIME;
double SRATE;
double datechange; /* UTC MJD for change in MCE filter parameters */
AstTimeFrame *tf=NULL; /* time frame for date conversion */
size_t i; /* Loop counter */
if( *status != SAI__OK ) return;
if( !filt ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "NULL smfFilter supplied.", status );
return;
}
if( filt->ndims != 1 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": function only generates filters for time-series",
status );
return;
}
if( !filt->fdims[0] ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": 0-length smfFilter supplied.",
status );
return;
}
if( filt->dateobs == VAL__BADD ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": dateobs (date of data to which filter will be "
"applied) is not set - can't determine correct MCE filter "
"parameters.", status );
return;
}
/* If filt->real is NULL, create a complex identity filter first. Similarly,
if the filter is currently only real-valued, add an imaginary part. */
if( !filt->real ) {
smf_filter_ident( filt, 1, status );
if( *status != SAI__OK ) return;
} else if( !filt->imag ) {
filt->imag = astCalloc( filt->fdims[0], sizeof(*filt->imag) );
if( *status != SAI__OK ) return;
filt->isComplex = 1;
}
/* Set up filter parameters */
tf = astTimeFrame( " " );
astSet( tf, "TimeScale=UTC" );
astSet( tf, "TimeOrigin=%s", "2011-06-03T00:00:00" );
datechange = astGetD( tf, "TimeOrigin" );
tf = astAnnul( tf );
if( filt->dateobs > datechange ) {
/* Data taken after 20110603 */
B_1_1 = -1.9712524; /* -2.*32297./2.^15. */
B_1_2 = 0.97253418; /* 2.*15934./2.^15. */
B_2_1 = -1.9337769; /* -2.*31683./2.^15. */
B_2_2 = 0.93505859; /* 2.*15320./2.^15. */
ROW_DWELL = 94.; /* time to dwell at each row (in clocks) */
msgOutiff(MSG__DEBUG, "", FUNC_NAME
": filter for data UTC MJD %lf after %lf", status,
filt->dateobs, datechange );
} else {
/* Older data */
B_1_1 = -1.9587402; /* -2.*32092./2.^15. */
B_1_2 = 0.96130371; /* 2.*15750./2.^15. */
B_2_1 = -1.9066162; /* -2.*31238./2.^15. */
B_2_2 = 0.90911865; /* 2.*14895./2.^15. */
ROW_DWELL = 128.; /* time to dwell at each row (in clocks) */
msgOutiff(MSG__DEBUG, "", FUNC_NAME
": filter for data UTC MJD %lf before %lf", status,
filt->dateobs, datechange );
}
CLOCK_PERIOD = 20E-9; /* 50 MHz clock */
NUM_ROWS = 41.; /* number of rows addressed */
DELTA_TIME = (CLOCK_PERIOD*ROW_DWELL*NUM_ROWS); /* sample length */
SRATE = (1./DELTA_TIME); /* sample rate */
/* Loop over all frequencies in the filter */
for( i=0; i<filt->fdims[0]; i++ ) {
double cos_m_o;
double sin_m_o;
double cos_m_2o;
double sin_m_2o;
double f;
gsl_complex den;
gsl_complex h1_omega;
gsl_complex h2_omega;
gsl_complex h_omega;
gsl_complex num;
gsl_complex temp;
double omega;
f = filt->df[0]*i; /* Frequency at this step */
omega = (f / SRATE)*2*AST__DPI; /* Angular frequency */
cos_m_o = cos(-omega);
sin_m_o = sin(-omega);
cos_m_2o = cos(-2*omega);
sin_m_2o = sin(-2*omega);
/*
h1_omega=(1 + 2*complex(cos_m_o,sin_m_o) + complex(cos_m_2o,sin_m_2o)) /
(1 + b_1_1*complex(cos_m_o,sin_m_o) + b_1_2 * complex(cos_m_2o,sin_m_2o))
*/
/* numerator */
GSL_SET_COMPLEX(&num, 1, 0);
GSL_SET_COMPLEX(&temp, cos_m_o, sin_m_o);
num = gsl_complex_add( num, gsl_complex_mul_real(temp, 2) );
GSL_SET_COMPLEX(&temp, cos_m_2o, sin_m_2o);
num = gsl_complex_add( num, temp );
/* denominator */
GSL_SET_COMPLEX(&den, 1, 0);
GSL_SET_COMPLEX(&temp, cos_m_o, sin_m_o);
den = gsl_complex_add( den, gsl_complex_mul_real(temp,B_1_1) );
GSL_SET_COMPLEX(&temp, cos_m_2o, sin_m_2o);
den = gsl_complex_add( den, gsl_complex_mul_real(temp,B_1_2) );
/* quotient */
h1_omega = gsl_complex_div( num, den );
/*
h2_omega=(1 + 2*complex(cos_m_o,sin_m_o) + complex(cos_m_2o,sin_m_2o)) /
(1 + b_2_1*complex(cos_m_o,sin_m_o) + b_2_2*complex(cos_m_2o,sin_m_2o))
note: we can re-use numerator from above
*/
/* denominator */
GSL_SET_COMPLEX(&den, 1, 0);
GSL_SET_COMPLEX(&temp, cos_m_o, sin_m_o);
den = gsl_complex_add( den, gsl_complex_mul_real(temp,B_2_1) );
GSL_SET_COMPLEX(&temp, cos_m_2o, sin_m_2o);
den = gsl_complex_add( den, gsl_complex_mul_real(temp,B_2_2) );
/* quotient */
h2_omega = gsl_complex_div( num, den );
/* And finally...
h_omega=h1_omega*h2_omega/2048.
*/
h_omega = gsl_complex_mul( h1_omega, gsl_complex_div_real(h2_omega,2048.) );
/* Normally we are applying the inverse of the filter to remove
its effect from the time-series. */
if( !noinverse ) {
h_omega = gsl_complex_inverse( h_omega );
}
/* Then apply this factor to the filter. */
GSL_SET_COMPLEX( &temp, filt->real[i], filt->imag[i] );
temp = gsl_complex_mul( temp, h_omega );
filt->real[i] = GSL_REAL( temp );
filt->imag[i] = GSL_IMAG( temp );
}
}
void smf_flat_malloc( size_t nheat, const smfData * refdata,
smfData **powvald, smfData **bolvald, int *status ) {
size_t rowidx = SC2STORE__ROW_INDEX;
size_t colidx = SC2STORE__COL_INDEX;
double * bolval = NULL; /* Data array inside bolrefd */
double * bolvalvar = NULL; /* Variance inside bolrefd */
dim_t dims[] = { 1, 1, 1 }; /* Default dimensions */
smfHead * hdr = NULL; /* New header */
int lbnd[] = { 1, 1, 1 }; /* Default pixel lower bounds */
size_t nelem = 0; /* Number of elements in first two dimensions of refdims */
smfHead * oldhdr = NULL; /* header from refdata */
void *pntr[] = { NULL, NULL }; /* pointers for smfData */
double * powval = NULL; /* Data array inside powrefd */
if (bolvald) *bolvald = NULL;
if (powvald) *powvald = NULL;
if ( *status != SAI__OK ) return;
if ( !bolvald && !powvald) {
*status = SAI__ERROR;
errRep( "", "Must provide at least one non-NULL pointer to smf_flat_malloc"
" (possible programming error)", status );
return;
}
/* Sanity check */
if ( nheat == 0 ) {
*status = SAI__ERROR;
errRep( "", "No flatfield information present for creating new smfData",
status );
return;
}
if ( !smf_validate_smfData( refdata, 1, 0, status ) ) return;
oldhdr = refdata->hdr;
if (powvald) {
powval = astCalloc( nheat, sizeof(*powval) );
pntr[0] = powval;
pntr[1] = NULL;
dims[0] = nheat;
*powvald = smf_construct_smfData( NULL, NULL, NULL, NULL, NULL, SMF__DOUBLE,
pntr, NULL, SMF__QFAM_NULL, NULL, 0, 1,
dims, NULL, 1, 0, 0, NULL,
NULL, status );
}
if (bolvald) {
/* Handle data ordering */
if ( ! refdata->isTordered ) {
rowidx++;
colidx++;
}
nelem = refdata->dims[rowidx] * refdata->dims[colidx];
bolval = astCalloc( nheat * nelem, sizeof(*bolval) );
bolvalvar = astCalloc( nheat * nelem, sizeof(*bolvalvar) );
pntr[0] = bolval;
pntr[1] = bolvalvar;
dims[SC2STORE__ROW_INDEX] = refdata->dims[rowidx];
dims[SC2STORE__COL_INDEX] = refdata->dims[colidx];
dims[2] = nheat;
lbnd[SC2STORE__ROW_INDEX] = refdata->lbnd[rowidx];
lbnd[SC2STORE__COL_INDEX] = refdata->lbnd[colidx];
lbnd[2] = 1;
/* Create a header to attach to the bolometer data. We only want the basic 2-d
information to propagate. */
hdr = smf_construct_smfHead( NULL, oldhdr->instrument, NULL, NULL,
astCopy( oldhdr->fitshdr ), NULL, 0,
oldhdr->instap, nheat, oldhdr->steptime,
oldhdr->scanvel, oldhdr->obsmode,
oldhdr->swmode, oldhdr->obstype,
oldhdr->seqtype, oldhdr->inbeam, 0, NULL, NULL,
NULL, NULL, 0, NULL,
"Flatfield measurement", "Response",
oldhdr->units, oldhdr->telpos,
NULL, oldhdr->obsidss, status );
*bolvald = smf_construct_smfData( NULL, NULL, hdr, NULL, NULL, SMF__DOUBLE,
pntr, NULL, SMF__QFAM_TSERIES, NULL, 0, 1,
dims, lbnd, 3, 0, 0, NULL, NULL, status );
}
return;
}
static int smf1_check_steps( const char *param, int first, dim_t nx,
double sizetol, int nold, int nnew,
smfStepFix *oldsteps, smfStepFix *newsteps,
int *status ){
/*
* Name:
* smf1_check_steps
* Purpose:
* Compare two sets of steps, issuing a warning for each step that
* has changed significantly.
* Invocation:
* int smf1_check_steps( const char *param, int first, dim_t nx,
* double sizetol, int nold, int nnew,
* smfStepFix *oldsteps, smfStepFix *newsteps,
* int *status )
* Arguments:
* param = const char * (Given)
* Name of parameter to use when asking the user whether to
* continue to look for further changes.
* first = int (Given)
* The index of the first change to report.
* nx = dim_t (Given)
* The length of the first axis of the bolometer array.
* sizetol = double (Given)
* The minimum significant relative error in step size.
* nold = int (Given)
* The number of steps in the "oldsteps" array.
* nnew = int (Given)
* The number of steps in the "newsteps" array.
* oldsteps = smfStepFix * (Given and Returned)
* A pointer to the first element of an array of smfStepFix structures
* describing the steps fixed in a previous invocation of this program.
* The array is sorted on exit.
* newsteps = smfStepFix * (Given and Returned)
* A pointer to the first element of an array of smfStepFix structures
* describing the steps fixed in the current invocation of this program.
* The array is sorted on exit.
* status = int * (Given and Returned)
* Pointer to global status.
* Description:
* Firstly, an attempt it made to associated each old step with a
* corresponding new step (i.e. a new step that occurs at the same
* time and in the same bolometer as the old step). Warning messages
* are issued about each old step for which no corresponding new step
* can be found, or for which the corresponding new step has a
* significantly different height to the old step. Finally, warnings
* messages are also issued for each new step that has not been
* associated with an old step.
* Returned Value:
* Zero if no significant differences were found. Non-zero otherwise.
*/
/* Local Variables: */
double abslim;
double dsize;
double dsize_min;
int *fnew;
int *new_flags;
int cont;
int inew;
int iold;
int jnew;
int match;
int result;
smfStepFix *pnew;
smfStepFix *pold;
/* Initialise the returned value. */
result = 0;
/* Check the inherited status. */
if( *status != SAI__OK ) return result;
/* Find the absolute minimum significant difference between step sizes.
This is "sizetol" times the clipped RMS step size in the new steps. */
abslim = sizetol*smf1_get_rmssize( nnew, newsteps, status );
msgSetd( "T", abslim );
msgOut( "", "Ignoring differences in step size smaller than ^T",
status );
msgBlank( status );
/* Allocate memory to hold an array with one element for each new step.
Each element holds zero if the new step has not yet been associated
with any old step. Otherwise, it holds the one-based index of the
associated old step. Initialise it to hold zero at every element. */
new_flags = astCalloc( nnew, sizeof( *new_flags ) );
if( *status == SAI__OK ) {
/* Loop round each old step. */
pold = oldsteps;
for( iold = 0; iold < nold; iold++,pold++ ) {
/* Ignore old steps with bolometer indicies greater than 5000 */
if( pold->ibolo > 5000 ) continue;
/* Indicate no new step has yet been associated with the old step. */
jnew = -1;
dsize_min = VAL__MAXD;
match = 0;
/* Loop round all new steps. */
pnew = newsteps;
fnew = new_flags;
for( inew = 0; inew < nnew; inew++,pnew++,fnew++ ) {
/* Ignore this new step if it has already been associated with a previous
old step. */
if( ! *fnew ) {
/* See if the current new and old steps occur in the same bolometer and
have overlapping time spans. If so they are considered to be at the
same time. */
if( pold->ibolo == pnew->ibolo &&
pold->start <= pnew->end &&
pold->end >= pnew->start ) {
/* Get the difference in step size between the old and new steps. */
dsize = fabs( pold->size - pnew->size );
/* Note the index of the matching new step that is most similar in height
to the old step. */
if( dsize < dsize_min ) {
jnew = inew;
dsize_min = dsize;
}
/* If the old and new step heights are about the same then we associate
the new step with the old step. Store the (one based) index of the
corresponding old step. We do not need to check any more new steps,
so break out of the new step loop. */
if( dsize < abslim ||
dsize < sizetol*fabs( 0.5*( pold->size + pnew->size ) ) ) {
match = 1;
*fnew = iold + 1;
break;
}
}
}
}
/* If a new step was found at the same time and place as the old step, and
with the same height, pass on to the next old step. */
if( ! match ) {
/* If no new step was found at the same time and place as the old step, an old
step has dissappeared. */
if( jnew == -1 ) {
/* If the old step was of significant height, tell the user. */
if( fabs( pold->size ) > abslim ){
result++;
if( result >= first ) {
msgSeti( "N", result );
msgSeti( "I", pold->id );
msgSetc( "W", pold->corr ? "secondary" : "primary" );
msgOut( "", "^N: An old ^W step (index ^I) is no longer found:", status );
msgSeti( "B", pold->ibolo );
msgSeti( "X", pold->ibolo % nx );
msgSeti( "Y", pold->ibolo / nx );
msgOut( "", " Bolometer = ^B (^X,^Y)", status );
msgSeti( "S", pold->start );
msgSeti( "E", pold->end );
msgOut( "", " Time slice range = ^S:^E", status );
msgSetd( "H", pold->size );
msgOut( "", " Height = ^H", status );
parGet0l( param, &cont, status );
parCancl( param, status );
if( !cont || *status != SAI__OK ) break;
msgBlank( status );
}
}
/* If one or more new step were found at the same time and place as the
old step (but with a significantly different height), tell the user
about the change in height. */
} else {
pnew = newsteps + jnew;
result++;
if( result >= first ) {
msgSeti( "I", result );
msgSetd( "O", pold->size );
msgSetd( "N", pnew->size );
msgOut( "", "^I: Step size changed from ^O to ^N:", status );
msgSeti( "I", pnew->id );
msgSetc( "W", pnew->corr ? "secondary" : "primary" );
msgOut( "", " New index = ^I (^W)", status );
msgSeti( "I", pold->id );
msgSetc( "W", pold->corr ? "secondary" : "primary" );
msgOut( "", " Old index = ^I (^W)", status );
msgSeti( "B", pold->ibolo );
msgSeti( "X", pold->ibolo % nx );
msgSeti( "Y", pold->ibolo / nx );
msgOut( "", " Bolometer = ^B (^X,^Y)", status );
msgSeti( "S", pold->start );
msgSeti( "E", pold->end );
msgOut( "", " Old time slice range = ^S:^E", status );
msgSeti( "S", pnew->start );
msgSeti( "E", pnew->end );
msgOut( "", " New time slice range = ^S:^E", status );
parGet0l( param, &cont, status );
parCancl( param, status );
if( !cont || *status != SAI__OK ) break;
msgBlank( status );
}
}
}
}
/* We have now checked all old steps for matching new steps. If no
significant change has yet been found, look for new steps that
have not been associated with an old step. */
pnew = newsteps;
fnew = new_flags;
for( inew = 0; inew < nnew && cont; inew++,pnew++,fnew++ ) {
if( ! *fnew ) {
/* If the new step is off significant height, tell the user. */
if( fabs( pnew->size ) > abslim ){
result++;
if( result >= first ) {
msgSeti( "N", result );
msgSeti( "I", pnew->id );
msgSetc( "W", pnew->corr ? "secondary" : "primary" );
msgOut( "", "^N: A new ^W step (index ^I) was found:", status );
msgSeti( "B", pnew->ibolo );
msgSeti( "X", pnew->ibolo % nx );
msgSeti( "Y", pnew->ibolo / nx );
msgOut( "", " Bolometer = ^B (^X,^Y)", status );
msgSeti( "S", pnew->start );
msgSeti( "E", pnew->end );
msgOut( "", " Time slice range = ^S:^E", status );
msgSetd( "H", pnew->size );
msgOut( "", " Height = ^H", status );
parGet0l( param, &cont, status );
parCancl( param, status );
if( !cont || *status != SAI__OK ) break;
msgBlank( status );
}
}
}
}
}
/* Free resources. */
new_flags = astFree( new_flags );
/* Return the result. */
return result;
}
void smf_calcmodel_noi( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel, int flags,
int *status) {
/* Local Variables */
dim_t bolostep; /* Number of bolos per thread */
dim_t boxsize; /* No. of time slices in each noise box */
smfData *box = NULL; /* SmfData holding one box of input data */
size_t bstride; /* bolometer stride */
int calcfirst=0; /* Were bolo noises already measured? */
int dclimcorr; /* Min number of correlated steps */
int dcmaxsteps; /* Maximum allowed number of dc jumps */
dim_t dcfitbox; /* Width of box for DC step detection */
double dcthresh; /* Threshold for DC step detection */
dim_t dcsmooth; /* Width of median filter in DC step detection*/
double *din; /* Pointer to next input value */
double *dout; /* Pointer to next output value */
int fillgaps; /* If set perform gap filling */
dim_t i; /* Loop counter */
dim_t ibolo; /* Bolometer index */
int ibox; /* Index of current noise box */
dim_t itime; /* Time slice index */
dim_t idx=0; /* Index within subgroup */
JCMTState *instate=NULL; /* Pointer to input JCMTState */
int iw; /* Thread index */
dim_t j; /* Loop counter */
AstKeyMap *kmap=NULL; /* Local keymap */
size_t mbstride; /* model bolometer stride */
dim_t mntslice; /* Number of model time slices */
size_t mtstride; /* model time slice stride */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
dim_t nbolo; /* Number of bolometers */
int nbox = 0; /* Number of noise boxes */
size_t nchisq; /* Number of data points in chisq calc */
dim_t nelbox; /* Number of data points in a noise box */
dim_t ndata; /* Total number of data points */
size_t nflag; /* Number of new flags */
int nleft; /* Number of samples not in a noise box */
dim_t ntslice; /* Number of time slices */
int nw; /* Number of worker threads */
size_t pend; /* Last non-PAD sample */
size_t pstart; /* First non-PAD sample */
smf_qual_t *qin; /* Pointer to next input quality value */
smf_qual_t *qout; /* Pointer to next output quality value */
smfArray *qua=NULL; /* Pointer to RES at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to RES at chunk */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
dim_t spikebox=0; /* Box size for spike detection */
double spikethresh=0; /* Threshold for spike detection */
size_t tend; /* Last input sample to copy */
size_t tstart; /* First input sample to copy */
size_t tstride; /* time slice stride */
double *var=NULL; /* Sample variance */
size_t xbstride; /* Box bolometer stride */
int zeropad; /* Pad with zeros? */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointer to sub-keymap containing NOI parameters */
astMapGet0A( keymap, "NOI", &kmap );
/* Assert bolo-ordered data */
smf_model_dataOrder( dat, allmodel, chunk, SMF__RES|SMF__QUA, 0, status );
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
model = allmodel[chunk];
/* Obtain parameters for NOI */
/* Data-cleaning parameters */
smf_get_cleanpar( kmap, res->sdata[0], NULL, &dcfitbox, &dcmaxsteps, &dcthresh,
&dcsmooth, &dclimcorr, NULL, &fillgaps, &zeropad, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
&spikethresh, &spikebox, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, status );
/* Did we already calculate the noise on each detector? */
astMapGet0I( kmap, "CALCFIRST", &calcfirst );
/* Initialize chisquared */
dat->chisquared[chunk] = 0;
nchisq = 0;
/* Loop over index in subgrp (subarray) */
for( idx=0; idx<res->ndat; idx++ ) {
/* Get pointers to DATA components */
res_data = (res->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Get the raw data dimensions */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice, &ndata,
&bstride, &tstride, status );
/* NOI model dimensions */
smf_get_dims( model->sdata[idx], NULL, NULL, NULL, &mntslice, NULL,
&mbstride, &mtstride, status );
/* Only estimate the white noise level once at the beginning - the
reason for this is to make measurements of the convergence
easier. We either do it prior to the start of iterations (in which
case the relative weights will be influeced by low-frequency noise,
this is initialized in smf_model_create), or else we calculate
the noise after the first iteration. */
if( (flags & SMF__DIMM_FIRSTITER) && (!calcfirst) ) {
/* There are two forms for the NOI model: one constant noise value
for each bolometer, or "ntslice" noise values for each bolometer.
Handle the first case now. */
if( mntslice == 1 ) {
var = astMalloc( nbolo*sizeof(*var) );
if (var) {
/* Measure the noise from power spectra */
smf_bolonoise( wf, res->sdata[idx], 0, 0.5, SMF__F_WHITELO,
SMF__F_WHITEHI, 0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT,
var, NULL, NULL, status );
for( i=0; i<nbolo; i++ ) if( !(qua_data[i*bstride]&SMF__Q_BADB) ) {
/* Loop over time and store the variance for each sample */
for( j=0; j<mntslice; j++ ) {
model_data[i*mbstride+(j%mntslice)*mtstride] = var[i];
}
}
var = astFree( var );
}
/* If the NOI model is of the second form, the noise is estimated
in boxes of samples lasting "NOI.BOX_SIZE" seconds, and then the
noise level in the box is assigned to all samples in the box. */
} else if( mntslice == ntslice ) {
/* If not already done, get NOI.BOX_SIZE and convert from seconds to
samples. */
if( idx == 0 ) {
boxsize = 0;
smf_get_nsamp( kmap, "BOX_SIZE", res->sdata[0], &boxsize, status );
msgOutf( "", FUNC_NAME ": Calculating a NOI variance for each "
"box of %d samples.", status, (int) boxsize );
/* Find the indices of the first and last non-PAD sample. */
smf_get_goodrange( qua_data, ntslice, tstride, SMF__Q_PAD,
&pstart, &pend, status );
/* How many whole boxes fit into this range? */
nbox = ( pend - pstart + 1 ) / boxsize;
if( nbox == 0 ) nbox = 1;
/* How many samples would be left over at the end if we used this
many boxes? */
nleft = ( pend - pstart + 1 ) - nbox*boxsize;
/* Increase "boxsize" to reduce this number as far as possible.
Any samples that are left over after this increase of boxsize
will not be used when calculating the noise levels in each
bolometer. */
boxsize += nleft/nbox;
/* Create a smfData to hold one box-worth of input data. We
do not need to copy jcmtstate information. */
if( res->sdata[idx]->hdr ) {
instate = res->sdata[idx]->hdr->allState;
res->sdata[idx]->hdr->allState = NULL;
}
box = smf_deepcopy_smfData( res->sdata[idx], 0,
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY,
0, 0, status );
if( instate ) res->sdata[idx]->hdr->allState = instate;
/* Set the length of the time axis to the box size plus padding,
and create empty data and quality arrays for it. */
if( *status == SAI__OK ) {
box->dims[ box->isTordered?2:0 ] = boxsize + pstart + (ntslice - pend - 1);
smf_get_dims( box, NULL, NULL, NULL, NULL, &nelbox,
&xbstride, NULL, status );
box->pntr[0] = astMalloc( sizeof( double )*nelbox );
box->qual = astMalloc( sizeof( smf_qual_t )*nelbox );
/* For every bolometer, flag the start and end of the quality
array as padding, and store zeros in the data array. */
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
dout = ((double *) box->pntr[0]) + xbstride*ibolo;
qout = box->qual + xbstride*ibolo;
for( itime = 0; itime < pstart; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
dout = ((double *) box->pntr[0]) + xbstride*ibolo + pstart + boxsize;;
qout = box->qual + xbstride*ibolo + pstart + boxsize;
for( itime = pend + 1; itime < ntslice; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
}
}
}
/* Work space to hold the variance for each bolometer in a box */
var = astMalloc( nbolo*sizeof(*var) );
if( *status == SAI__OK ) {
/* Index of the first time slice within the input smfData
that is included in the first box. */
tstart = pstart;
/* Loop round each noise box */
for( ibox = 0; ibox < nbox; ibox++ ) {
/* Copy the data and quality values for this box from the
input smfData into "box", leaving room for padding at
both ends of box. Note, data is bolo-ordered so we
can assume that "tstride" is 1. */
din = ((double *)(res->sdata[idx]->pntr[0])) + tstart;
dout = ((double *)(box->pntr[0])) + pstart;
qin = qua_data + tstart;
qout = box->qual + pstart;
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
memcpy( dout, din, boxsize*sizeof( *din ) );
memcpy( qout, qin, boxsize*sizeof( *qin ) );
din += bstride;
dout += xbstride;
qin += bstride;
qout += xbstride;
}
/* Measure the noise from power spectra in the box. */
smf_bolonoise( wf, box, 0, 0.5, SMF__F_WHITELO, SMF__F_WHITEHI,
0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT, var,
NULL, NULL, status );
/* Loop over time and store the variance for each sample in
the NOI model. On the last box, pick up any left over time
slices. */
if( ibox < nbox - 1 ) {
tend = tstart + boxsize - 1;
} else {
tend = pend;
}
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
if( !( qua_data[ ibolo*bstride ] & SMF__Q_BADB ) ) {
dout = model_data + ibolo*bstride + tstart;
for( itime = tstart; itime <= tend; itime++ ) {
*(dout++) = var[ ibolo ];
}
}
}
/* Update the index of the first time slice within the input
smfData that is included in the next box. */
tstart += boxsize;
}
var = astFree( var );
}
/* Report an error if the number of samples for each bolometer in
the NOI model is not 1 or "ntslice". */
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": NOI model has %d samples - should be "
"%d or 1.", status, (int) mntslice, (int) ntslice);
}
}
if( kmap ) {
/* Flag spikes in the residual after first iteration */
if( spikethresh && !(flags&SMF__DIMM_FIRSTITER) ) {
/* Now re-flag */
smf_flag_spikes( wf, res->sdata[idx], SMF__Q_MOD,
spikethresh, spikebox, &nflag, status );
msgOutiff(MSG__VERB," ", " flagged %zu new %lf-sig spikes",
status, nflag, spikethresh );
}
if( dcthresh && dcfitbox ) {
smf_fix_steps( wf, res->sdata[idx], dcthresh, dcsmooth,
dcfitbox, dcmaxsteps, dclimcorr, 1, &nflag, NULL,
NULL, status );
msgOutiff(MSG__VERB, ""," detected %zu bolos with DC steps\n",
status, nflag);
}
}
/* Now calculate contribution to chi^2. This bit takes along time
if there is a lot of data so share the work out amongst the available
worker threads. How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many bolometers to process in each worker thread. */
bolostep = nbolo/nw;
if( bolostep == 0 ) bolostep = 1;
/* Allocate job data for threads, and store the range of bolos to be
processed by each one. Ensure that the last thread picks up any
left-over bolos. */
SmfCalcModelNoiData *job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
SmfCalcModelNoiData *pdata;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->b1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->b2 = pdata->b1 + bolostep - 1;
} else {
pdata->b2 = nbolo - 1 ;
}
/* Store other values common to all jobs. */
pdata->ntslice = ntslice;
pdata->mntslice = mntslice;
pdata->qua_data = qua_data;
pdata->model_data = model_data;
pdata->res_data = res_data;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->mbstride = mbstride;
pdata->mtstride = mtstride;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_calcmodel_noi, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Accumulate the results from all the worker threads. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
dat->chisquared[chunk] += pdata->chisquared;
nchisq += pdata->nchisq;
}
/* Free the job data. */
job_data = astFree( job_data );
}
}
}
/* Free resources */
if( box ) {
box->pntr[0] = astFree( box->pntr[0] );
box->qual = astFree( box->qual );
smf_close_file( &box, status );
}
/* Normalize chisquared for this chunk */
if( (*status == SAI__OK) && (nchisq >0) ) {
dat->chisquared[chunk] /= (double) nchisq;
}
/* Clean Up */
if( kmap ) kmap = astAnnul( kmap );
}
void smf_write_bolomap( ThrWorkForce *wf, smfArray *res, smfArray *lut,
smfArray *qua, smfDIMMData *dat, dim_t msize,
const Grp *bolrootgrp, int varmapmethod,
const int *lbnd_out, const int *ubnd_out,
AstFrameSet *outfset, int *status ) {
int addtomap=0; /* Set if adding to existing map */
size_t bstride; /* Bolometer stride */
double *curmap=NULL; /* Pointer to current map being rebinned */
double *curvar=NULL; /* Pointer to variance associate with curmap */
dim_t dsize; /* Size of data arrays in containers */
size_t idx=0; /* index within subgroup */
size_t k; /* loop counter */
int *lut_data=NULL; /* Pointer to DATA component of lut */
char name[GRP__SZNAM+1]; /* Buffer for storing names */
dim_t nbolo; /* Number of bolometers */
size_t nbolomaps = 0; /* Number of bolomaps written */
char *pname=NULL; /* Poiner to name */
smf_qual_t *qua_data=NULL; /* Pointer to DATA component of qua */
double *res_data=NULL; /* Pointer to DATA component of res */
if( *status != SAI__OK ) return;
if( !res || !lut || !qua || !dat || !bolrootgrp ||
!lbnd_out || !ubnd_out || !outfset ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL inputs supplied", status );
return;
}
/* Loop over subgroup index (subarray) */
for( idx=0; idx<res->ndat; idx++ ) {
smf_qual_t *bolomask = NULL;
double *bmapweight = NULL;
double *bmapweightsq = NULL;
int *bhitsmap = NULL;
/* Pointers to everything we need */
res_data = res->sdata[idx]->pntr[0];
lut_data = lut->sdata[idx]->pntr[0];
qua_data = qua->sdata[idx]->pntr[0];
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, NULL,
&dsize, &bstride, NULL, status );
/* Make a copy of the quality at first time slice as a good
bolo mask, and then set quality to SMF__Q_BADB. Later we
will unset BADB for one bolo at a time to make individual
maps. */
bolomask = astMalloc( nbolo*sizeof(*bolomask) );
bmapweight = astMalloc( msize*sizeof(*bmapweight) );
bmapweightsq = astMalloc( msize*sizeof(*bmapweightsq) );
bhitsmap = astMalloc( msize*sizeof(*bhitsmap) );
if( *status == SAI__OK ) {
for( k=0; k<nbolo; k++ ) {
bolomask[k] = qua_data[k*bstride];
qua_data[k*bstride] = SMF__Q_BADB;
}
/* Identify good bolos in the copied mask and produce a map */
for( k=0; (k<nbolo)&&(*status==SAI__OK); k++ ) {
if( !(bolomask[k]&SMF__Q_BADB) ) {
Grp *mgrp=NULL; /* Temporary group to hold map names */
smfData *mapdata=NULL;/* smfData for new map */
char tmpname[GRP__SZNAM+1]; /* temp name buffer */
char thisbol[20]; /* name particular to this bolometer */
size_t col, row;
char subarray[10];
nbolomaps++;
/* Set the quality back to good for this single bolometer */
qua_data[k*bstride] = bolomask[k];
/* Create a name for the new map, take into account the
chunk number and subarray. Only required if we are using a single
output container. */
pname = tmpname;
grpGet( bolrootgrp, 1, 1, &pname, sizeof(tmpname), status );
one_strlcpy( name, tmpname, sizeof(name), status );
one_strlcat( name, ".", sizeof(name), status );
/* Subarray, column and row. HDS does not care about case but we
convert to upper case anyhow. */
smf_find_subarray( res->sdata[idx]->hdr, subarray, sizeof(subarray),
NULL, status );
if (*status == SAI__OK) {
size_t len = strlen(subarray);
size_t n = 0;
for (n=0; n<len; n++) {
subarray[n] = toupper(subarray[n]);
}
}
col = (k % res->sdata[idx]->dims[1])+1;
row = (k / res->sdata[idx]->dims[1])+1;
sprintf( thisbol, "%3sC%02zuR%02zu",
subarray,
col, /* x-coord */
row ); /* y-coord */
one_strlcat( name, thisbol, sizeof(name), status );
mgrp = grpNew( "bolomap", status );
grpPut1( mgrp, name, 0, status );
msgOutf( "", "*** Writing single bolo map %s", status,
name );
/* Try to open an existing extention first. Create a new map
array, and then later we'll add it to the existing
one. If it isn't there, create it. */
smf_open_file( mgrp, 1, "UPDATE", 0, &mapdata, status );
if( *status == SAI__OK ) {
/* Allocate memory for the new rebinned data */
curmap = astCalloc( msize, sizeof(*curmap) );
curvar = astCalloc( msize, sizeof(*curvar) );
addtomap = 1;
} else if( *status == DAT__NAMIN ) {
/* Create a new extension */
errAnnul( status );
smf_open_newfile ( mgrp, 1, SMF__DOUBLE, 2, lbnd_out,
ubnd_out, SMF__MAP_VAR, &mapdata, status);
/* Rebin directly into the newly mapped space */
if( *status == SAI__OK ) {
curmap = mapdata->pntr[0];
curvar = mapdata->pntr[1];
addtomap = 0;
}
}
/* Rebin the data for this single bolometer. Don't care
about variance weighting because all samples from
same detector are about the same. */
smf_rebinmap1( wf, res->sdata[idx],
dat->noi ? dat->noi[0]->sdata[idx] : NULL,
lut_data, 0, 0, 0, NULL, 0,
SMF__Q_GOOD, varmapmethod,
AST__REBININIT | AST__REBINEND,
curmap, bmapweight, bmapweightsq, bhitsmap,
curvar, msize, NULL, status );
/* If required, add this new map to the existing one */
if( addtomap ) {
size_t i;
double *oldmap=NULL;
double *oldvar=NULL;
double weight;
if( *status == SAI__OK ) {
oldmap = mapdata->pntr[0];
oldvar = mapdata->pntr[1];
for( i=0; i<msize; i++ ) {
if( oldmap[i]==VAL__BADD ) {
/* No data in this pixel in the old map, just copy */
oldmap[i] = curmap[i];
oldvar[i] = curvar[i];
} else if( curmap[i]!=VAL__BADD &&
oldvar[i]!=VAL__BADD && oldvar[i]!=0 &&
curvar[i]!=VAL__BADD && curvar[i]!=0 ) {
/* Both old and new values available */
weight = 1/oldvar[i] + 1/curvar[i];
oldmap[i] = (oldmap[i]/oldvar[i] + curmap[i]/curvar[i]) /
weight;
oldvar[i] = 1/weight;
}
}
}
/* Free up temporary arrays */
curmap = astFree( curmap );
curvar = astFree( curvar );
}
/* Write out COLNUM and ROWNUM to FITS header */
if( *status == SAI__OK ) {
AstFitsChan *fitschan=NULL;
fitschan = astFitsChan ( NULL, NULL, " " );
atlPtfti( fitschan, "COLNUM", col, "bolometer column", status);
atlPtfti( fitschan, "ROWNUM", row, "bolometer row", status );
atlPtfts( fitschan, "SUBARRAY", subarray, "Subarray identifier",
status );
kpgPtfts( mapdata->file->ndfid, fitschan, status );
if( fitschan ) fitschan = astAnnul( fitschan );
/* Set the bolo to bad quality again */
qua_data[k*bstride] = SMF__Q_BADB;
/* Write WCS */
smf_set_moving(outfset,NULL,status);
ndfPtwcs( outfset, mapdata->file->ndfid, status );
}
/* Clean up */
if( mgrp ) grpDelet( &mgrp, status );
if( mapdata ) smf_close_file( &mapdata, status );
}
}
/* Set quality back to its original state */
for( k=0; k<nbolo; k++ ) {
qua_data[k*bstride] = bolomask[k];
}
}
/* Free up memory */
bolomask = astFree( bolomask );
bmapweight = astFree( bmapweight );
bmapweightsq = astFree( bmapweightsq );
bhitsmap = astFree( bhitsmap );
}
msgOutf( "", "*** Wrote %zu bolo maps", status, nbolomaps );
}
void sc2sim_getbous
(
double angle, /* angle of pattern relative to telescope
axes in radians anticlockwise (given) */
double bouswidth, /* width of bous pattern (arcsec) */
double bousheight, /* height of bous pattern (arcsec) */
double spacing, /* distance between scans (arcsec) (given) */
double accel[2], /* telescope accelerations (arcsec) (given) */
double vmax[2], /* telescope maximum velocities (arcsec) (given) */
double samptime, /* sample interval in sec (given) */
int *bouscount, /* number of positions in pattern (returned) */
double **posptr, /* list of positions (returned) */
int *status /* global status (given and returned) */
)
{
/* Local variables */
int scancount; /* number of scan paths across sky */
double cend[2]; /* ending coordinates in arcsec */
double cstart[2]; /* starting coordinates in arcsec */
int curroff; /* index of next free slot in position list */
int j; /* loop counter */
int direction = 1; /* flag for direction of scan */
/* Check status */
if ( !StatusOkP(status) ) return;
/* Determine the number of scan paths */
scancount = bousheight / spacing;
/* Set up the beginning of the scan */
cend[0] = 0.0;
cend[1] = 0.0;
(*bouscount) = 0;
/* Get the size of the scan */
for ( j = 0; j < scancount; j++ ) {
/* Do a scan across the sky */
cstart[0] = cend[0];
cstart[1] = cend[1];
cend[0] = cend[0] + ( direction * ( bouswidth * cos ( angle ) ) );
cend[1] = cend[1] + ( direction * ( bouswidth * sin ( angle ) ) );
/* Increment the number of positions */
sc2sim_getscansegsize ( samptime, cstart, cend, accel, vmax, &curroff,
status );
(*bouscount) += curroff;
/* Move up to the next scan path */
cstart[0] = cend[0];
cstart[1] = cend[1];
cend[0] = cend[0] - ( spacing * sin ( angle ) );
cend[1] = cend[1] + ( spacing * cos ( angle ) );
/* Increment the number of positions */
sc2sim_getscansegsize ( samptime, cstart, cend, accel, vmax, &curroff,
status );
(*bouscount) += curroff;
/* Change directions */
if ( direction > 0 )
direction = -1;
else
direction = 1;
}
/* Allocate memory for the list of positions */
*posptr = astCalloc( (*bouscount)*2, sizeof(**posptr) );
curroff = 0;
cend[0] = 0.0;
cend[1] = 0.0;
direction = 1;
/* Get the scan positions */
for ( j = 0; j < scancount; j++ ) {
/* Do a scan across the sky */
cstart[0] = cend[0];
cstart[1] = cend[1];
cend[0] = cend[0] + ( direction * ( bouswidth * cos ( angle ) ) );
cend[1] = cend[1] + ( direction * ( bouswidth * sin ( angle ) ) );
/* Get positions for this scan */
sc2sim_getscanseg ( samptime, cstart, cend, accel, vmax, *bouscount,
&curroff, *posptr, status );
/* Move up to the next scan path */
cstart[0] = cend[0];
cstart[1] = cend[1];
cend[0] = cend[0] - ( spacing * sin ( angle ) );
cend[1] = cend[1] + ( spacing * cos ( angle ) );
/* Get positions for moving up */
/* Get positions for this scan */
sc2sim_getscanseg ( samptime, cstart, cend, accel, vmax, *bouscount,
&curroff, *posptr, status );
/* Change directions */
if ( direction > 0 )
direction = -1;
else
direction = 1;
}
}
void smf_flag_spikes( ThrWorkForce *wf, smfData *data, smf_qual_t mask,
double thresh, size_t box, size_t *nflagged,
int *status ){
/* Local Variables */
int i; /* Loop counter */
smfFlagSpikesData *job_data=NULL;/* Array of job data for each thread */
dim_t nbolo; /* Number of bolometers */
dim_t ntime; /* Number of time-slices */
double *dat = NULL; /* Pointer to bolo data */
smfFlagSpikesData *pdata=NULL;/* Pointer to job data */
int nw; /* Number of worker threads */
size_t bstride; /* Vector stride between bolometer samples */
size_t nflag; /* Number of samples flagged */
size_t tstride; /* Vector stride between time samples */
smf_qual_t *qua = NULL; /* Pointer to quality flags */
size_t step; /* step size for dividing up work */
int njobs=0; /* Number of jobs to be processed */
/* Check inherited status. Also return immediately if no spike flagging
is to be performed. */
if( *status != SAI__OK || thresh == 0.0 ) return;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Check we have double precision data. */
smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status );
/* Get a pointer to the quality array to use. */
qua = smf_select_qualpntr( data, NULL, status );
/* Report an error if we have no quality array. */
if( !qua && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": No valid QUALITY array was provided", status );
}
/* Get a pointer to the data array to use. Report an error if we have
no data array. */
dat = data->pntr[0];
if( !dat && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": smfData does not contain a DATA component",
status);
}
/* Check the supplied thresh value is valid. */
if( thresh <= 0 && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSetd( "THRESH", thresh );
errRep( " ", FUNC_NAME ": Can't find spikes: thresh=^THRESH, must be > 0",
status);
}
/* Check the supplied box value is valid. */
if( box <= 2 && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "BOX", box );
errRep( " ", FUNC_NAME ": Can't find spikes: box=^BOX, must be > 2",
status);
}
/* Obtain data dimensions, and the stride between adjacent elements on
each axis (bolometer and time). Use the existing data order to avoid
the cost of re-ordering. */
smf_get_dims( data, NULL, NULL, &nbolo, &ntime, NULL, &bstride, &tstride,
status );
/* Check we have room for at least 3 boxes along the time axis. */
if( 3*box > ntime && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "BOX", box );
msgSeti( "MAX", ntime/3 - 1 );
errRep( " ", FUNC_NAME ": Can't find spikes: box=^BOX is too large, "
" must be < ^MAX.", status);
}
/* Set up the job data */
if( nw > (int) nbolo ) {
step = 1;
} else {
step = nbolo/nw;
if( !step ) {
step = 1;
}
}
job_data = astCalloc( nw, sizeof(*job_data) );
for( i=0; (*status==SAI__OK)&&i<nw; i++ ) {
pdata = job_data + i;
pdata->b1 = i*step;
pdata->b2 = (i+1)*step-1;
/* if b1 is greater than the number of bolometers, we've run out of jobs... */
if( pdata->b1 >= nbolo ) {
break;
}
/* increase the jobs counter */
njobs++;
/* Ensure that the last thread picks up any left-over bolometers */
if( (i==(nw-1)) && (pdata->b1<(nbolo-1)) ) {
pdata->b2=nbolo-1;
}
pdata->ijob = -1; /* Flag job as ready to start */
pdata->box = box;
pdata->bstride = bstride;
pdata->dat = dat;
pdata->mask = mask;
pdata->thresh = thresh;
pdata->nbolo = nbolo;
pdata->ntime = ntime;
pdata->qua = qua;
pdata->tstride = tstride;
}
/* Submit jobs to find spikes in each block of bolos */
thrBeginJobContext( wf, status );
for( i=0; (*status==SAI__OK)&&i<njobs; i++ ) {
pdata = job_data + i;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfFlagSpikesPar, 0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
thrEndJobContext( wf, status );
/* Count flagged samples from all of the jobs and free resources */
nflag=0;
if( job_data ) {
for( i=0; i<njobs; i++ ) {
pdata = job_data + i;
nflag += pdata->nflag;
}
job_data = astFree( job_data );
}
/* Return the number of flagged samples, if requested */
if( nflagged ) *nflagged = nflag;
}
unsigned char *smf_get_mask( ThrWorkForce *wf, smf_modeltype mtype,
AstKeyMap *config, smfDIMMData *dat, int flags,
int *status ) {
/* Local Variables: */
AstCircle *circle; /* AST Region used to mask a circular area */
AstKeyMap *akm; /* KeyMap holding AST config values */
AstKeyMap *subkm; /* KeyMap holding model config values */
char refparam[ DAT__SZNAM ];/* Name for reference NDF parameter */
char words[100]; /* Buffer for variable message words */
const char *cval; /* The ZERO_MASK string value */
const char *modname; /* The name of the model being masked */
const char *skyrefis; /* Pointer to SkyRefIs attribute value */
dim_t i; /* Pixel index */
double *pd; /* Pointer to next element of map data */
double *predef; /* Pointer to mask defined by previous run */
double *ptr; /* Pointer to NDF Data array */
double *pv; /* Pointer to next element of map variance */
double centre[ 2 ]; /* Coords of circle centre in radians */
double meanhits; /* Mean hits in the map */
double radius[ 1 ]; /* Radius of circle in radians */
double zero_circle[ 3 ]; /* LON/LAT/Radius of circular mask */
double zero_lowhits; /* Fraction of mean hits at which to threshold */
double zero_snr; /* Higher SNR at which to threshold */
double zero_snrlo; /* Lower SNR at which to threshold */
int *ph; /* Pointer to next hits value */
int have_mask; /* Did a mask already exist on entry? */
int imask; /* Index of next mask type */
int indf1; /* Id. for supplied reference NDF */
int indf2; /* Id. for used section of reference NDF */
int isstatic; /* Are all used masks static? */
int lbnd_grid[ 2 ]; /* Lower bounds of map in GRID coords */
int mask_types[ NTYPE ]; /* Identifier for the types of mask to use */
int munion; /* Use union of supplied masks */
int nel; /* Number of mapped NDF pixels */
int nmask; /* The number of masks to be combined */
int nsource; /* No. of source pixels in final mask */
int skip; /* No. of iters for which AST is not subtracted */
int thresh; /* Absolute threshold on hits */
int ubnd_grid[ 2 ]; /* Upper bounds of map in GRID coords */
int zero_c_n; /* Number of zero circle parameters read */
int zero_mask; /* Use the reference NDF as a mask? */
int zero_niter; /* Only mask for the first "niter" iterations. */
int zero_notlast; /* Don't zero on last iteration? */
size_t ngood; /* Number good samples for stats */
smf_qual_t *pq; /* Pinter to map quality */
unsigned char **mask; /* Address of model's mask pointer */
unsigned char *newmask; /* Individual mask work space */
unsigned char *pm; /* Pointer to next returned mask pixel */
unsigned char *pn; /* Pointer to next new mask pixel */
unsigned char *result; /* Returned mask pointer */
/* Initialise returned values */
result = NULL;
/* Check inherited status. Also check that a map is being created. */
if( *status != SAI__OK || !dat || !dat->map ) return result;
/* Begin an AST context. */
astBegin;
/* Get the sub-keymap containing the configuration parameters for the
requested model. Also get a pointer to the mask array to use (there is
one for each maskable model)*/
if( mtype == SMF__COM ) {
modname = "COM";
mask = &(dat->com_mask);
} else if( mtype == SMF__AST ) {
modname = "AST";
mask = &(dat->ast_mask);
} else if( mtype == SMF__FLT ) {
modname = "FLT";
mask = &(dat->flt_mask);
} else {
modname = NULL;
mask = NULL;
*status = SAI__ERROR;
errRepf( " ", "smf_get_mask: Unsupported model type %d supplied - "
"must be COM, FLT or AST.", status, mtype );
}
subkm = NULL;
astMapGet0A( config, modname, &subkm );
/* Get the "ast.skip" value - when considering "zero_niter" and
"zero_freeze", we only count iterations for which the AST model
is subtracted (i.e. the ones following the initial "ast.skip"
iterations). */
astMapGet0A( config, "AST", &akm );
astMapGet0I( akm, "SKIP", &skip );
akm = astAnnul( akm );
/* Get the number of iterations over which the mask is to be applied. Zero
means all. Return with no mask if this number of iterations has
already been performed. */
zero_niter = 0;
astMapGet0I( subkm, "ZERO_NITER", &zero_niter );
if( zero_niter == 0 || dat->iter < zero_niter + skip ) {
/* Only return a mask if this is not the last iteration, or if ZERO_NOTLAST
is unset. */
zero_notlast = 0;
astMapGet0I( subkm, "ZERO_NOTLAST", &zero_notlast );
if( !( flags & SMF__DIMM_LASTITER ) || !zero_notlast ) {
/* Create a list of the mask types to be combined to get the final mask by
looking for non-default values for the corresponding configuration
parameters in the supplied KeyMap. Static masks (predefined, circles
or external NDFs) may be used on any iteration, but dynamic masks
(lowhits, snr) will only be avialable once the map has been determined
at the end of the first iteration. This means that when masking anything
but the AST model (which is determined after the map), the dynamic masks
cannot be used on the first iteration. Make a note if all masks being
used are static. */
isstatic = 1;
nmask = 0;
zero_lowhits = 0.0;
astMapGet0D( subkm, "ZERO_LOWHITS", &zero_lowhits );
if( zero_lowhits > 0.0 ) {
if( mtype == SMF__AST || !( flags & SMF__DIMM_FIRSTITER ) ) {
mask_types[ nmask++] = LOWHITS;
isstatic = 0;
}
} else if( zero_lowhits < 0.0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter %s.ZERO_LOWHITS (%g) - "
"it must not be negative.", status, modname, zero_lowhits );
}
if( astMapGet1D( subkm, "ZERO_CIRCLE", 3, &zero_c_n, zero_circle ) ) {
if( zero_c_n == 1 || zero_c_n == 3 ) {
mask_types[ nmask++] = CIRCLE;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad number of values (%d) for config parameter "
"%s.ZERO_CIRCLE - must be 1 or 3.", status, zero_c_n,
modname );
}
}
cval = NULL;
astMapGet0C( subkm, "ZERO_MASK", &cval );
if( cval ) {
if( !astChrMatch( cval, "REF" ) &&
!astChrMatch( cval, "MASK2" ) &&
!astChrMatch( cval, "MASK3" ) ) {
astMapGet0I( subkm, "ZERO_MASK", &zero_mask );
cval = ( zero_mask > 0 ) ? "REF" : NULL;
}
if( cval ) {
strcpy( refparam, cval );
astChrCase( NULL, refparam, 1, 0 );
mask_types[ nmask++] = REFNDF;
}
}
zero_snr = 0.0;
astMapGet0D( subkm, "ZERO_SNR", &zero_snr );
if( zero_snr > 0.0 ) {
if( mtype == SMF__AST || !( flags & SMF__DIMM_FIRSTITER ) ) {
mask_types[ nmask++] = SNR;
isstatic = 0;
}
} else if( zero_snr < 0.0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter %s.ZERO_SNR (%g) - "
"it must not be negative.", status, modname, zero_snr );
}
if( astMapHasKey( subkm, "ZERO_MASK_POINTER" ) ) {
astMapGet0P( subkm, "ZERO_MASK_POINTER", (void **) &predef );
if( predef ) mask_types[ nmask++] = PREDEFINED;
}
/* No need to create a mask if no masking was requested or possible. */
if( nmask > 0 ) {
/* Decide if we are using the union or intersection of the masks. */
astMapGet0I( subkm, "ZERO_UNION", &munion );
/* Note if a mask existed on entry. If not, create a mask now, and
initialise it to hold the mask defined by the initial sky map. */
if( *mask == NULL ) {
have_mask = 0;
if( dat->initqual ) {
*mask = astMalloc( dat->msize*sizeof( **mask ) );
if( *mask ) {
pm = *mask;
pq = dat->initqual;
for( i = 0; i < dat->msize; i++ ) {
*(pm++) = ( *(pq++) != 0 );
}
}
} else{
*mask = astCalloc( dat->msize, sizeof( **mask ) );
}
} else {
have_mask = 1;
}
/* If we are combining more than one mask, we need work space to hold
an individual mask independently of the total mask. If we are using
only one mask, then just use the main mask array. */
if( nmask > 1 ) {
newmask = astMalloc( dat->msize*sizeof( *newmask ) );
} else {
newmask = *mask;
}
/* Get the number of iterations after which the mask is to be frozen.
Zero means "never freeze the mask". */
int zero_freeze = 0;
astMapGet0I( subkm, "ZERO_FREEZE", &zero_freeze );
/* Loop round each type of mask to be used. */
for( imask = 0; imask < nmask && *status == SAI__OK; imask++ ){
/* If the mask is now frozen, we just return the existing mask. So leave the
loop. */
if( zero_freeze != 0 && dat->iter > zero_freeze + skip ) {
break;
/* Low hits masking... */
} else if( mask_types[ imask ] == LOWHITS ) {
/* Set hits pixels with 0 hits to VAL__BADI so that stats1 ignores them */
ph = dat->hitsmap;
for( i = 0; i < dat->msize; i++,ph++ ) {
if( *ph == 0 ) *ph = VAL__BADI;
}
/* Find the mean hits in the map */
smf_stats1I( dat->hitsmap, 1, dat->msize, NULL, 0, 0, &meanhits,
NULL, NULL, &ngood, status );
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: mean hits = %lf, ngood "
"= %zd", status, meanhits, ngood );
/* Create the mask */
thresh = meanhits*zero_lowhits;
ph = dat->hitsmap;
pn = newmask;
for( i = 0; i < dat->msize; i++,ph++ ) {
*(pn++) = ( *ph != VAL__BADI && *ph < thresh ) ? 1 : 0;
}
/* Report masking info. */
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: masking %s "
"model at hits = %d.", status, modname, thresh );
/* Circle masking... */
} else if( mask_types[ imask ] == CIRCLE ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the circle
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* If only one parameter supplied it is radius, assume reference
LON/LAT from the frameset to get the centre. If the SkyFrame
represents offsets from the reference position (i.e. the source is
moving), assume the circle is to be centred on the origin. */
if( zero_c_n == 1 ) {
zero_circle[ 2 ] = zero_circle[ 0 ];
skyrefis = astGetC( dat->outfset, "SkyRefIs" );
if( skyrefis && !strcmp( skyrefis, "Origin" ) ) {
zero_circle[ 0 ] = 0.0;
zero_circle[ 1 ] = 0.0;
} else {
zero_circle[ 0 ] = astGetD( dat->outfset, "SkyRef(1)" );
zero_circle[ 1 ] = astGetD( dat->outfset, "SkyRef(2)" );
}
zero_circle[ 0 ] *= AST__DR2D;
zero_circle[ 1 ] *= AST__DR2D;
}
/* The supplied bounds are for pixel coordinates... we need bounds for grid
coordinates which have an offset */
lbnd_grid[ 0 ] = 1;
lbnd_grid[ 1 ] = 1;
ubnd_grid[ 0 ] = dat->ubnd_out[ 0 ] - dat->lbnd_out[ 0 ] + 1;
ubnd_grid[ 1 ] = dat->ubnd_out[ 1 ] - dat->lbnd_out[ 1 ] + 1;
/* Coordinates & radius of the circular region converted from degrees
to radians */
centre[ 0 ] = zero_circle[ 0 ]*AST__DD2R;
centre[ 1 ] = zero_circle[ 1 ]*AST__DD2R;
radius[ 0 ] = zero_circle[ 2 ]*AST__DD2R;
/* Create the Circle, defined in the current Frame of the FrameSet (i.e.
the sky frame). */
circle = astCircle( astGetFrame( dat->outfset, AST__CURRENT), 1,
centre, radius, NULL, " " );
/* Fill the mask with zeros. */
memset( newmask, 0, sizeof( *newmask )*dat->msize );
/* Get the mapping from the sky frame (current) to the grid frame (base),
and then set the mask to 1 for all of the values outside this circle */
astMaskUB( circle, astGetMapping( dat->outfset, AST__CURRENT,
AST__BASE ),
0, 2, lbnd_grid, ubnd_grid, newmask, 1 );
/* Report masking info. */
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model will"
" be masked %s using a circle of "
"radius %g arc-secs, centred at %s=%s, %s=%s.",
status, modname, words, radius[0]*AST__DR2D*3600,
astGetC( dat->outfset, "Symbol(1)" ),
astFormat( dat->outfset, 1, centre[ 0 ] ),
astGetC( dat->outfset, "Symbol(2)" ),
astFormat( dat->outfset, 2, centre[ 1 ] ) );
}
/* Reference NDF masking... */
} else if( mask_types[ imask ] == REFNDF ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the NDF
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* Begin an NDF context. */
ndfBegin();
/* Get an identifier for the NDF using the associated ADAM parameter. */
ndfAssoc( refparam, "READ", &indf1, status );
/* Get a section from this NDF that matches the bounds of the map. */
ndfSect( indf1, 2, dat->lbnd_out, dat->ubnd_out, &indf2,
status );
/* Map the section. */
ndfMap( indf2, "DATA", "_DOUBLE", "READ", (void **) &ptr,
&nel, status );
/* Check we can use the pointer safely. */
if( *status == SAI__OK ) {
/* Find bad pixels in the NDF and set those pixels to 1 in the mask. */
pn = newmask;
for( i = 0; i < dat->msize; i++ ) {
*(pn++) = ( *(ptr++) == VAL__BADD ) ? 1 : 0;
}
/* Report masking info. */
ndfMsg( "N", indf2 );
msgSetc( "M", modname );
if( zero_niter == 0 ) {
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The ^M "
"model will be masked on each iteration "
"using the bad pixels in NDF '^N'.",
status );
} else {
msgSeti( "I", zero_niter );
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The ^M "
"model will be masked for ^I iterations "
"using the bad pixels in NDF '^N'.",
status );
}
}
/* End the NDF context. */
ndfEnd( status );
}
/* SNR masking... */
} else if( mask_types[ imask ] == SNR ) {
/* Get the lower SNR limit. */
zero_snrlo = 0.0;
astMapGet0D( subkm, "ZERO_SNRLO", &zero_snrlo );
if( zero_snrlo <= 0.0 ) {
zero_snrlo = zero_snr;
} else if( zero_snrlo > zero_snr && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter "
"%s.ZERO_SNRLO (%g) - it must not be higher "
"than %s.ZERO_SNR (%g).", status, modname,
zero_snrlo, modname, zero_snr );
}
/* If the higher and lower SNR limits are equal, just do a simple
threshold on the SNR values to get the mask. */
if( zero_snr == zero_snrlo ) {
pd = dat->map;
pv = dat->mapvar;
pn = newmask;
for( i = 0; i < dat->msize; i++,pd++,pv++ ) {
*(pn++) = ( *pd != VAL__BADD && *pv != VAL__BADD &&
*pv >= 0.0 && *pd < zero_snr*sqrt( *pv ) ) ? 1 : 0;
}
/* Report masking info. */
if( !have_mask ) {
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using an SNR limit of %g.",
status, modname, words, zero_snr );
}
/* If the higher and lower SNR limits are different, create an initial
mask by thresholding at the ZERO_SNR value, and then extend the source
areas within the mask down to an SNR limit of ZERO_SNRLO. */
} else {
smf_snrmask( wf, dat->map, dat->mapvar, dat->mdims,
zero_snr, zero_snrlo, newmask, status );
/* Report masking info. */
if( !have_mask ) {
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using an SNR limit of %g "
"extended down to %g.", status, modname,
words, zero_snr, zero_snrlo );
}
}
/* Predefined masking... */
} else if( mask_types[ imask ] == PREDEFINED ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* Find bad pixels in the predefined array and set those pixels to 1 in
the mask. */
pn = newmask;
for( i = 0; i < dat->msize; i++ ) {
*(pn++) = ( *(predef++) == VAL__BADD ) ? 1 : 0;
}
/* Report masking info. */
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using a smoothed form of "
"the final mask created with the previous map.",
status, modname, words );
}
}
/* If required, add the new mask into the returned mask. If this is the
first mask, we just copy the new mask to form the returned mask.
Otherwise, we combine it with the existing returned mask. */
if( ! have_mask || ! isstatic ) {
if( nmask > 1 ) {
if( imask == 0 ) {
memcpy( *mask, newmask, dat->msize*sizeof(*newmask));
} else {
pm = *mask;
pn = newmask;
if( munion ) {
for( i = 0; i < dat->msize; i++,pm++ ) {
if( *(pn++) == 0 ) *pm = 0;
}
} else {
for( i = 0; i < dat->msize; i++,pm++ ) {
if( *(pn++) == 1 ) *pm = 1;
}
}
}
}
}
}
/* Free the individual mask work array if it was used. */
if( nmask > 1 ) newmask = astFree( newmask );
/* Check that the mask has some source pixels (i.e. pixels that have non-bad data values -
we do not also check variance values since they are not available until the second
iteration). */
if( *status == SAI__OK ) {
nsource = 0;
pm = *mask;
pd = dat->map;
for( i = 0; i < dat->msize; i++,pd++,pv++,pm++ ) {
if( *pd != VAL__BADD && *pm == 0 ) nsource++;
}
if( nsource < 5 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", "The %s mask being used has fewer than 5 "
"source pixels.", status, modname );
if( zero_snr > 0.0 ) {
errRepf( "", "Maybe your zero_snr value (%g) is too high?",
status, zero_snr );
}
}
}
/* Return the mask pointer if all has gone well. */
if( *status == SAI__OK ) result = *mask;
}
}
}
/* End the AST context, annulling all AST Objects created in the context. */
astEnd;
/* Return the pointer to the boolean mask. */
return result;
}
void * smf_dataOrder_array( ThrWorkForce *wf, void * oldbuf, smf_dtype oldtype,
smf_dtype newtype, size_t ndata, size_t ntslice,
size_t nbolo, size_t tstr1, size_t bstr1,
size_t tstr2, size_t bstr2, int inPlace,
int freeOld, int * status ) {
size_t sznew = 0; /* Size of new data type */
void * newbuf = NULL; /* Space to do the reordering */
void * retval = NULL; /* Return value with reordered buffer */
SmfDataOrderArrayData *job_data = NULL;
SmfDataOrderArrayData *pdata;
int nw;
size_t step;
int iw;
retval = oldbuf;
if (*status != SAI__OK) return retval;
if (!retval) return retval;
/* Can't do inPlace without realloc'ing if data types don't match...
so generate bad status for now */
if( (newtype != oldtype) && inPlace ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": inPlace not supported if newtype != oldtype",
status );
return retval;
}
/* For now the only data conversion that is supported is from SMF__INTEGER
to SMF__DOUBLE */
if( (oldtype!=newtype) &&
((newtype!=SMF__DOUBLE) || (oldtype!=SMF__INTEGER)) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": type conversion only supported from integer -> double",
status );
return retval;
}
/* Special case the inPlace variant with no reordering / typecasting */
if ( inPlace && tstr1 == tstr2 && bstr1 == bstr2 && oldtype == newtype ) {
return retval;
}
/* Size of data type */
sznew = smf_dtype_sz(newtype, status);
/* Allocate buffer */
newbuf = astMalloc( ndata*sznew );
if( *status == SAI__OK ) {
/* if the input and output strides are the same, and the data types are
the same, we just memcpy */
if ( tstr1 == tstr2 && bstr1 == bstr2 && oldtype == newtype ) {
memcpy( newbuf, oldbuf, sznew*ndata );
} else {
/* We currently ignore any supplied WorkForce, since using multiple
threads slows down the re-ordered rather than speeding it up - I
presume because of contention issues. I've tried ensuring that
the output array is accessed sequenctially but that does not seem
to improve anything. Leave the mult-threaded infrastructure
here, in case a better solution is found. */
wf = NULL;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many time slices to process in each worker thread. */
step = ntslice/nw;
if( ntslice == 0 ) step = 1;
/* Allocate job data for threads, and store common values. Ensure that
the last thread picks up any left-over time slices. Store the
info required by the wrker threads, then submit jobs to the work
force. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->itime1 = iw*step;
if( iw < nw - 1 ) {
pdata->itime2 = pdata->itime1 + step - 1;
} else {
pdata->itime2 = ntslice - 1 ;
}
pdata->nbolo = nbolo;
pdata->bstr1 = bstr1;
pdata->tstr1 = tstr1;
pdata->bstr2 = bstr2;
pdata->tstr2 = tstr2;
pdata->newtype = newtype;
pdata->oldtype = oldtype;
pdata->newbuf = newbuf;
pdata->oldbuf = oldbuf;
thrAddJob( wf, 0, pdata, smf1_dataOrder_array, 0, NULL, status );
}
/* Wait for the jobs to complete. */
thrWait( wf, status );
}
job_data = astFree( job_data );
}
if( inPlace ) {
/* Copy newbuf to oldbuf */
memcpy( oldbuf, newbuf, ndata*sznew );
/* Free newbuf */
newbuf = astFree( newbuf );
retval = oldbuf;
} else {
if( freeOld ) {
/* Free oldbuf */
oldbuf = astFree( oldbuf );
}
/* Set pntr to newbuf */
retval = newbuf;
}
}
return retval;
}
void smurf_sc2fft( int *status ) {
int avpspec=0; /* Flag for doing average power spectrum */
double avpspecthresh=0; /* Threshold noise for detectors in avpspec */
Grp * basegrp = NULL; /* Basis group for output filenames */
smfArray *bbms = NULL; /* Bad bolometer masks */
smfArray *concat=NULL; /* Pointer to a smfArray */
size_t contchunk; /* Continuous chunk counter */
smfArray *darks = NULL; /* dark frames */
int ensureflat; /* Flag for flatfielding data */
Grp *fgrp = NULL; /* Filtered group, no darks */
smfArray *flatramps = NULL;/* Flatfield ramps */
AstKeyMap *heateffmap = NULL; /* Heater efficiency data */
size_t gcount=0; /* Grp index counter */
size_t i; /* Loop counter */
smfGroup *igroup=NULL; /* smfGroup corresponding to igrp */
Grp *igrp = NULL; /* Input group of files */
int inverse=0; /* If set perform inverse transform */
int isfft=0; /* Are data fft or real space? */
dim_t maxconcat=0; /* Longest continuous chunk length in samples */
size_t ncontchunks=0; /* Number continuous chunks outside iter loop */
smfData *odata=NULL; /* Pointer to output smfData to be exported */
Grp *ogrp = NULL; /* Output group of files */
size_t outsize; /* Total number of NDF names in the output group */
int polar=0; /* Flag for FFT in polar coordinates */
int power=0; /* Flag for squaring amplitude coeffs */
size_t size; /* Number of files in input group */
smfData *tempdata=NULL; /* Temporary smfData pointer */
int weightavpspec=0; /* Flag for 1/noise^2 weighting */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
int zerobad; /* Zero VAL__BADD before taking FFT? */
/* Main routine */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get input file(s) */
kpg1Rgndf( "IN", 0, 1, "", &igrp, &size, status );
/* Filter out darks */
smf_find_science( igrp, &fgrp, 1, NULL, NULL, 1, 1, SMF__NULL, &darks,
&flatramps, &heateffmap, NULL, status );
/* input group is now the filtered group so we can use that and
free the old input group */
size = grpGrpsz( fgrp, status );
grpDelet( &igrp, status);
igrp = fgrp;
fgrp = NULL;
/* We now need to combine files from the same subarray and same sequence
to form a continuous time series */
smf_grp_related( igrp, size, 1, 0, 0, NULL, NULL, &maxconcat, NULL, &igroup,
&basegrp, NULL, status );
/* Get output file(s) */
size = grpGrpsz( basegrp, status );
if( size > 0 ) {
kpg1Wgndf( "OUT", basegrp, size, size, "More output files required...",
&ogrp, &outsize, status );
} else {
msgOutif(MSG__NORM, " ", TASK_NAME ": All supplied input frames were DARK,"
" nothing to do", status );
}
/* Get group of bolometer masks and read them into a smfArray */
smf_request_mask( "BBM", &bbms, status );
/* Obtain the number of continuous chunks and subarrays */
if( *status == SAI__OK ) {
ncontchunks = igroup->chunk[igroup->ngroups-1]+1;
}
msgOutiff( MSG__NORM, "", "Found %zu continuous chunk%s", status, ncontchunks,
(ncontchunks > 1 ? "s" : "") );
/* Are we flatfielding? */
parGet0l( "FLAT", &ensureflat, status );
/* Are we doing an inverse transform? */
parGet0l( "INVERSE", &inverse, status );
/* Are we using polar coordinates instead of cartesian for the FFT? */
parGet0l( "POLAR", &polar, status );
/* Are we going to assume amplitudes are squared? */
parGet0l( "POWER", &power, status );
/* Are we going to zero bad values first? */
parGet0l( "ZEROBAD", &zerobad, status );
/* Are we calculating the average power spectrum? */
parGet0l( "AVPSPEC", &avpspec, status );
if( avpspec ) {
power = 1;
parGet0d( "AVPSPECTHRESH", &avpspecthresh, status );
parGet0l( "WEIGHTAVPSPEC", &weightavpspec, status );
}
/* If power is true, we must be in polar form */
if( power && !polar) {
msgOutif( MSG__NORM, " ", TASK_NAME
": power spectrum requested so setting POLAR=TRUE", status );
polar = 1;
}
gcount = 1;
for( contchunk=0;(*status==SAI__OK)&&contchunk<ncontchunks; contchunk++ ) {
size_t idx;
/* Concatenate this continuous chunk but forcing a raw data read.
We will need quality. */
smf_concat_smfGroup( wf, NULL, igroup, darks, NULL, flatramps, heateffmap,
contchunk, ensureflat, 1, NULL, 0, NULL, NULL, 0, 0, 0,
&concat, NULL, status );
/* Now loop over each subarray */
/* Export concatenated data for each subarray to NDF file */
for( idx=0; (*status==SAI__OK)&&idx<concat->ndat; idx++ ) {
if( concat->sdata[idx] ) {
smfData * idata = concat->sdata[idx];
int provid = NDF__NOID;
dim_t nbolo; /* Number of detectors */
dim_t ndata; /* Number of data points */
/* Apply a mask to the quality array and data array */
smf_apply_mask( idata, bbms, SMF__BBM_QUAL|SMF__BBM_DATA, 0, status );
smf_get_dims( idata, NULL, NULL, &nbolo, NULL, &ndata, NULL, NULL,
status );
/* Check for double precision data */
if( idata->dtype != SMF__DOUBLE ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": data are not double precision.", status );
}
/* Are we zeroing VAL__BADD? */
if( (*status==SAI__OK) && zerobad ) {
double *data= (double *) idata->pntr[0];
for( i=0; i<ndata; i++ ) {
if( data[i] == VAL__BADD ) {
data[i] = 0;
}
}
}
/* Check whether we need to transform the data at all */
isfft = smf_isfft(idata,NULL,NULL,NULL,NULL,NULL,status);
if( isfft && avpspec && (*status == SAI__OK) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": to calculate average power spectrum input data cannot "
"be FFT", status );
}
if( (*status == SAI__OK) && (isfft == inverse) ) {
if( avpspec ) {
/* If calculating average power spectrum do the transforms with
smf_bolonoise so that we can also measure the noise of
each detector */
double *whitenoise=NULL;
smf_qual_t *bolomask=NULL;
double mean, sig, freqlo;
size_t ngood, newgood;
whitenoise = astCalloc( nbolo, sizeof(*whitenoise) );
bolomask = astCalloc( nbolo, sizeof(*bolomask) );
freqlo = 1. / (idata->hdr->steptime * idata->hdr->nframes);
smf_bolonoise( wf, idata, 1, freqlo, SMF__F_WHITELO,
SMF__F_WHITEHI, 1, 0, whitenoise, NULL, &odata,
status );
/* Initialize quality */
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] == VAL__BADD ) {
bolomask[i] = SMF__Q_BADB;
} else {
/* smf_bolonoise returns a variance, so take sqrt */
whitenoise[i] = sqrt(whitenoise[i]);
}
}
ngood=-1;
newgood=0;
/* Iteratively cut n-sigma noisy outlier detectors */
while( ngood != newgood ) {
ngood = newgood;
smf_stats1D( whitenoise, 1, nbolo, bolomask, 1, SMF__Q_BADB,
&mean, &sig, NULL, NULL, status );
msgOutiff( MSG__DEBUG, "", TASK_NAME
": mean=%lf sig=%lf ngood=%li\n", status,
mean, sig, ngood);
newgood=0;
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] != VAL__BADD ){
if( (whitenoise[i] - mean) > avpspecthresh *sig ) {
whitenoise[i] = VAL__BADD;
bolomask[i] = SMF__Q_BADB;
} else {
newgood++;
}
}
}
}
msgOutf( "", TASK_NAME
": Calculating average power spectrum of best %li "
" bolometers.", status, newgood);
/* If using 1/noise^2 weights, calculate 1/whitenoise^2 in-place
to avoid allocating another array */
if( weightavpspec ) {
msgOutif( MSG__VERB, "", TASK_NAME ": using 1/noise^2 weights",
status );
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] && (whitenoise[i] != VAL__BADD) ) {
whitenoise[i] = 1/(whitenoise[i]*whitenoise[i]);
}
}
}
/* Calculate the average power spectrum of good detectors */
tempdata = smf_fft_avpspec( odata, bolomask, 1, SMF__Q_BADB,
weightavpspec ? whitenoise : NULL,
status );
smf_close_file( &odata, status );
whitenoise = astFree( whitenoise );
bolomask = astFree( bolomask );
odata = tempdata;
tempdata = NULL;
/* Store the number of good bolometers */
parPut0i( "NGOOD", newgood, status );
} else {
/* Otherwise do forward/inverse transforms here as needed */
/* If inverse transform convert to cartesian representation first */
if( inverse && polar ) {
smf_fft_cart2pol( wf, idata, 1, power, status );
}
/* Tranform the data */
odata = smf_fft_data( wf, idata, NULL, inverse, 0, status );
smf_convert_bad( wf, odata, status );
if( inverse ) {
/* If output is time-domain, ensure that it is ICD bolo-ordered */
smf_dataOrder( odata, 1, status );
} else if( polar ) {
/* Store FFT of data in polar form */
smf_fft_cart2pol( wf, odata, 0, power, status );
}
}
/* open a reference input file for provenance propagation */
ndgNdfas( basegrp, gcount, "READ", &provid, status );
/* Export the data to a new file */
smf_write_smfData( odata, NULL, NULL, ogrp, gcount, provid,
MSG__VERB, 0, status );
/* Free resources */
ndfAnnul( &provid, status );
smf_close_file( &odata, status );
} else {
msgOutif( MSG__NORM, " ",
"Data are already transformed. No output will be produced",
status );
}
}
/* Update index into group */
gcount++;
}
/* Close the smfArray */
smf_close_related( &concat, status );
}
/* Write out the list of output NDF names, annulling the error if a null
parameter value is supplied. */
if( *status == SAI__OK ) {
grpList( "OUTFILES", 0, 0, NULL, ogrp, status );
if( *status == PAR__NULL ) errAnnul( status );
}
/* Tidy up after ourselves: release the resources used by the grp routines */
grpDelet( &igrp, status);
grpDelet( &ogrp, status);
if (basegrp) grpDelet( &basegrp, status );
if( igroup ) smf_close_smfGroup( &igroup, status );
if( flatramps ) smf_close_related( &flatramps, status );
if (heateffmap) heateffmap = smf_free_effmap( heateffmap, status );
if (bbms) smf_close_related( &bbms, status );
ndfEnd( status );
/* Ensure that FFTW doesn't have any used memory kicking around */
fftw_cleanup();
}
void smf_clipnoise( double *clipdata, size_t ndata, int cliplog,
double cliplow, double cliphigh, size_t *nclipped,
int *status ) {
const float clips[] = {5,3,1};
const size_t nclips = sizeof(clips)/sizeof(*clips);
size_t i;
size_t nlow=0;
size_t nhigh=0;
double *work=NULL;
double median, mean, sigma;
if( *status != SAI__OK ) return;
if( cliplog ) msgOutif( MSG__DEBUG, "", FUNC_NAME
": taking log10 of the data", status );
if( (cliphigh > 0) || (cliplow > 0 ) ) {
work = astCalloc( ndata, sizeof(*work) );
/* Copy the data, or its log, into a buffer */
if( *status == SAI__OK ) {
for( i=0; i<ndata; i++ ) {
if( (clipdata[i] != VAL__BADD) && (clipdata[i] > 0) ) {
work[i] = cliplog ? log10(clipdata[i]) : clipdata[i];
} else {
work[i] = VAL__BADD;
}
}
}
/* Measure the clipped median, mean and standard deviation. We
step down from 5- to 1-sigma, using the median rather than the
mean as our central measure to ensure robustness against large
outliers. This should end up near the mode of the distribution,
although the RMS of the sample will under-estimate, by nearly a
factor of 2, the standard deviation for a Gaussian distribution
since we've clipped so much of the wings. We scale it back up
so that it would give the right answer for a Gaussian. */
smf_clipped_stats1D( work, nclips, clips, 1, ndata, NULL, 0, 0,
&mean, &sigma, &median, 1, NULL, status );
/* Assume that we do not need to clip if we can not get good
statistics */
if (*status == SMF__INSMP) {
errAnnul( status );
msgOutif(MSG__NORM, "",
"Noise clipping disabled as there are too few bolometers",
status );
goto CLEANUP;
}
sigma *= 1.85;
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": mean=%lg median=%lg standard dev=%lg",
status, mean, median, sigma );
/* Then clip the high/low outliers relative to the median */
if( *status==SAI__OK ) {
for( i=0; i<ndata; i++ ) {
if( work[i] != VAL__BADD ) {
double d = work[i] - median;
if( (cliphigh>0) &&
(d >= sigma*cliphigh) ) {
clipdata[i] = VAL__BADD;
nhigh++;
}
if( (cliplow>0) &&
(-d >= sigma*cliplow) ) {
clipdata[i] = VAL__BADD;
nlow++;
}
}
}
}
if( nhigh ) msgOutiff( MSG__VERB, "", FUNC_NAME
": clipped %zu values >= %lg",
status, nhigh,
cliplog ? pow(10,median + sigma*cliphigh) :
median + sigma*cliphigh );
if( nlow ) msgOutiff( MSG__VERB, "", FUNC_NAME
": clipped %zu values <= %lg",
status, nlow,
cliplog ? pow(10,median - sigma*cliplow) :
median - sigma*cliplow );
/* Return number of clipped values */
if( nclipped ) *nclipped = nhigh+nlow;
CLEANUP:
/* Free temporary buffer */
if( work ) work = astFree( work );
}
}
void
smf_flat_standardpow( const smfData * bolvald, double refohms,
const double resistance[],
smfData ** powrefd, smfData ** bolrefd, int * status) {
double current; /* current through heaters */
double *heatframe = NULL; /* Pointer to flatfield heater data */
double *heatframevar = NULL; /* Pointer to flatfield heatframe variance */
double * heatref = NULL; /* local copy of heater settings */
size_t i; /* loop counter */
size_t j; /* loop counter */
size_t k; /* loop counter */
size_t nheat; /* Number of input frames */
size_t numbol; /* Number of bolometers */
double *powbol = NULL; /* pointer to individual bolometer powers */
double s; /* interpolation slope factor */
double * powref = NULL; /* Data array inside powrefd */
double * bolref = NULL; /* Data array inside bolrefd */
double * bolrefvar = NULL; /* Variance inside bolrefd */
*powrefd = NULL;
*bolrefd = NULL;
if (*status != SAI__OK) return;
if ( !smf_dtype_check_fatal( bolvald, NULL, SMF__DOUBLE, status ) ) return;
if (!bolvald->da) {
*status = SAI__ERROR;
errRep( "", "Unable to proceed with flatfield calculation. Heater settings not "
"present in smfData", status);
return;
}
nheat = (bolvald->dims)[2];
numbol = (bolvald->dims)[0] * (bolvald->dims)[1];
heatref = bolvald->da->heatval;
/* Create a smfData for powref and bolref */
smf_flat_malloc( nheat, bolvald, powrefd, bolrefd, status );
if (*status == SAI__OK) {
powref = (*powrefd)->pntr[0];
bolref = (*bolrefd)->pntr[0];
bolrefvar = (*bolrefd)->pntr[1];
}
/* Get some memory -
bolometer power per heater setting */
powbol = astCalloc( nheat, sizeof(*powbol) );
/* pointers to the flatfield data and variance */
heatframe = (bolvald->pntr)[0];
heatframevar = (bolvald->pntr)[1];
/* Choose the reference heater powers to be the actual heater settings acting on
the adopted reference resistance. */
for ( j=0; j<nheat; j++ ) {
current = heatref[j] * SC2FLAT__DTOI;
powref[j] = SIMULT * current * current * refohms;
}
if (*status != SAI__OK) goto CLEANUP;
/* For each bolometer interpolate to find the measurement it would report at the
standard power input values */
for ( i=0; i<numbol; i++ )
{
/* Calculate the actual power seen by each bolometer at each heater setting */
for ( j=0; j<nheat; j++ )
{
current = heatref[j] * SC2FLAT__DTOI;
if ( resistance[i] == VAL__BADD )
{
powbol[j] = VAL__BADD;
}
else
{
powbol[j] = SIMULT * current * current * resistance[i];
}
}
/* Trap bolometers that have any bad values in their measurements */
for ( j=0; j<nheat; j++) {
if ( heatframe[i + j*numbol] == VAL__BADD) {
powbol[0] = VAL__BADD;
break;
}
}
/* Trap bolometers which are unresponsive */
if ( powbol[0] != VAL__BADD &&
(heatframe[i] == VAL__BADD || heatframe[i + (nheat-1)*numbol] == VAL__BADD ||
(fabs ( heatframe[i] - heatframe[i + (nheat-1)*numbol] ) < 1.0) ) )
{
powbol[0] = VAL__BADD;
}
for ( j=0; j<nheat; j++ )
{
if ( powbol[0] == VAL__BADD )
{
bolref[j*numbol+i] = VAL__BADD;
bolrefvar[j*numbol+i] = VAL__BADD;
}
else
{
if ( powref[j] < powbol[0] )
{
double var = VAL__BADD;
/* Standard point is below actual measured range for this bolometer,
extrapolate */
s = ( heatframe[i + 1*numbol] - heatframe[i + 0*numbol] ) /
( powbol[1] - powbol[0] );
/* calculate error in gradient using standard rules */
if (heatframevar && heatframevar[i+ 1*numbol] != VAL__BADD &&
heatframevar[i + 0*numbol] != VAL__BADD) {
var = ( heatframevar[i + 1*numbol] + heatframevar[i + 0*numbol] )
/ pow( powbol[1] - powbol[0], 2 );
}
bolref[j*numbol+i] = heatframe[i+ 0*numbol] +
s * ( powref[j] - powbol[0] );
if (var != VAL__BADD) {
bolrefvar[j*numbol+i] = heatframevar[i + 0*numbol] +
( var * pow( powref[j] - powbol[0], 2 ));
} else {
bolrefvar[j*numbol+i] = VAL__BADD;
}
}
else if ( powref[j] > powbol[nheat-1] )
{
double var = VAL__BADD;
/* Standard point is above actual measured range for this bolometer,
extrapolate */
s = ( heatframe[i + (nheat-1)*numbol] - heatframe[i + (nheat-2)*numbol] ) /
( powbol[nheat-1] - powbol[nheat-2] );
/* calculate error in gradient using standard rules */
if (heatframevar && heatframevar[i + (nheat-1)*numbol] != VAL__BADD &&
heatframevar[i + (nheat-2)*numbol] != VAL__BADD) {
var = ( heatframevar[i + (nheat-1)] + heatframevar[i + (nheat-2)*numbol] )
/ pow( powbol[nheat-1] - powbol[nheat-2], 2 );
}
bolref[j*numbol+i] = heatframe[i + (nheat-2)*numbol] +
s * ( powref[j] - powbol[nheat-2] );
if (var != VAL__BADD) {
bolrefvar[j*numbol+i] = heatframevar[i +(nheat-2)*numbol] +
( var * pow( powref[j] - powbol[nheat-2], 2 ));
} else {
bolrefvar[j*numbol+i] = VAL__BADD;
}
}
else
{
/* Standard point is within actual measured range for this bolometer,
search for the points to interpolate */
for ( k=1; k<nheat; k++ )
{
if ( powref[j] <= powbol[k] )
{
double var = VAL__BADD;
s = ( heatframe[i + k*numbol] -
heatframe[i + (k-1)*numbol] ) /
( powbol[k] - powbol[k-1] );
/* calculate error in gradient using standard rules */
if (heatframevar && heatframevar[i + k*numbol] != VAL__BADD &&
heatframevar[i +(k-1)*numbol] != VAL__BADD) {
var = ( heatframevar[i + k*numbol] + heatframevar[i + (k-1)*numbol] )
/ pow( powbol[k] - powbol[k-1], 2 );
}
bolref[j*numbol+i] = heatframe[i + (k-1)*numbol] +
s * ( powref[j] - powbol[k-1] );
if (var != VAL__BADD) {
bolrefvar[j*numbol+i] = heatframevar[i +(k-1)*numbol] +
( var * pow( powref[j] - powbol[k-1], 2 ));
} else {
bolrefvar[j*numbol+i] = VAL__BADD;
}
break;
}
}
}
}
}
}
CLEANUP:
if (powbol) powbol = astFree( powbol );
if (*status != SAI__OK) {
if (*bolrefd) smf_close_file( NULL, bolrefd, status );
if (*powrefd) smf_close_file( NULL, powrefd, status );
}
}
void smf_fit_pspec( const double *pspec, dim_t nf, size_t box, double df,
double minfreq, double whitefreq, double maxfreq,
double *a, double *b, double *w, int *status ) {
double A=VAL__BADD; /* Amplitude of 1/f component */
double B=VAL__BADD; /* Exponent of 1/f component */
int converged; /* Has white noise level calc converged? */
double fit[2]; /* fit coefficients */
size_t i; /* Loop counter */
size_t i_flo; /* Index of lowest frequency for calculating 1/f */
size_t i_whi; /* Index of high-freq. edge for white-noise level */
size_t i_wlo; /* Index of low-freq. edge for white-noise level */
size_t nbad; /* Number of outliers in white noise calc */
smf_qual_t *qual=NULL; /* Quality to go with pspec */
double sigma; /* Uncertainty in white noise level */
size_t thisnbad; /* Outliers in white noise calc, this iteration */
double white=VAL__BADD;/* White noise level */
double *x=NULL; /* Independent axis for 1/f fit */
double *y=NULL; /* Dependent axis for 1/f fit */
if (*status != SAI__OK) return;
if( (df<=0) || (nf<1) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": invalid frequency information supplied.", status );
return;
}
/* Convert frequencies to array indices (but only consider above DC) */
i_flo = smf_get_findex( minfreq, df, nf, status );
i_wlo = smf_get_findex( whitefreq, df, nf, status );
if( !maxfreq ) {
/* If maxfreq is 0 set to Nyquist */
i_whi = nf-1;
} else {
i_whi = smf_get_findex( maxfreq, df, nf, status );
}
if( i_flo+box*2 > nf ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": box too large compared to number of frequencies",
status );
return;
}
/* Create a local quality array to go with pspec */
qual = astCalloc( nf, sizeof(*qual) );
if( i_flo == 0 ) i_flo = 1;
if( (*status==SAI__OK) && ((i_flo > i_wlo) || (i_wlo > i_whi)) ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME
": must have i_maxfreq (%zu) > i_whitefreq (%zu) > "
"i_minfreq (%zu)", status, i_whi, i_wlo, i_flo );
}
/* Calculate the white noise level */
converged = 0;
nbad = 0;
while( (!converged) && (*status==SAI__OK) ) {
double thresh;
smf_stats1D( pspec+i_wlo, 1, i_whi-i_wlo+1, qual, 1, SMF__Q_SPIKE, &white,
&sigma, NULL, NULL, status );
if( *status==SAI__OK ) {
thresh = white + SMF__FPSPEC_WHITESNR*sigma;
thisnbad = 0;
for( i=i_wlo; i<=i_whi; i++ ) {
if( pspec[i] > thresh ) {
qual[i] = SMF__Q_SPIKE;
thisnbad++;
}
}
if( thisnbad==nbad ) {
converged = 1;
}
nbad = thisnbad;
}
}
/* Now identify and fit the 1/f part of the power spectrum. We use a
rolling box and increase the frequency of its centre until the fraction
of samples below some threshold above the white level is exceeded.
We then fit a straight line to the logarithm of the x- and y-axes. */
if( *status == SAI__OK ) {
size_t nfit;
size_t ngood = 0;
size_t nused;
double thresh = white + SMF__FPSPEC_KNEESNR*sigma;
/* Initialize ngood -- skip the DC term in the FFT */
for( i=i_flo; i<(i_flo+box); i++ ) {
if( pspec[i] > thresh ) {
ngood++;
}
}
/* Continuing from next element, and go until we hit the end or we reach
the threshold number of samples below the threshold SNR above
the white noise level. */
for( i=i_flo+1; (i<(nf-(box-1))) && (ngood >= SMF__FPSPEC_KNEETHRESH*box);
i++ ) {
/* If the previous first element was good remove it */
if( pspec[i-1] >= thresh ) {
ngood--;
}
/* If the new last element is good add it */
if( pspec[i+box-1] >= thresh ) {
ngood++;
}
}
/* We will fit the power-law from elements i_flo to nfit-1. We then
evaluate the fitted power law as
y = A * x^B
where x is the index in the frequency array
A = exp( fit[0] )
B = fit[1] */
nfit = i+box/2-i_flo;
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": i_flow=%zu nfit=%zu i_wlo=%zu i_whi=%zu\n", status,
i_flo, nfit-1, i_wlo, i_whi);
/* If we've entered the white-noise band in order to fit the 1/f
noise give up */
if( i >= i_wlo ) {
*status = SMF__BADFIT;
errRep( "", FUNC_NAME
": unable to fit 1/f spectrum with provided frequency ranges",
status);
goto CLEANUP;
}
/* Now fit a straight line to the log of the two axes */
x = astMalloc( (nfit-1)*sizeof(*x) );
y = astMalloc( (nfit-1)*sizeof(*y) );
if( *status == SAI__OK ) {
for( i=0; i<(nfit-1); i++ ) {
x[i] = log((i+i_flo)*df);
y[i] = log(pspec[i+i_flo]);
}
}
smf_fit_poly1d( 1, nfit-1, 0, x, y, NULL, NULL, fit, NULL, NULL, &nused,
status );
if( *status == SAI__OK ) {
/* if the exponent is positive the fit is either garbage, or
else there just isn't very much low frequency noise compared
to the white noise level. Set B to 0 but generate a warning
message */
if( fit[1] >= 0 ) {
*status = SMF__BADFIT;
errRep( "", FUNC_NAME
": fit to 1/f component encountered rising spectrum!",
status);
goto CLEANUP;
} else {
B = fit[1];
}
A = (exp(fit[0]));
}
}
/* Return fit values */
if( a ) *a = A;
if( b ) *b = B;
if( w ) *w = white;
CLEANUP:
qual = astFree( qual );
x = astFree( x );
y = astFree( y );
}
