void smf_calc_mapcoord( ThrWorkForce *wf, AstKeyMap *config, smfData *data,
AstFrameSet *outfset, int moving, int *lbnd_out,
int *ubnd_out, fts2Port fts_port, int flags,
int *status ) {
/* Local Variables */
AstSkyFrame *abskyfrm = NULL;/* Output SkyFrame (always absolute) */
AstMapping *bolo2map=NULL; /* Combined mapping bolo->map coordinates */
int bndndf=NDF__NOID; /* NDF identifier for map bounds */
void *data_pntr[1]; /* Array of pointers to mapped arrays in ndf */
int *data_index; /* Mapped DATA_ARRAY part of NDF */
int docalc=1; /* If set calculate the LUT */
int doextension=0; /* Try to write LUT to MAPCOORD extension */
smfFile *file=NULL; /* smfFile pointer */
AstObject *fstemp = NULL; /* AstObject version of outfset */
int ii; /* loop counter */
int indf_lat = NDF__NOID; /* Identifier for NDF to receive lat values */
int indf_lon = NDF__NOID; /* Identifier for NDF to receive lon values */
smfCalcMapcoordData *job_data=NULL; /* Array of job */
int lbnd[1]; /* Pixel bounds for 1d pointing array */
int lbnd_old[2]; /* Pixel bounds for existing LUT */
int lbnd_temp[1]; /* Bounds for bounds NDF component */
int lutndf=NDF__NOID; /* NDF identifier for coordinates */
AstMapping *map2sky_old=NULL;/* Existing mapping map->celestial coord. */
HDSLoc *mapcoordloc=NULL; /* HDS locator to the MAPCOORD extension */
int nw; /* Number of worker threads */
AstFrameSet *oldfset=NULL; /* Pointer to existing WCS info */
AstSkyFrame *oskyfrm = NULL; /* SkyFrame from the output WCS Frameset */
smfCalcMapcoordData *pdata=NULL; /* Pointer to job data */
double *lat_ptr = NULL; /* Pointer to array to receive lat values */
double *lon_ptr = NULL; /* Pointer to array to receive lon values */
int ubnd[1]; /* Pixel bounds for 1d pointing array */
int ubnd_old[2]; /* Pixel bounds for existing LUT */
int ubnd_temp[1]; /* Bounds for bounds NDF component */
int *lut = NULL; /* The lookup table */
dim_t nbolo=0; /* Number of bolometers */
dim_t ntslice=0; /* Number of time slices */
int nmap; /* Number of mapped elements */
AstMapping *sky2map=NULL; /* Mapping celestial->map coordinates */
size_t step; /* step size for dividing up work */
AstCmpMap *testcmpmap=NULL; /* Combined forward/inverse mapping */
AstMapping *testsimpmap=NULL;/* Simplified testcmpmap */
double *theta = NULL; /* Scan direction at each time slice */
int tstep; /* Time slices between full Mapping calculations */
int exportlonlat; /* Dump longitude and latitude values? */
/* Main routine */
if (*status != SAI__OK) return;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Initialize bounds to avoid compiler warnings */
lbnd_old[0] = 0;
lbnd_old[1] = 0;
ubnd_old[0] = 0;
ubnd_old[1] = 0;
/* Check for pre-existing LUT and de-allocate it. This will only waste
time if the MAPCOORD extension is found to be valid and it has
to be re-loaded from disk. */
smf_close_mapcoord( data, status );
/* Assert ICD data order */
smf_dataOrder( data, 1, status );
/* Get the data dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, NULL, NULL, status );
/* If SMF__NOCREATE_FILE is not set, and file associated with an NDF,
map a new MAPCOORD extension (or verify an existing one) */
if( !(flags & SMF__NOCREATE_FILE) && data->file ) {
doextension = 1;
} else {
doextension = 0;
docalc = 1;
}
/* Create / check for existing MAPCOORD extension */
if( doextension ) {
file = data->file;
/* Check type of file before proceeding */
if( file->isSc2store ) {
*status = SAI__ERROR;
errRep(FUNC_NAME,
"File was opened by sc2store library (raw data?)",
status);
}
if( !file->isTstream ) {
*status = SAI__ERROR;
errRep(FUNC_NAME, "File does not contain time stream data",status);
}
/* Get HDS locator to the MAPCOORD extension */
mapcoordloc = smf_get_xloc( data, "MAPCOORD", "MAP_PROJECTION", "UPDATE",
0, 0, status );
/* Obtain NDF identifier/placeholder for LUT in MAPCOORD extension*/
lbnd[0] = 0;
ubnd[0] = nbolo*ntslice-1;
lutndf = smf_get_ndfid( mapcoordloc, "LUT", "UPDATE", "UNKNOWN",
"_INTEGER", 1, lbnd, ubnd, status );
if( *status == SAI__OK ) {
/* store the NDF identifier */
file->mapcoordid = lutndf;
/* Create sky to output grid mapping using the base coordinates to
get the coordinates of the tangent point if it hasn't been done
yet. */
sky2map = astGetMapping( outfset, AST__CURRENT, AST__BASE );
}
/* Before mapping the LUT, first check for existing WCS information
and LBND/UBND for the output map. If they are already correct don't
bother re-calculating the LUT! */
if( *status == SAI__OK ) {
/* Try reading in the WCS information */
kpg1Wread( mapcoordloc, "WCS", &fstemp, status );
oldfset = (AstFrameSet*)fstemp;
if( *status == SAI__OK ) {
/* Check that the old and new mappings are the same by
checking that combining one with the inverse of the other
reduces to a UnitMap. */
map2sky_old = astGetMapping( oldfset, AST__BASE, AST__CURRENT );
testcmpmap = astCmpMap( map2sky_old, sky2map, 1, " " );
testsimpmap = astSimplify( testcmpmap );
if( astIsAUnitMap( testsimpmap ) ) {
/* The mappings are the same, now just check the pixel
bounds in the output map */
lbnd_temp[0] = 1;
ubnd_temp[0] = 2;
bndndf = smf_get_ndfid( mapcoordloc, "LBND", "READ", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp,
status );
if( *status == SAI__OK ) {
ndfMap( bndndf, "DATA", "_INTEGER", "READ", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
lbnd_old[0] = data_index[0];
lbnd_old[1] = data_index[1];
}
ndfAnnul( &bndndf, status );
}
bndndf = smf_get_ndfid( mapcoordloc, "UBND", "READ", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp,
status );
if( *status == SAI__OK ) {
ndfMap( bndndf, "DATA", "_INTEGER", "READ", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
ubnd_old[0] = data_index[0];
ubnd_old[1] = data_index[1];
}
ndfAnnul( &bndndf, status );
}
if( *status == SAI__OK ) {
/* If we get this far finally do the bounds check! */
if( (lbnd_old[0] == lbnd_out[0]) &&
(lbnd_old[1] == lbnd_out[1]) &&
(ubnd_old[0] == ubnd_out[0]) &&
(ubnd_old[1] == ubnd_out[1]) ) {
docalc = 0; /* We don't have to re-calculate the LUT */
msgOutif(MSG__VERB," ",FUNC_NAME ": Existing LUT OK",
status);
}
}
}
/* Bad status / AST errors at this point due to problems with
MAPCOORD. Annul and continue calculating new MAPCOORD extension. */
astClearStatus;
errAnnul(status);
} else {
/* Bad status due to non-existence of MAPCOORD. Annul and continue */
errAnnul(status);
}
}
}
/* If we need to calculate the LUT do it here */
if( docalc && (*status == SAI__OK) ) {
msgOutif(MSG__VERB," ", FUNC_NAME ": Calculate new LUT",
status);
/* Get the increment in time slices between full Mapping calculations.
The Mapping for intermediate time slices will be approximated. */
dim_t dimval;
smf_get_nsamp( config, "TSTEP", data, &dimval, status );
tstep = dimval;
/* Get space for the LUT */
if( doextension ) {
/* Map the LUT array */
ndfMap( lutndf, "DATA", "_INTEGER", "WRITE", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
lut = data_index;
} else {
errRep( FUNC_NAME, "Unable to map LUT in MAPCOORD extension",
status);
}
} else {
/* alloc the LUT and THETA arrays */
lut = astMalloc( (nbolo*ntslice)*sizeof(*(data->lut)) );
theta = astMalloc( ntslice*sizeof(*(data->theta)) );
}
/* Retrieve the sky2map mapping from the output frameset (actually
map2sky) */
oskyfrm = astGetFrame( outfset, AST__CURRENT );
sky2map = astGetMapping( outfset, AST__BASE, AST__CURRENT );
/* If the longitude and latitude is being dumped, create new NDFs to
hold them, and map them. */
if( config ) {
astMapGet0I( config, "EXPORTLONLAT", &exportlonlat );
if( exportlonlat ) {
lon_ptr = smf1_calc_mapcoord1( data, nbolo, ntslice, oskyfrm,
&indf_lon, 1, status );
lat_ptr = smf1_calc_mapcoord1( data, nbolo, ntslice, oskyfrm,
&indf_lat, 2, status );
}
}
/* Invert the mapping to get Output SKY to output map coordinates */
astInvert( sky2map );
/* Create a SkyFrame in absolute coordinates */
abskyfrm = astCopy( oskyfrm );
astClear( abskyfrm, "SkyRefIs" );
astClear( abskyfrm, "SkyRef(1)" );
astClear( abskyfrm, "SkyRef(2)" );
if( *status == SAI__OK ) {
/* --- Begin parellelized portion ------------------------------------ */
/* Start a new job context. Each call to thrWait within this
context will wait until all jobs created within the context have
completed. Jobs created in higher contexts are ignored by thrWait. */
thrBeginJobContext( wf, status );
/* Allocate job data for threads */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
/* Set up job data, and start calculating pointing for blocks of
time slices in different threads */
if( nw > (int) ntslice ) {
step = 1;
} else {
step = ntslice/nw;
}
for( ii=0; (*status==SAI__OK)&&(ii<nw); ii++ ) {
pdata = job_data + ii;
/* Blocks of time slices */
pdata->t1 = ii*step;
pdata->t2 = (ii+1)*step-1;
/* Ensure that the last thread picks up any left-over tslices */
if( (ii==(nw-1)) && (pdata->t1<(ntslice-1)) ) {
pdata->t2=ntslice-1;
}
pdata->ijob = -1;
pdata->lut = lut;
pdata->theta = theta;
pdata->lbnd_out = lbnd_out;
pdata->moving = moving;
pdata->ubnd_out = ubnd_out;
pdata->tstep = tstep;
pdata->lat_ptr = lat_ptr;
pdata->lon_ptr = lon_ptr;
pdata->fts_port = fts_port;
/* Make deep copies of AST objects and unlock them so that each
thread can then lock them for their own exclusive use */
pdata->abskyfrm = astCopy( abskyfrm );
astUnlock( pdata->abskyfrm, 1 );
pdata->sky2map = astCopy( sky2map );
astUnlock( pdata->sky2map, 1 );
/* Similarly, make a copy of the smfData, including only the header
information which each thread will need in order to make calls to
smf_rebin_totmap */
pdata->data = smf_deepcopy_smfData( data, 0, SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS |
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY, 0, 0,
status );
smf_lock_data( pdata->data, 0, status );
}
for( ii=0; ii<nw; ii++ ) {
/* Submit the job */
pdata = job_data + ii;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfCalcMapcoordPar, 0, NULL, status );
}
/* Wait until all of the jobs submitted within the current job
context have completed */
thrWait( wf, status );
}
/* End the current job context. */
thrEndJobContext( wf, status );
/* --- End parellelized portion -------------------------------------- */
/* Set the lut pointer in data to the buffer */
data->lut = lut;
data->theta = theta;
/* Write the WCS for the projection to the extension */
if( doextension ) {
kpg1Wwrt( (AstObject*)outfset, "WCS", mapcoordloc, status );
/* Write the pixel bounds for the map to the extension */
lbnd_temp[0] = 1; /* Don't get confused! Bounds for NDF that will */
ubnd_temp[0] = 2; /* contain the bounds for the output 2d map! */
bndndf = smf_get_ndfid( mapcoordloc, "LBND", "UPDATE", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp, status );
ndfMap( bndndf, "DATA", "_INTEGER", "WRITE", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
data_index[0] = lbnd_out[0];
data_index[1] = lbnd_out[1];
} else {
errRep( FUNC_NAME, "Unable to map LBND in MAPCOORD extension",
status);
}
ndfAnnul( &bndndf, status );
bndndf = smf_get_ndfid( mapcoordloc, "UBND", "UPDATE", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp, status );
ndfMap( bndndf, "DATA", "_INTEGER", "WRITE", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
data_index[0] = ubnd_out[0];
data_index[1] = ubnd_out[1];
} else {
errRep( FUNC_NAME, "Unable to map UBND in MAPCOORD extension",
status);
}
ndfAnnul( &bndndf, status );
}
}
}
/* Clean Up */
if( testsimpmap ) testsimpmap = astAnnul( testsimpmap );
if( testcmpmap ) testcmpmap = astAnnul( testcmpmap );
if( map2sky_old ) map2sky_old = astAnnul( map2sky_old );
if( oldfset ) oldfset = astAnnul( oldfset );
if (sky2map) sky2map = astAnnul( sky2map );
if (bolo2map) bolo2map = astAnnul( bolo2map );
if( abskyfrm ) abskyfrm = astAnnul( abskyfrm );
if( oskyfrm ) oskyfrm = astAnnul( oskyfrm );
if( mapcoordloc ) datAnnul( &mapcoordloc, status );
if( indf_lat != NDF__NOID ) ndfAnnul( &indf_lat, status );
if( indf_lon != NDF__NOID ) ndfAnnul( &indf_lon, status );
/* If we get this far, docalc=0, and status is OK, there must be
a good LUT in there already. Map it so that it is accessible to
the caller; "UPDATE" so that the caller can modify it if desired. */
if( (*status == SAI__OK) && (docalc == 0) ) {
smf_open_mapcoord( data, "UPDATE", status );
}
/* Clean up job data */
if( job_data ) {
for( ii=0; (*status==SAI__OK)&&(ii<nw); ii++ ) {
pdata = job_data + ii;
if( pdata->data ) {
smf_lock_data( pdata->data, 1, status );
smf_close_file( &(pdata->data), status );
}
astLock( pdata->abskyfrm, 0 );
pdata->abskyfrm = astAnnul( pdata->abskyfrm );
astLock( pdata->sky2map, 0 );
pdata->sky2map = astAnnul( pdata->sky2map );
}
job_data = astFree( job_data );
}
}
void smf_fit_qui( ThrWorkForce *wf, smfData *idata, smfData **odataq,
smfData **odatau, smfData **odatai, dim_t box, int ipolcrd,
int pasign, double paoff, double angrot, int north,
int *status ){
/* Local Variables: */
AstFrameSet *wcs; /* WCS FrameSet for current time slice */
JCMTState *instate=NULL; /* Pointer to input JCMTState */
JCMTState *outstate=NULL;/* Pointer to output JCMTState */
const char *usesys; /* Tracking system */
dim_t *box_starts; /* Array holding time slice at start of each box */
dim_t box_size; /* First time slice in box */
dim_t intslice; /* ntslice of idata */
dim_t istart; /* Input time index at start of fitting box */
dim_t itime; /* Time slice index */
dim_t nbolo; /* No. of bolometers */
dim_t ncol; /* No. of columns of bolometers in the array */
dim_t ntime; /* Time slices to check */
dim_t ondata; /* ndata of odata */
dim_t ontslice; /* ntslice of odata */
double scale; /* how much longer new samples are */
int bstep; /* Bolometer step between threads */
int iworker; /* Index of a worker thread */
int nworker; /* No. of worker threads */
size_t i; /* loop counter */
smfData *indksquid=NULL; /* Pointer to input dksquid data */
smfFitQUIJobData *job_data = NULL; /* Pointer to all job data */
smfFitQUIJobData *pdata = NULL;/* Pointer to next job data */
smfHead *hdr; /* Pointer to data header this time slice */
smf_qual_t *qua; /* Input quality pointer */
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Check supplied arguments. */
if( !idata || !odataq || !odatau ) {
*status = SAI__ERROR;
errRep( "", "smf_fit_qui: NULL inputs supplied", status );
return;
}
if( idata->ndims != 3 ) {
*status = SAI__ERROR;
errRep( "", "smf_fit_qui: idata is not 3-dimensional", status );
return;
}
/* Ensure the supplied smfData is time-ordered. So "bstride" is 1 and "tstride"
is nbolo. */
smf_dataOrder( wf, idata, 1, status );
/* Dimensions of input. */
smf_get_dims( idata, NULL, &ncol, &nbolo, &intslice, NULL, NULL, NULL,
status );
/* Store a pointer to the quality array for the input smfData. */
qua = smf_select_qualpntr( idata, NULL, status );;
/* Go through the first thousand POL_ANG values to see if they are in
units of radians (new data) or arbitrary encoder units (old data).
They are assumed to be in radians if no POL_ANG value is larger than
20. This function can only handle new data. */
hdr = idata->hdr;
instate = hdr->allState;
ntime = ( intslice > 1000 ) ? 1000 : intslice;
for( itime = 0; itime < ntime; itime++,instate++ ) {
if( instate->pol_ang > 20 ) {
*status = SAI__ERROR;
errRep( " "," POL2 data contains POL_ANG values in encoder "
"units - connot fit to such old data.", status );
break;
}
}
/* Find the input time slice at which each fitting box starts, and the
length of the output time axis (in time-slices). */
smf1_find_boxes( intslice, hdr->allState, box, &ontslice, &box_starts,
status );
/* Time axis scaling factor. */
scale = (double) intslice / (double) ontslice;
/* First copy everything from input to output except for the data that needs
to be downsampled */
/* We want to copy everything in the smfHead except for allState. So we
make a copy of the allState pointer, and then set it to NULL in the
header before the copy */
if( idata->hdr ) {
instate = idata->hdr->allState;
idata->hdr->allState = NULL;
}
/* Similarly, we want everything in the smfDa except for the dksquid. */
if( idata->da ) {
indksquid = idata->da->dksquid;
idata->da->dksquid = NULL;
}
/* Create copies, storing them in the supplied output smfData
structures. Omit the header for U and I, as we will be copying the Q
header into them. */
*odataq = smf_deepcopy_smfData( wf, idata, 0, SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE | SMF__NOCREATE_QUALITY,
0, 0, status );
*odatau = smf_deepcopy_smfData( wf, idata, 0, SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE | SMF__NOCREATE_QUALITY |
SMF__NOCREATE_HEAD, 0, 0, status );
if( odatai ) {
*odatai = smf_deepcopy_smfData( wf, idata, 0, SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE | SMF__NOCREATE_QUALITY |
SMF__NOCREATE_HEAD, 0, 0, status );
}
/* Restore values in idata now that we're done */
if( instate ) idata->hdr->allState = instate;
if( indksquid ) idata->da->dksquid = indksquid;
/* Store the required length for the output time axis. The time axis is
axis two because the data is time-ordered. */
(*odataq)->dims[ 2 ] = ontslice;
(*odatau)->dims[ 2 ] = ontslice;
if( odatai) (*odatai)->dims[ 2 ] = ontslice;
/* Get output dimensions - assumed to be the same for all three outputs. */
ondata = ontslice*idata->dims[0]*idata->dims[1];
/* Allocate the data arrays for the outputs. */
(*odataq)->pntr[0] = astCalloc( ondata, sizeof(double) );
(*odatau)->pntr[0] = astCalloc( ondata, sizeof(double) );
if( odatai ) (*odatai)->pntr[0] = astCalloc( ondata, sizeof(double) );
/* Allocate arrays for the output variances. */
(*odataq)->pntr[1] = astCalloc( ondata, sizeof(double) );
(*odatau)->pntr[1] = astCalloc( ondata, sizeof(double) );
if( odatai ) (*odatai)->pntr[1] = astCalloc( ondata, sizeof(double) );
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
if( *status == SAI__OK ) {
/* Determine which bolometers are to be processed by which threads. */
bstep = nbolo/nworker;
if( bstep < 1 ) bstep = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->b1 = iworker*bstep;
pdata->b2 = pdata->b1 + bstep - 1;
}
/* Ensure that the last thread picks up any left-over bolometers */
pdata->b2 = nbolo - 1;
/* Loop round all output time slices. */
for( itime = 0; itime < ontslice; itime++ ) {
/* Get the index of the first input time slice that contributes to the
current output time slice. */
istart = box_starts[ itime ];
/* Get the number of input time slices that contribute to the output time
slice. */
box_size = box_starts[ itime + 1 ] - istart;
/* If we are using north as the reference direction, get the WCS FrameSet
for the input time slice that is at the middle of the output time
slice, and set its current Frame to the tracking frame. */
if( north ) {
smf_tslice_ast( idata, istart + box_size/2, 1, NO_FTS, status );
wcs = idata->hdr->wcs;
usesys = sc2ast_convert_system( (idata->hdr->allState)[0].tcs_tr_sys,
status );
astSetC( wcs, "System", usesys );
} else {
wcs = NULL;
}
/* Now enter the parellel code in which each thread calculates the values
for a range of bolometers at the current output slice. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->dat = ((double *) idata->pntr[0] ) + istart*nbolo;
pdata->qua = qua + istart*nbolo;
pdata->allstates = hdr->allState + istart;
pdata->ipi = odatai ? ( (double*) (*odatai)->pntr[0] ) + itime*nbolo : NULL;
pdata->ipq = ( (double*) (*odataq)->pntr[0] ) + itime*nbolo;
pdata->ipu = ( (double*) (*odatau)->pntr[0] ) + itime*nbolo;
pdata->ipv = ( (double*) (*odataq)->pntr[1] ) + itime*nbolo;
pdata->nbolo = nbolo;
pdata->ncol = ncol;
pdata->box_size = box_size;
pdata->ipolcrd = ipolcrd;
pdata->pasign = pasign ? +1: -1;
pdata->paoff = paoff;
pdata->angrot = angrot;
if( wcs ) {
pdata->wcs = astCopy( wcs );
astUnlock( pdata->wcs, 1 );
} else {
pdata->wcs = NULL;
}
/* Pass the job to the workforce for execution. */
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_fit_qui_job, 0, NULL,
status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
/* Lock and annul the AST objects used by each thread. */
if( wcs ) {
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
astLock( pdata->wcs, 0 );
pdata->wcs = astAnnul( pdata->wcs );
}
}
}
/* Down-sample the smfHead -------------------------------------------------*/
smfHead *hdr = (*odataq)->hdr;
hdr->curframe = (dim_t) (((double) hdr->curframe + 0.5) / scale);
hdr->nframes = ontslice;
hdr->steptime *= scale;
strcpy( hdr->dlabel, "Q" );
strncpy( hdr->title, "POL-2 Stokes parameter Q", SMF__CHARLABEL );
/* Down-sample all the JCMTState values using nearest neighbours */
instate = idata->hdr->allState;
if( instate ) {
hdr->allState = astCalloc( ontslice, sizeof(*instate) );
outstate = hdr->allState;
if( *status == SAI__OK ) {
size_t frame; /* index of nearest neighbour JCMTState */
for( i=0; i<ontslice; i++ ) {
frame = (size_t) round(((double) i + 0.5)*scale);
memcpy( outstate + i, instate + frame, sizeof(*instate) );
}
/* Then go back and properly down-sample the more important fast-changing
fields like pointing. Note that since there are approximate values there
already we need to explicitly re-initialize to 0. */
RESAMPSTATE(instate, outstate, rts_end, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, smu_az_jig_x, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, smu_az_jig_y, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, smu_az_chop_x, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, smu_az_chop_y, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, smu_tr_jig_x, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, smu_tr_jig_y, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, smu_tr_chop_x, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, smu_tr_chop_y, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, tcs_tai, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, tcs_airmass, intslice, ontslice, 0);
/* Second coordinates (Dec, El etc) can not wrap 0 to 360 so we do not need
to test for those cases */
RESAMPSTATE(instate, outstate, tcs_az_ang, intslice, ontslice, 1);
RESAMPSTATE(instate, outstate, tcs_az_ac1, intslice, ontslice, 1);
RESAMPSTATE(instate, outstate, tcs_az_ac2, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, tcs_az_dc1, intslice, ontslice, 1);
RESAMPSTATE(instate, outstate, tcs_az_dc2, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, tcs_az_bc1, intslice, ontslice, 1);
RESAMPSTATE(instate, outstate, tcs_az_bc2, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, tcs_tr_ang, intslice, ontslice, 1);
RESAMPSTATE(instate, outstate, tcs_tr_ac1, intslice, ontslice, 1);
RESAMPSTATE(instate, outstate, tcs_tr_ac2, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, tcs_tr_dc1, intslice, ontslice, 1);
RESAMPSTATE(instate, outstate, tcs_tr_dc2, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, tcs_tr_bc1, intslice, ontslice, 1);
RESAMPSTATE(instate, outstate, tcs_tr_bc2, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, tcs_en_dc1, intslice, ontslice, 1);
RESAMPSTATE(instate, outstate, tcs_en_dc2, intslice, ontslice, 0);
RESAMPSTATE(instate, outstate, tcs_dm_abs, intslice, ontslice, 1);
RESAMPSTATE(instate, outstate, tcs_dm_rel, intslice, ontslice, 0);
/* Wait for all the above smf_downsamp1 jobs to finish. */
thrWait( wf, status );
}
}
/* Add a keyword to the Q header indicating the polarimetric reference
direction. */
smf_fits_updateL( (*odataq)->hdr, "POLNORTH", north,
north ? "Pol ref dir is tracking north" :
"Pol ref dir is focal plane Y", status );
/* Copy the Q header to the other outputs. */
hdr = smf_deepcopy_smfHead( (*odataq)->hdr, status );
(*odatau)->hdr = hdr;
if( *status == SAI__OK ) {
strcpy( hdr->dlabel, "U" );
strncpy( hdr->title, "POL-2 Stokes parameter U", SMF__CHARLABEL );
}
if( odatai ) {
hdr = smf_deepcopy_smfHead( (*odataq)->hdr, status );
(*odatai)->hdr = hdr;
if( *status == SAI__OK ) {
strcpy( hdr->dlabel, "I" );
strncpy( hdr->title, "POL-2 Stokes parameter I", SMF__CHARLABEL );
}
}
}
/* Copy the variances from the Q smfData into the U and (and I) smfData. */
if( *odataq && *status == SAI__OK ) {
if( *odatau ) {
memcpy( (*odatau)->pntr[1], (*odataq)->pntr[1], ondata*sizeof(double));
}
if( odatai && *odatai ) {
memcpy( (*odatai)->pntr[1], (*odataq)->pntr[1], ondata*sizeof(double));
}
}
/* Ensure all smfDatas are time-ordered. */
smf_dataOrder( wf, idata, 1, status );
if( odatai && *odatai ) smf_dataOrder( wf, *odatai, 1, status );
if( *odataq ) smf_dataOrder( wf, *odataq, 1, status );
if( *odatau ) smf_dataOrder( wf, *odatau, 1, status );
/* Free resources. */
job_data = astFree( job_data );
box_starts = astFree( box_starts );
}
void smf_calcmodel_noi( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel, int flags,
int *status) {
/* Local Variables */
dim_t bolostep; /* Number of bolos per thread */
dim_t boxsize; /* No. of time slices in each noise box */
smfData *box = NULL; /* SmfData holding one box of input data */
size_t bstride; /* bolometer stride */
int calcfirst=0; /* Were bolo noises already measured? */
int dclimcorr; /* Min number of correlated steps */
int dcmaxsteps; /* Maximum allowed number of dc jumps */
dim_t dcfitbox; /* Width of box for DC step detection */
double dcthresh; /* Threshold for DC step detection */
dim_t dcsmooth; /* Width of median filter in DC step detection*/
double *din; /* Pointer to next input value */
double *dout; /* Pointer to next output value */
int fillgaps; /* If set perform gap filling */
dim_t i; /* Loop counter */
dim_t ibolo; /* Bolometer index */
int ibox; /* Index of current noise box */
dim_t itime; /* Time slice index */
dim_t idx=0; /* Index within subgroup */
JCMTState *instate=NULL; /* Pointer to input JCMTState */
int iw; /* Thread index */
dim_t j; /* Loop counter */
AstKeyMap *kmap=NULL; /* Local keymap */
size_t mbstride; /* model bolometer stride */
dim_t mntslice; /* Number of model time slices */
size_t mtstride; /* model time slice stride */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
dim_t nbolo; /* Number of bolometers */
int nbox = 0; /* Number of noise boxes */
size_t nchisq; /* Number of data points in chisq calc */
dim_t nelbox; /* Number of data points in a noise box */
dim_t ndata; /* Total number of data points */
size_t nflag; /* Number of new flags */
int nleft; /* Number of samples not in a noise box */
dim_t ntslice; /* Number of time slices */
int nw; /* Number of worker threads */
size_t pend; /* Last non-PAD sample */
size_t pstart; /* First non-PAD sample */
smf_qual_t *qin; /* Pointer to next input quality value */
smf_qual_t *qout; /* Pointer to next output quality value */
smfArray *qua=NULL; /* Pointer to RES at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to RES at chunk */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
dim_t spikebox=0; /* Box size for spike detection */
double spikethresh=0; /* Threshold for spike detection */
size_t tend; /* Last input sample to copy */
size_t tstart; /* First input sample to copy */
size_t tstride; /* time slice stride */
double *var=NULL; /* Sample variance */
size_t xbstride; /* Box bolometer stride */
int zeropad; /* Pad with zeros? */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointer to sub-keymap containing NOI parameters */
astMapGet0A( keymap, "NOI", &kmap );
/* Assert bolo-ordered data */
smf_model_dataOrder( dat, allmodel, chunk, SMF__RES|SMF__QUA, 0, status );
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
model = allmodel[chunk];
/* Obtain parameters for NOI */
/* Data-cleaning parameters */
smf_get_cleanpar( kmap, res->sdata[0], NULL, &dcfitbox, &dcmaxsteps, &dcthresh,
&dcsmooth, &dclimcorr, NULL, &fillgaps, &zeropad, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
&spikethresh, &spikebox, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, status );
/* Did we already calculate the noise on each detector? */
astMapGet0I( kmap, "CALCFIRST", &calcfirst );
/* Initialize chisquared */
dat->chisquared[chunk] = 0;
nchisq = 0;
/* Loop over index in subgrp (subarray) */
for( idx=0; idx<res->ndat; idx++ ) {
/* Get pointers to DATA components */
res_data = (res->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Get the raw data dimensions */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice, &ndata,
&bstride, &tstride, status );
/* NOI model dimensions */
smf_get_dims( model->sdata[idx], NULL, NULL, NULL, &mntslice, NULL,
&mbstride, &mtstride, status );
/* Only estimate the white noise level once at the beginning - the
reason for this is to make measurements of the convergence
easier. We either do it prior to the start of iterations (in which
case the relative weights will be influeced by low-frequency noise,
this is initialized in smf_model_create), or else we calculate
the noise after the first iteration. */
if( (flags & SMF__DIMM_FIRSTITER) && (!calcfirst) ) {
/* There are two forms for the NOI model: one constant noise value
for each bolometer, or "ntslice" noise values for each bolometer.
Handle the first case now. */
if( mntslice == 1 ) {
var = astMalloc( nbolo*sizeof(*var) );
if (var) {
/* Measure the noise from power spectra */
smf_bolonoise( wf, res->sdata[idx], 0, 0.5, SMF__F_WHITELO,
SMF__F_WHITEHI, 0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT,
var, NULL, NULL, status );
for( i=0; i<nbolo; i++ ) if( !(qua_data[i*bstride]&SMF__Q_BADB) ) {
/* Loop over time and store the variance for each sample */
for( j=0; j<mntslice; j++ ) {
model_data[i*mbstride+(j%mntslice)*mtstride] = var[i];
}
}
var = astFree( var );
}
/* If the NOI model is of the second form, the noise is estimated
in boxes of samples lasting "NOI.BOX_SIZE" seconds, and then the
noise level in the box is assigned to all samples in the box. */
} else if( mntslice == ntslice ) {
/* If not already done, get NOI.BOX_SIZE and convert from seconds to
samples. */
if( idx == 0 ) {
boxsize = 0;
smf_get_nsamp( kmap, "BOX_SIZE", res->sdata[0], &boxsize, status );
msgOutf( "", FUNC_NAME ": Calculating a NOI variance for each "
"box of %d samples.", status, (int) boxsize );
/* Find the indices of the first and last non-PAD sample. */
smf_get_goodrange( qua_data, ntslice, tstride, SMF__Q_PAD,
&pstart, &pend, status );
/* How many whole boxes fit into this range? */
nbox = ( pend - pstart + 1 ) / boxsize;
if( nbox == 0 ) nbox = 1;
/* How many samples would be left over at the end if we used this
many boxes? */
nleft = ( pend - pstart + 1 ) - nbox*boxsize;
/* Increase "boxsize" to reduce this number as far as possible.
Any samples that are left over after this increase of boxsize
will not be used when calculating the noise levels in each
bolometer. */
boxsize += nleft/nbox;
/* Create a smfData to hold one box-worth of input data. We
do not need to copy jcmtstate information. */
if( res->sdata[idx]->hdr ) {
instate = res->sdata[idx]->hdr->allState;
res->sdata[idx]->hdr->allState = NULL;
}
box = smf_deepcopy_smfData( res->sdata[idx], 0,
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY,
0, 0, status );
if( instate ) res->sdata[idx]->hdr->allState = instate;
/* Set the length of the time axis to the box size plus padding,
and create empty data and quality arrays for it. */
if( *status == SAI__OK ) {
box->dims[ box->isTordered?2:0 ] = boxsize + pstart + (ntslice - pend - 1);
smf_get_dims( box, NULL, NULL, NULL, NULL, &nelbox,
&xbstride, NULL, status );
box->pntr[0] = astMalloc( sizeof( double )*nelbox );
box->qual = astMalloc( sizeof( smf_qual_t )*nelbox );
/* For every bolometer, flag the start and end of the quality
array as padding, and store zeros in the data array. */
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
dout = ((double *) box->pntr[0]) + xbstride*ibolo;
qout = box->qual + xbstride*ibolo;
for( itime = 0; itime < pstart; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
dout = ((double *) box->pntr[0]) + xbstride*ibolo + pstart + boxsize;;
qout = box->qual + xbstride*ibolo + pstart + boxsize;
for( itime = pend + 1; itime < ntslice; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
}
}
}
/* Work space to hold the variance for each bolometer in a box */
var = astMalloc( nbolo*sizeof(*var) );
if( *status == SAI__OK ) {
/* Index of the first time slice within the input smfData
that is included in the first box. */
tstart = pstart;
/* Loop round each noise box */
for( ibox = 0; ibox < nbox; ibox++ ) {
/* Copy the data and quality values for this box from the
input smfData into "box", leaving room for padding at
both ends of box. Note, data is bolo-ordered so we
can assume that "tstride" is 1. */
din = ((double *)(res->sdata[idx]->pntr[0])) + tstart;
dout = ((double *)(box->pntr[0])) + pstart;
qin = qua_data + tstart;
qout = box->qual + pstart;
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
memcpy( dout, din, boxsize*sizeof( *din ) );
memcpy( qout, qin, boxsize*sizeof( *qin ) );
din += bstride;
dout += xbstride;
qin += bstride;
qout += xbstride;
}
/* Measure the noise from power spectra in the box. */
smf_bolonoise( wf, box, 0, 0.5, SMF__F_WHITELO, SMF__F_WHITEHI,
0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT, var,
NULL, NULL, status );
/* Loop over time and store the variance for each sample in
the NOI model. On the last box, pick up any left over time
slices. */
if( ibox < nbox - 1 ) {
tend = tstart + boxsize - 1;
} else {
tend = pend;
}
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
if( !( qua_data[ ibolo*bstride ] & SMF__Q_BADB ) ) {
dout = model_data + ibolo*bstride + tstart;
for( itime = tstart; itime <= tend; itime++ ) {
*(dout++) = var[ ibolo ];
}
}
}
/* Update the index of the first time slice within the input
smfData that is included in the next box. */
tstart += boxsize;
}
var = astFree( var );
}
/* Report an error if the number of samples for each bolometer in
the NOI model is not 1 or "ntslice". */
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": NOI model has %d samples - should be "
"%d or 1.", status, (int) mntslice, (int) ntslice);
}
}
if( kmap ) {
/* Flag spikes in the residual after first iteration */
if( spikethresh && !(flags&SMF__DIMM_FIRSTITER) ) {
/* Now re-flag */
smf_flag_spikes( wf, res->sdata[idx], SMF__Q_MOD,
spikethresh, spikebox, &nflag, status );
msgOutiff(MSG__VERB," ", " flagged %zu new %lf-sig spikes",
status, nflag, spikethresh );
}
if( dcthresh && dcfitbox ) {
smf_fix_steps( wf, res->sdata[idx], dcthresh, dcsmooth,
dcfitbox, dcmaxsteps, dclimcorr, 1, &nflag, NULL,
NULL, status );
msgOutiff(MSG__VERB, ""," detected %zu bolos with DC steps\n",
status, nflag);
}
}
/* Now calculate contribution to chi^2. This bit takes along time
if there is a lot of data so share the work out amongst the available
worker threads. How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many bolometers to process in each worker thread. */
bolostep = nbolo/nw;
if( bolostep == 0 ) bolostep = 1;
/* Allocate job data for threads, and store the range of bolos to be
processed by each one. Ensure that the last thread picks up any
left-over bolos. */
SmfCalcModelNoiData *job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
SmfCalcModelNoiData *pdata;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->b1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->b2 = pdata->b1 + bolostep - 1;
} else {
pdata->b2 = nbolo - 1 ;
}
/* Store other values common to all jobs. */
pdata->ntslice = ntslice;
pdata->mntslice = mntslice;
pdata->qua_data = qua_data;
pdata->model_data = model_data;
pdata->res_data = res_data;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->mbstride = mbstride;
pdata->mtstride = mtstride;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_calcmodel_noi, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Accumulate the results from all the worker threads. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
dat->chisquared[chunk] += pdata->chisquared;
nchisq += pdata->nchisq;
}
/* Free the job data. */
job_data = astFree( job_data );
}
}
}
/* Free resources */
if( box ) {
box->pntr[0] = astFree( box->pntr[0] );
box->qual = astFree( box->qual );
smf_close_file( &box, status );
}
/* Normalize chisquared for this chunk */
if( (*status == SAI__OK) && (nchisq >0) ) {
dat->chisquared[chunk] /= (double) nchisq;
}
/* Clean Up */
if( kmap ) kmap = astAnnul( kmap );
}
void smf_rebinmap1( ThrWorkForce *wf, smfData *data, smfData *variance, int *lut,
size_t tslice1, size_t tslice2, int trange,
int *whichmap, dim_t nmap, smf_qual_t mask, int sampvar,
int flags, double *map, double *mapweight,
double *mapweightsq, int *hitsmap,
double *mapvar, dim_t msize, double *scalevariance,
int *status ) {
/* Local Variables */
SmfRebinMap1Data *job_data = NULL;
SmfRebinMap1Data *pdata;
double *dat=NULL; /* Pointer to data array */
size_t dbstride; /* bolo stride of data */
size_t dtstride; /* tstride of data */
int iw; /* Thread index */
dim_t mbufsize; /* Size of full (multi-map) map buffers */
dim_t nbolo; /* number of bolos */
dim_t ntslice; /* number of time slices */
int nw; /* Number of worker threads */
size_t pixstep; /* Number of map pixels per thread */
dim_t bolostep; /* Number of bolos per thread */
smf_qual_t * qual = NULL; /* Quality pointer */
double scalevar; /* variance scale factor */
double scaleweight; /* weights for calculating scalevar */
size_t t1, t2; /* range of time slices to re-grid */
double *var=NULL; /* Pointer to variance array */
size_t vbstride; /* bolo stride of variance */
dim_t vnbolo; /* number of bolos in variance */
dim_t vntslice; /* number of bolos in variance */
size_t vtstride; /* tstride of variance */
/* Main routine */
if (*status != SAI__OK) return;
/* Check inputs */
if( !data || !map || !lut || !mapweight || !mapweightsq || !mapvar ||
!hitsmap ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": Null inputs", status );
return;
}
if( !data->pntr[0] ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": supplied data is empty", status );
return;
}
dat = data->pntr[0];
qual = smf_select_qualpntr( data, NULL, status );
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &dbstride,
&dtstride, status );
/* Size of full map buffers */
if( whichmap ) {
mbufsize = nmap * msize;
} else {
mbufsize = msize;
}
if( variance ) {
var = variance->pntr[0];
smf_get_dims( variance, NULL, NULL, &vnbolo, &vntslice, NULL, &vbstride,
&vtstride, status );
/* Check that the variance dimensions are compatible with data */
if( (*status==SAI__OK) &&
((vnbolo != nbolo) || ((vntslice>1)&&(vntslice!=ntslice))) ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": variance dimensions incompatible with data",
status );
return;
}
}
/* Range of time slices to regrid */
if( trange ) {
if( tslice2 >= ntslice ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": tslice2 (%zu) can't be >= ntslice (%zu)",
status, tslice2, ntslice );
return;
}
if( tslice1 > tslice2 ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": tslice1 (%zu) > tslice2 (%zu)",
status, tslice1, tslice2 );
return;
}
t1 = tslice1;
t2 = tslice2;
} else {
t1 = 0;
t2 = ntslice-1;
}
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* If this is the first data to be accumulated zero the arrays */
if( flags & AST__REBININIT ) {
memset( map, 0, nw*mbufsize*sizeof(*map) );
memset( mapweight, 0, nw*mbufsize*sizeof(*mapweight) );
memset( mapweightsq, 0, nw*mbufsize*sizeof(*mapweightsq) );
memset( mapvar, 0, nw*mbufsize*sizeof(*mapvar) );
memset( hitsmap, 0, nw*mbufsize*sizeof(*hitsmap) );
}
/* Find how many bolos to process in each worker thread. */
bolostep = nbolo/nw;
if ( bolostep == 0 ) bolostep = 1;
/* Allocate job data for threads, and store the range of bolos to be
processed by each one. Ensure that the last thread picks up any
left-over bolos. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->p1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->p2 = pdata->p1 + bolostep - 1;
} else {
pdata->p2 = nbolo - 1 ;
}
/* Store other values common to all jobs. */
pdata->msize = msize;
pdata->nbolo = nbolo;
pdata->t1 = t1;
pdata->t2 = t2;
pdata->vntslice = vntslice;
pdata->dat = dat;
pdata->map = map;
pdata->mapvar = mapvar;
pdata->mapweightsq = mapweightsq;
pdata->mapweight = mapweight;
pdata->var = var;
pdata->hitsmap = hitsmap;
pdata->lut = lut;
pdata->whichmap = whichmap;
pdata->dbstride = dbstride;
pdata->dtstride = dtstride;
pdata->vbstride = vbstride;
pdata->vtstride = vtstride;
pdata->mask = mask;
pdata->qual = qual;
pdata->mbufsize = mbufsize;
pdata->nw = nw; /* used for final summing/rescaling */
pdata->iw = iw; /* so the thread knows with chunk it's working on */
}
}
if( var ) {
/* Accumulate data and weights in the case that variances are given*/
if( sampvar ) {
/* Measure weighted sample variance for varmap */
if( qual ) { /* QUALITY checking version */
/* Set up jobs to add the previous estimate of COM back on to the
residuals, and then wait for the jobs to complete. These jobs also
clear any SMF__Q_COM flags set by previous iterations. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 1;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
} else { /* VAL__BADD checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 2;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
}
} else {
/* Otherwise use simple error propagation for varmap */
if( qual ) { /* QUALITY checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 3;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
} else { /* VAL__BADD checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 4;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
}
}
} else {
/* Accumulate data and weights when no variances are given. In this case
the variance map is always estimated from the sample variance */
if( qual ) { /* QUALITY checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 5;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
} else { /* VAL__BADD checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 6;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
}
}
/* If this is the last data to be accumulated re-normalize */
if( flags & AST__REBINEND ) {
/* Find how many buffer pixels to process in each worker thread. */
pixstep = mbufsize/nw;
if( pixstep == 0 ) pixstep = 1;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->p1 = iw*pixstep;
if( iw < nw - 1 ) {
pdata->p2 = pdata->p1 + pixstep - 1;
} else {
pdata->p2 = mbufsize - 1 ;
}
}
if( sampvar || !var ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 7;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
scaleweight=0;
scalevar=0;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
scaleweight += pdata->scaleweight;
scalevar += pdata->scalevar;
}
/* Re-normalize scalevar */
if( scaleweight ) {
scalevar /= scaleweight;
if( scalevariance ) {
*scalevariance = scalevar;
}
}
} else {
/* Re-normalization for error propagation case */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 8;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
}
}
job_data = astFree( job_data );
}
void smf_subip( ThrWorkForce *wf, smfArray *res, smfArray *lut, int *lbnd_out,
int *ubnd_out, AstKeyMap *keymap, AstFrameSet *outfs, int *status ) {
/* Local Variables: */
HDSLoc *loc = NULL;
HDSLoc *sloc = NULL;
SmfSubIPData *job_data = NULL;
SmfSubIPData *pdata;
char ipref[200];
char subname[10];
const char *ipdata;
const char *qu;
dim_t bolostep;
dim_t nbolo;
dim_t ntslice;
double *angcdata;
double *c0data;
double *imapdata;
double *ipang;
double *p0data;
double *p1data;
int angcndf;
int c0ndf;
int imapndf;
int iw;
int nmap;
int nw;
int p0ndf;
int p1ndf;
size_t bstride;
size_t idx;
size_t tstride;
smfData *data;
smf_qual_t *qua_data;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Check if we have pol2 data, and see if it is Q or U. */
qu = NULL;
for( idx = 0; idx < res->ndat; idx++ ) {
data = res->sdata[idx];
if( !strcmp( data->hdr->dlabel, "Q" ) ){
if( !qu ) {
qu = "Q";
} else if( strcmp( qu, "Q" ) ) {
*status = SAI__ERROR;
break;
}
} else if( !strcmp( data->hdr->dlabel, "U" ) ) {
if( !qu ) {
qu = "U";
} else if( strcmp( qu, "U" ) ) {
*status = SAI__ERROR;
break;
}
} else if( qu ) {
*status = SAI__ERROR;
qu = NULL;
break;
}
}
/* Report an error if there is a mix of pol2 and non-pol2, or a mix of Q
and U. */
if( *status != SAI__OK ) {
if( qu ) {
errRep( "", "smf_subip: Input data contains mix of Q and U "
"data", status );
} else {
errRep( "", "smf_subip: Input data contains mix of POL2 and "
"non-POL2 data", status );
}
/* If we have pol2 data, get the path to the total intensity image that
is to be used to define the level of IP correction required. If no
value is supplied, annul the error and set "qu" NULL to indicate we should
leave immediately. */
} else if( qu && *status == SAI__OK ) {
parGet0c( "IPREF", ipref, sizeof(ipref), status );
if( *status == PAR__NULL ) {
errAnnul( status );
qu = NULL;
}
}
/* If we are applying IP correction... */
if( qu && *status == SAI__OK ) {
msgOutf( "", "smf_subip: applying instrumental polarisation %s "
"correction based on total intensity map `%s'", status,
qu, ipref );
/* Get an identifier for the IPREF NDF. */
ndfFind( NULL, ipref, &imapndf, status );
/* Resample the NDFs data values onto the output map grid. */
imapdata = smf_alignndf( imapndf, outfs, lbnd_out, ubnd_out,
status );
/* Annul the NDF identifier. */
ndfAnnul( &imapndf, status );
/* Create structures used to pass information to the worker threads. */
nw = wf ? wf->nworker : 1;
job_data = astMalloc( nw*sizeof( *job_data ) );
/* Get the path to the container file holding the IP model parameters. */
ipdata = "$STARLINK_DIR/share/smurf/ipdata.sdf";
astMapGet0C( keymap, "IPDATA", &ipdata );
/* Open the container file. */
hdsOpen( ipdata, "READ", &loc, status );
/* Do the IP correction for each subarray (s8a, s8b, etc) in turn. */
for( idx = 0; idx < res->ndat && *status == SAI__OK; idx++ ) {
data = res->sdata[idx];
/* Get an array holding the angle (rad.s) from north to focal plane Y,
measured positive in the sense of rotation from focal plane Y to focal
plane X, for every bolometer sample in the smfData. The values are bolo
ordered so that "bstride" is 1 and "tstsride" is nbolo. */
ipang = smf1_calcang( data, status );
/* Get the number of bolometers and time slices for the current subarray,
together with the strides between adjacent bolometers and adjacent
time slices. */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &bstride,
&tstride, status );
/* Get a locator for the structure holding the IP parameters for the
current subarray */
smf_find_subarray( data->hdr, subname, sizeof( subname ), NULL,
status );
datFind( loc, subname, &sloc, status );
/* Begin an NDF context. */
ndfBegin();
/* Get identifiers for the NDFs holding the individual parameters. Each
NDF holds a parameter value for each bolometer. */
ndfFind( sloc, "P0", &p0ndf, status );
ndfFind( sloc, "P1", &p1ndf, status );
ndfFind( sloc, "C0", &c0ndf, status );
ndfFind( sloc, "ANGC", &angcndf, status );
/* Map them. Check each one has the expected number of elements. */
ndfMap( p0ndf, "DATA", "_DOUBLE", "READ", (void **) &p0data,
&nmap, status );
if( nmap != (int) nbolo && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", p0ndf );
errRep( "", "smf_subip: Bad dimensions for ^N - should be 32x40.", status );
}
ndfMap( p1ndf, "DATA", "_DOUBLE", "READ", (void **) &p1data,
&nmap, status );
if( nmap != (int) nbolo && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", p1ndf );
errRep( "", "smf_subip: Bad dimensions for ^N - should be 32x40.", status );
}
ndfMap( c0ndf, "DATA", "_DOUBLE", "READ", (void **) &c0data,
&nmap, status );
if( nmap != (int) nbolo && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", c0ndf );
errRep( "", "smf_subip: Bad dimensions for ^N - should be 32x40.", status );
}
ndfMap( angcndf, "DATA", "_DOUBLE", "READ", (void **) &angcdata,
&nmap, status );
if( nmap != (int) nbolo && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", angcndf );
errRep( "", "smf_subip: Bad dimensions for ^N - should be 32x40.", status );
}
/* Get a pointer to the quality array for the residuals. */
qua_data = smf_select_qualpntr( data, NULL, status );
/* See how many bolometers to process in each thread. */
bolostep = nbolo/nw;
if( bolostep == 0 ) bolostep = 1;
/* Create jobs to apply the IP correction to a range of bolometers. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
/* Set the range of bolometers (b1 to b2) to be processed by the current
job. */
pdata->b1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->b2 = pdata->b1 + bolostep - 1;
} else {
pdata->b2 = nbolo - 1 ;
}
/* Store the other info needed by the worker thread. */
pdata->ntslice = ntslice;
pdata->nbolo = nbolo;
pdata->res_data = res->sdata[idx]->pntr[0];
pdata->lut_data = lut->sdata[idx]->pntr[0];
pdata->qua_data = qua_data;
pdata->ipang = ipang;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->imapdata = imapdata;
pdata->qu = qu;
pdata->p0data = p0data;
pdata->p1data = p1data;
pdata->c0data = c0data;
pdata->angcdata = angcdata;
pdata->allstate = data->hdr->allState;
/* Submit the job for execution by the next available thread. */
thrAddJob( wf, 0, pdata, smf1_subip, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* End the NDF context, thus unmapping and freeing all NDF identifiers
created since the context was started. */
ndfEnd( status );
/* Free locator for subarray IP parameters. */
datAnnul( &sloc, status );
ipang = astFree( ipang );
}
/* Free resources. */
datAnnul( &loc, status );
imapdata = astFree( imapdata );
job_data = astFree( job_data );
}
}
void smf_rebincube_nn( ThrWorkForce *wf, smfData *data, int first, int last,
int *ptime, dim_t nchan, dim_t ndet, dim_t nslice,
dim_t nxy, dim_t nout, dim_t dim[3],
int badmask, int is2d, AstMapping *ssmap,
AstSkyFrame *abskyfrm, AstMapping *oskymap,
Grp *detgrp, int moving, int usewgt, int genvar,
double tfac, double fcon, float *data_array,
float *var_array, double *wgt_array,
float *texp_array, float *teff_array, int *nused,
int *nreject, int *naccept, int *good_tsys,
int *status ){
/* Local Variables */
AstMapping *totmap = NULL; /* WCS->GRID Mapping from input WCS FrameSet */
const char *name = NULL; /* Pointer to current detector name */
const double *tsys = NULL; /* Pointer to Tsys value for first detector */
dim_t gxout; /* Output X grid index */
dim_t gyout; /* Output Y grid index */
dim_t ichan; /* Input channel index */
dim_t idet; /* detector index */
dim_t itime; /* Index of current time slice */
dim_t nchanout; /* No of spectral channels in the output */
dim_t timeslice_size; /* No of detector values in one time slice */
double *detxin = NULL; /* Work space for input X grid coords */
double *detxout = NULL; /* Work space for output X grid coords */
double *detyin = NULL; /* Work space for input Y grid coords */
double *detyout = NULL; /* Work space for output Y grid coords */
double invar; /* Input variance */
double tcon; /* Variance factor for whole time slice */
double wgt; /* Weight for input value */
float *ddata = NULL; /* Pointer to start of input detector data */
float *tdata = NULL; /* Pointer to start of input time slice data */
float *work = NULL; /* Pointer to start of work array */
float rtsys; /* Tsys value */
float teff; /* Effective integration time */
float texp; /* Total time ( = ton + toff ) */
int *nexttime; /* Pointer to next time slice index to use */
int *specpop = NULL; /* Input channels per output channel */
int *spectab = NULL; /* I/p->o/p channel number conversion table */
int first_ts; /* Is this the first time slice? */
int found; /* Was current detector name found in detgrp? */
int ignore; /* Ignore this time slice? */
int init_detector_data; /* Should detector_data be initialised? */
int iv0; /* Offset for pixel in 1st o/p spectral channel */
int jdet; /* Detector index */
int naccept_old; /* Previous number of accepted spectra */
int ochan; /* Output channel index */
int use_threads; /* Use multiple threads? */
smfHead *hdr = NULL; /* Pointer to data header for this time slice */
static smfRebincubeNNArgs1 *common_data = NULL; /* Holds data common to all detectors */
static smfRebincubeNNArgs2 *detector_data = NULL; /* Holds data for each detector */
static int *pop_array = NULL;/* I/p spectra pasted into each output spectrum */
static dim_t ndet_max = 0; /* Max number of detectors */
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Store a pointer to the input NDFs smfHead structure. */
hdr = data->hdr;
/* Store the number of pixels in one time slice */
timeslice_size = ndet*nchan;
/* Use this mapping to get the zero-based output channel number corresponding
to each input channel number. */
smf_rebincube_spectab( nchan, dim[ 2 ], ssmap, &spectab, status );
if( !spectab ) goto L999;
/* The 2D weighting scheme assumes that each output channel receives
contributions from one and only one input channel in each input file.
Create an array with an element for each output channel, holding the
number of input channels that contribute to the output channel. */
nchanout = dim[ 2 ];
if( is2d ) {
specpop = astMalloc( nchanout*sizeof( int ) );
memset( specpop, 0, nchanout*sizeof( int ) );
for( ichan = 0; ichan < nchan; ichan++ ) {
ochan = spectab[ ichan ];
if( ochan != -1 ) specpop[ ochan ]++;
}
}
/* If this is the first pass through this file, initialise the arrays. */
if( first ){
smf_rebincube_init( is2d, nxy, nout, genvar, data_array, var_array,
wgt_array, texp_array, teff_array, nused, status );
/* Allocate an extra work array and initialise it to zero. This holds the
total number of input spectra pasted into each output spectrum. It is
not needed by the AST-based function and so has not been put into
smf_rebincube_init. */
if( is2d ) {
pop_array = astMalloc( nxy*sizeof( int ) );
memset( pop_array, 0, nxy*sizeof( int ) );
}
}
/* Allocate work arrays to hold the input and output grid coords for each
detector. */
detxin = astMalloc( ndet*sizeof( double ) );
detyin = astMalloc( ndet*sizeof( double ) );
detxout = astMalloc( ndet*sizeof( double ) );
detyout = astMalloc( ndet*sizeof( double ) );
/* Initialise a string to point to the name of the first detector for which
data is available */
name = hdr->detname;
/* Fill the input arrays with the grid coords of each detector. */
for( idet = 0; idet < ndet; idet++ ) {
detxin[ idet ] = (double) idet + 1.0;
detyin[ idet ] = 1.0;
/* If a group of detectors to be used was supplied, search the group for
the name of the current detector. If not found, set the GRID coord
bad. */
if( detgrp ) {
found = grpIndex( name, detgrp, 1, status );
if( !found ) {
detxin[ idet ] = AST__BAD;
detyin[ idet ] = AST__BAD;
}
}
/* Move on to the next available detector name. */
name += strlen( name ) + 1;
}
/* Initialise a pointer to the ntex time slice index to be used. */
nexttime = ptime;
/* Count the number of time slices to be processed. */
if( ptime ) {
itime = 0;
while( ptime[ itime ] != VAL__MAXI ) itime++;
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Selecting %d time "
"slices from data file '%s'.", status, (int) itime,
data->file->name );
}
} else {
itime = nslice;
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Using all %d time "
"slices from data file '%s'.", status, (int) itime,
data->file->name );
}
}
/* Initialise the progress meter. */
smf_reportprogress( itime, status );
/* Loop round all time slices in the input NDF. */
use_threads = 0;
first_ts = 1;
for( itime = 0; itime < nslice && *status == SAI__OK; itime++ ) {
/* If this time slice is not being pasted into the output cube, pass on. */
if( nexttime ){
if( *nexttime != (int) itime ) continue;
nexttime++;
}
/* Store a pointer to the first input data value in this time slice. */
tdata = ( (float *) (data->pntr)[ 0 ] ) + itime*timeslice_size;
/* Begin an AST context. Having this context within the time slice loop
helps keep the number of AST objects in use to a minimum. */
astBegin;
/* Get a Mapping from the spatial GRID axes in the input the spatial
GRID axes in the output for the current time slice. Note this has
to be done first since it stores details of the current time slice
in the "smfHead" structure inside "data", and this is needed by
subsequent functions. */
totmap = smf_rebin_totmap( data, itime, abskyfrm, oskymap, moving,
status );
if( !totmap ) {
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Cannot get "
"Mapping for slice %d from data file '%s'.", status,
(int) itime, data->file->name );
}
astEnd;
break;
}
/* Get the effective exposure time, the total exposure time, and the
Tsys->Variance onversion factor for this time slice. Also get a
pointer to the start of the Tsys array. */
tsys = smf_rebincube_tcon( hdr, itime, fcon, &texp, &teff, &tcon,
status );
/* Use this Mapping to get the output spatial grid coords for each input
detector. */
astTran2( totmap, ndet, detxin, detyin, 1, detxout, detyout );
/* If this is the first time slice from the current input file to be pasted
into the output, see if any of the spectra will end up being pasted on
top of each other in the output. If not we can use a separate thread to
paste each spectrum. Otherwise, we cannot use multiple threads since they
may end up trying to write to the same output pixel at the same time. */
if( first_ts ) {
first_ts = 0;
use_threads = wf ? 1 : 0;
for( idet = 0; idet < ndet - 1 && use_threads; idet++ ) {
if( detxout[ idet ] != AST__BAD && detyout[ idet ] != AST__BAD ){
gxout = floor( detxout[ idet ] + 0.5 );
gyout = floor( detyout[ idet ] + 0.5 );
if( gxout >= 1 && gxout <= dim[ 0 ] &&
gyout >= 1 && gyout <= dim[ 1 ] ) {
for( jdet = idet + 1; idet < ndet; idet++ ) {
if( detxout[ jdet ] != AST__BAD &&
detyout[ jdet ] != AST__BAD ){
if( floor( detxout[ jdet ] + 0.5 ) == gxout &&
floor( detyout[ jdet ] + 0.5 ) == gyout ) {
use_threads = 0;
break;
}
}
}
}
}
}
/* If we will be using mutiple threads, do some preparation. */
if( use_threads ) {
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Using multiple "
"threads to process data file '%s'.", status,
data->file->name );
}
/* Ensure we have a structure holding information which is common to all
detectors and time slices. */
common_data = astGrow( common_data, sizeof( smfRebincubeNNArgs1 ), 1 );
if( astOK ) {
common_data->badmask = badmask;
common_data->nchan = nchan;
common_data->nchanout = nchanout;
common_data->spectab = spectab;
common_data->specpop = specpop;
common_data->nxy = nxy;
common_data->genvar = genvar;
common_data->data_array = data_array;
common_data->var_array = var_array;
common_data->wgt_array = wgt_array;
common_data->pop_array = pop_array;
common_data->nout = nout;
common_data->is2d = is2d;
}
/* Ensure we have a structure for each detector to hold the detector-specific
data, plus a pointer to the common data. */
init_detector_data = ( detector_data == NULL );
if( init_detector_data ) ndet_max = 0;
detector_data = astGrow( detector_data, sizeof( smfRebincubeNNArgs2 ),
ndet ) ;
/* Initialise pointers stored within any new elements added to the
"detector_data" array. */
if( ndet > ndet_max && astOK ) {
for( idet = ndet_max; idet < ndet; idet++ ) {
detector_data[ idet ].common = NULL;
detector_data[ idet ].work = NULL;
detector_data[ idet ].ddata = NULL;
}
ndet_max = ndet;
}
/* Allocate work space for each detector and store the common data
pointer. */
if( astOK ) {
for( idet = 0; idet < ndet; idet++ ) {
detector_data[ idet ].common = common_data;
detector_data[ idet ].work = astGrow( detector_data[ idet ].work,
sizeof( float ), nchanout );
}
}
/* If we are using a single threads, do some alternative preparation. */
} else {
if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Using a single "
"thread to process data file '%s'.", status,
data->file->name );
}
/* We need an extra work array for 2D weighting that can hold a single
output spectrum. This is used as a staging post for each input
spectrum prior to pasting it into the output cube. */
work = astMalloc( nchanout*sizeof( float ) );
}
}
/* Loop round each detector, pasting its spectral values into the output
cube. */
for( idet = 0; idet < ndet; idet++ ) {
/* If multi-threaded, initialise a bad value for the detector's weight
to indicate that it is not being used. */
if( use_threads ) detector_data[ idet ].wgt = VAL__BADD;
/* See if any good tsys values are present. */
rtsys = tsys ? (float) tsys[ idet ] : VAL__BADR;
if( rtsys <= 0.0 ) rtsys = VAL__BADR;
if( rtsys != VAL__BADR ) *good_tsys = 1;
/* Check the detector has a valid position in output grid coords */
if( detxout[ idet ] != AST__BAD && detyout[ idet ] != AST__BAD ){
/* Find the closest output pixel and check it is within the bounds of the
output cube. */
gxout = floor( detxout[ idet ] + 0.5 );
gyout = floor( detyout[ idet ] + 0.5 );
if( gxout >= 1 && gxout <= dim[ 0 ] &&
gyout >= 1 && gyout <= dim[ 1 ] ) {
/* Get the offset of the output array element that corresponds to this
pixel in the first spectral channel. */
iv0 = ( gyout - 1 )*dim[ 0 ] + ( gxout - 1 );
/* If required calculate the variance associated with this detector, based on
the input Tsys values. */
invar = VAL__BADR;
if( usewgt || genvar == 2 ) {
if( rtsys != VAL__BADR ) {
if( tcon != VAL__BADD ) invar = tcon*rtsys*rtsys;
}
}
/* Calculate the weight for this detector. If we need the input variance,
either to weight the input or to calculate output variances, but the
input variance is not available, then ignore this detector. */
ignore = 0;
if( usewgt ) {
if( invar > 0.0 && invar != VAL__BADR ) {
wgt = 1.0/invar;
} else {
ignore = 1;
}
} else if( genvar == 2 ) {
ignore = ( invar <= 0.0 || invar == VAL__BADR );
wgt = 1.0;
} else {
wgt = 1.0;
}
/* If we are not ignoring this input spectrum, get a pointer to the start
of the input spectrum data and paste it into the output cube using
either the 2D or 3D algorithm. */
if( !ignore ) {
ddata = tdata + idet*nchan;
/* First deal with cases where we are using a single thread (the current
thread). */
if( !use_threads ) {
naccept_old = *naccept;
if( is2d ) {
smf_rebincube_paste2d( badmask, nchan, nchanout, spectab,
specpop, iv0, nxy, wgt, genvar,
invar, ddata, data_array,
var_array, wgt_array, pop_array,
nused, nreject, naccept, work,
status );
} else {
smf_rebincube_paste3d( nchan, nout, spectab, iv0, nxy,
wgt, genvar, invar, ddata,
data_array, var_array,
wgt_array, nused, status );
(*naccept)++;
}
/* Now we update the total and effective exposure time arrays for the
output spectrum that receives this input spectrum. Scale the exposure
times of this time slice in order to reduce its influence on the
output expsoure times if it does not have much spectral overlap with
the output cube. Only update the exposure time arrays if the spectrum
was used (as shown by an increase in the number of accepted spectra). */
if( texp != VAL__BADR && *naccept > naccept_old ) {
texp_array[ iv0 ] += texp*tfac;
teff_array[ iv0 ] += teff*tfac;
}
/* Now deal with cases where we are using several threads. */
} else {
/* Set up the detector specific data. */
detector_data[ idet ].iv0 = iv0;
detector_data[ idet ].wgt = wgt;
detector_data[ idet ].invar = invar;
detector_data[ idet ].ddata = ddata;
detector_data[ idet ].nused = 0;
detector_data[ idet ].nreject = 0;
detector_data[ idet ].naccept = 0;
/* Add a job to the workforce's job list. This job calls smf_rebincube_paste2d
or smf_rebincube_paste3d to paste the detector input spectrum into the
output cube. */
thrAddJob( wf, 0, detector_data + idet,
smf_rebincube_paste_thread, 0, NULL, status );
}
} else if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Detector %d "
"is being ignored when processing data file '%s'.",
status, idet, data->file->name );
}
} else if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Detector %d "
"fell outside the output cube when processing "
"data file '%s'.", status, idet, data->file->name );
}
} else if( data->file ) {
msgOutiff( MSG__DEBUG, " ", "smf_rebincube_nn: Detector %d has "
"an unknown position in the output cube when processing "
"data file '%s'.", status, idet, data->file->name );
}
}
/* If using multiple threads, wait until all spectra for this time slice
have been pasted into the output cube. Then transfer the output values
from the detector data structures to the returned variables. */
if( use_threads ) {
thrWait( wf, status );
for( idet = 0; idet < ndet; idet++ ) {
if( detector_data[ idet ].wgt != VAL__BADD ) {
(*nused) += detector_data[ idet ].nused;
(*nreject) += detector_data[ idet ].nreject;
(*naccept) += detector_data[ idet ].naccept;
if( texp != VAL__BADR && detector_data[ idet ].naccept > 0 ) {
texp_array[ detector_data[ idet ].iv0 ] += texp*tfac;
teff_array[ detector_data[ idet ].iv0 ] += teff*tfac;
}
}
}
}
/* Update the progress meter. */
smf_reportprogress( 0, status );
/* End the AST context. */
astEnd;
}
/* If this is the final pass through this function, normalise the returned
data and variance values, and release any static resources allocated
within this function. */
if( last ) {
if( is2d ) {
smf_rebincube_norm2d( nout, nxy, genvar, data_array,
var_array, wgt_array, pop_array, status );
} else {
smf_rebincube_norm3d( nout, genvar, *nused, data_array,
var_array, wgt_array, status );
}
pop_array = astFree( pop_array );
if( use_threads ) {
common_data = astFree( common_data );
for( idet = 0; idet < ndet_max; idet++ ) {
detector_data[ idet ].work = astFree( detector_data[ idet ].work );
}
detector_data = astFree( detector_data );
}
}
/* Free non-static resources. */
L999:;
work = astFree( work );
spectab = astFree( spectab );
specpop = astFree( specpop );
detxin = astFree( detxin );
detyin = astFree( detyin );
detxout = astFree( detxout );
detyout = astFree( detyout );
}
void smf_filter_execute( ThrWorkForce *wf, smfData *data, smfFilter *filt,
int complement, int whiten, int *status ) {
/* Local Variables */
size_t apod_length=0; /* apodization length */
fftw_iodim dims; /* I/O dimensions for transformations */
size_t first; /* First sample apodization at start */
int i; /* Loop counter */
smfFilterExecuteData *job_data=NULL;/* Array of job data for each thread */
size_t last; /* Last sample apodization at end */
dim_t nbolo=0; /* Number of bolometers */
dim_t ndata=0; /* Total number of data points */
int nw; /* Number of worker threads */
dim_t ntslice=0; /* Number of time slices */
smf_qual_t *qua=NULL; /* Pointer to quality flags */
smfFilterExecuteData *pdata=NULL; /* Pointer to current job data */
size_t step; /* step size for dividing up work */
/* Main routine */
if (*status != SAI__OK) return;
/* Check for NULL pointers */
if( !data ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfData pointer", status );
return;
}
if( !filt ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfFilter pointer", status );
return;
}
if( filt->ndims == 2 ) {
smf_filter2d_execute( wf, data, filt, complement, status );
return;
}
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Ensure that the smfData is ordered correctly (bolo ordered) */
smf_dataOrder( wf, data, 0, status );
/* Obtain the dimensions of the array being filtered */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, NULL, NULL, status);
if( *status != SAI__OK ) return;
/* Using complement of the filter? */
if( complement ) smf_filter_complement( filt, status );
/* Pointers to quality */
qua = smf_select_qualpntr( data, NULL, status );
/* Determine the first and last samples to apodize (after padding), if
any. Assumed to be the same for all bolometers. */
if( qua ) {
smf_get_goodrange( qua, ntslice, 1, SMF__Q_PAD, &first, &last, status );
} else {
first = 0;
last = ntslice - 1;
}
/* Can we apodize? */
apod_length = filt->apod_length;
if( *status == SAI__OK ) {
if( apod_length == SMF__MAXAPLEN ) {
apod_length = (last-first+1)/2;
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": Using maximum apodization length, %zu samples.",
status, apod_length );
} else if( (last-first+1) < (2*apod_length) && apod_length != SMF__BADSZT ){
*status = SAI__ERROR;
errRepf("", FUNC_NAME
": Can't apodize, not enough samples (%zu < %zu).", status,
last-first+1, 2*apod_length);
}
}
/* If apodising is switched off, fill gaps in the data and re-create
the artifical data used for padding based on the current contents of
the smfData. */
if( apod_length == SMF__BADSZT ) {
smf_fillgaps( wf, data, SMF__Q_PAD | SMF__Q_GAP, status );
/* If apodising is switched on, fill the data (retaining the zero padding)
and apodise the data. */
} else {
smf_fillgaps( wf, data, SMF__Q_GAP, status );
if( apod_length > 0 ) smf_apodize( data, apod_length, 1, status );
}
/* Describe the input and output array dimensions for FFTW guru interface.
- dims describes the length and stepsize of time slices within a bolometer
*/
dims.n = ntslice;
dims.is = 1;
dims.os = 1;
/* Set up the job data */
if( nw > (int) nbolo ) {
step = 1;
} else {
step = nbolo/nw;
}
job_data = astCalloc( nw, sizeof(*job_data) );
for( i=0; (*status==SAI__OK)&&i<nw; i++ ) {
pdata = job_data + i;
pdata->b1 = i*step;
pdata->b2 = (i+1)*step-1;
/* Ensure that the last thread picks up any left-over bolometers */
if( (i==(nw-1)) && (pdata->b1<(nbolo-1)) ) {
pdata->b2=nbolo-1;
}
pdata->data = data;
pdata->qua = qua;
pdata->data_fft_r = astMalloc(filt->fdims[0]*sizeof(*pdata->data_fft_r));
pdata->data_fft_i = astMalloc(filt->fdims[0]*sizeof(*pdata->data_fft_i));
pdata->filt = filt;
pdata->whiten = whiten;
pdata->complement = complement;
pdata->ijob = -1; /* Flag job as ready to start */
/* Setup forward FFT plan using guru interface. Requires protection
with a mutex */
thrMutexLock( &smf_filter_execute_mutex, status );
if( *status == SAI__OK ) {
/* Just use the data_fft_* arrays from the first chunk of job data since
the guru interface allows you to use the same plans for multiple
transforms. */
pdata->plan_forward = fftw_plan_guru_split_dft_r2c( 1, &dims, 0, NULL,
data->pntr[0],
pdata->data_fft_r,
pdata->data_fft_i,
FFTW_ESTIMATE |
FFTW_UNALIGNED );
}
thrMutexUnlock( &smf_filter_execute_mutex, status );
if( !pdata->plan_forward && (*status == SAI__OK) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": FFTW3 could not create plan for forward transformation",
status);
}
/* Setup inverse FFT plan using guru interface */
thrMutexLock( &smf_filter_execute_mutex, status );
if( *status == SAI__OK ) {
pdata->plan_inverse = fftw_plan_guru_split_dft_c2r( 1, &dims, 0, NULL,
pdata->data_fft_r,
pdata->data_fft_i,
data->pntr[0],
FFTW_ESTIMATE |
FFTW_UNALIGNED);
}
thrMutexUnlock( &smf_filter_execute_mutex, status );
if( !pdata->plan_inverse && (*status==SAI__OK) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": FFTW3 could not create plan for inverse transformation",
status);
}
}
/* Execute the filter */
for( i=0; (*status==SAI__OK)&&i<nw; i++ ) {
pdata = job_data + i;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfFilterExecuteParallel, 0,
NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Clean up the job data array */
if( job_data ) {
for( i=0; i<nw; i++ ) {
pdata = job_data + i;
if( pdata->data_fft_r ) pdata->data_fft_r = astFree( pdata->data_fft_r );
if( pdata->data_fft_i ) pdata->data_fft_i = astFree( pdata->data_fft_i );
/* Destroy the plans */
thrMutexLock( &smf_filter_execute_mutex, status );
fftw_destroy_plan( pdata->plan_forward );
fftw_destroy_plan( pdata->plan_inverse );
thrMutexUnlock( &smf_filter_execute_mutex, status );
}
job_data = astFree( job_data );
}
/* Return the filter to its original state if required */
if( complement == -1 ) smf_filter_complement( filt, status );
/* Remove the effects of the apodisation from the filtered data. */
if( apod_length != SMF__BADSZT && apod_length > 0 ) {
smf_apodize( data, apod_length, 0, status );
}
}
void smf_flag_spikes( ThrWorkForce *wf, smfData *data, smf_qual_t mask,
double thresh, size_t box, size_t *nflagged,
int *status ){
/* Local Variables */
int i; /* Loop counter */
smfFlagSpikesData *job_data=NULL;/* Array of job data for each thread */
dim_t nbolo; /* Number of bolometers */
dim_t ntime; /* Number of time-slices */
double *dat = NULL; /* Pointer to bolo data */
smfFlagSpikesData *pdata=NULL;/* Pointer to job data */
int nw; /* Number of worker threads */
size_t bstride; /* Vector stride between bolometer samples */
size_t nflag; /* Number of samples flagged */
size_t tstride; /* Vector stride between time samples */
smf_qual_t *qua = NULL; /* Pointer to quality flags */
size_t step; /* step size for dividing up work */
int njobs=0; /* Number of jobs to be processed */
/* Check inherited status. Also return immediately if no spike flagging
is to be performed. */
if( *status != SAI__OK || thresh == 0.0 ) return;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Check we have double precision data. */
smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status );
/* Get a pointer to the quality array to use. */
qua = smf_select_qualpntr( data, NULL, status );
/* Report an error if we have no quality array. */
if( !qua && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": No valid QUALITY array was provided", status );
}
/* Get a pointer to the data array to use. Report an error if we have
no data array. */
dat = data->pntr[0];
if( !dat && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": smfData does not contain a DATA component",
status);
}
/* Check the supplied thresh value is valid. */
if( thresh <= 0 && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSetd( "THRESH", thresh );
errRep( " ", FUNC_NAME ": Can't find spikes: thresh=^THRESH, must be > 0",
status);
}
/* Check the supplied box value is valid. */
if( box <= 2 && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "BOX", box );
errRep( " ", FUNC_NAME ": Can't find spikes: box=^BOX, must be > 2",
status);
}
/* Obtain data dimensions, and the stride between adjacent elements on
each axis (bolometer and time). Use the existing data order to avoid
the cost of re-ordering. */
smf_get_dims( data, NULL, NULL, &nbolo, &ntime, NULL, &bstride, &tstride,
status );
/* Check we have room for at least 3 boxes along the time axis. */
if( 3*box > ntime && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "BOX", box );
msgSeti( "MAX", ntime/3 - 1 );
errRep( " ", FUNC_NAME ": Can't find spikes: box=^BOX is too large, "
" must be < ^MAX.", status);
}
/* Set up the job data */
if( nw > (int) nbolo ) {
step = 1;
} else {
step = nbolo/nw;
if( !step ) {
step = 1;
}
}
job_data = astCalloc( nw, sizeof(*job_data) );
for( i=0; (*status==SAI__OK)&&i<nw; i++ ) {
pdata = job_data + i;
pdata->b1 = i*step;
pdata->b2 = (i+1)*step-1;
/* if b1 is greater than the number of bolometers, we've run out of jobs... */
if( pdata->b1 >= nbolo ) {
break;
}
/* increase the jobs counter */
njobs++;
/* Ensure that the last thread picks up any left-over bolometers */
if( (i==(nw-1)) && (pdata->b1<(nbolo-1)) ) {
pdata->b2=nbolo-1;
}
pdata->ijob = -1; /* Flag job as ready to start */
pdata->box = box;
pdata->bstride = bstride;
pdata->dat = dat;
pdata->mask = mask;
pdata->thresh = thresh;
pdata->nbolo = nbolo;
pdata->ntime = ntime;
pdata->qua = qua;
pdata->tstride = tstride;
}
/* Submit jobs to find spikes in each block of bolos */
thrBeginJobContext( wf, status );
for( i=0; (*status==SAI__OK)&&i<njobs; i++ ) {
pdata = job_data + i;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfFlagSpikesPar, 0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
thrEndJobContext( wf, status );
/* Count flagged samples from all of the jobs and free resources */
nflag=0;
if( job_data ) {
for( i=0; i<njobs; i++ ) {
pdata = job_data + i;
nflag += pdata->nflag;
}
job_data = astFree( job_data );
}
/* Return the number of flagged samples, if requested */
if( nflagged ) *nflagged = nflag;
}
size_t smf_check_quality( ThrWorkForce *wf, smfData *data, int showbad,
int *status ) {
double *d=NULL; /* Pointer to data array */
size_t nbad=0; /* inconsistency counter */
size_t nnan = 0; /* Number of nan values found */
size_t ninf = 0; /* Number of inf values found */
size_t nqualincon = 0; /* Number of inconsistent bad/qual */
dim_t ndata; /* Number of data points */
size_t bstride; /* bol stride */
size_t tstride; /* time slice stride */
smf_qual_t *qual=NULL; /* Pointer to the QUALITY array */
int nw; /* Number of worker threads */
int iw; /* Thread index */
SmfCheckQualityData *job_data = NULL; /* Array of job descriptions */
SmfCheckQualityData *pdata; /* Pointer to next job description */
size_t sampstep; /* Number of samples per thread */
if ( *status != SAI__OK ) return 0;
/* Check for DATA */
if( !data ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL data supplied", status );
return 0;
}
if( !data->pntr[0] ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData does not contain a DATA component",
status );
return 0;
}
if( data->dtype != SMF__DOUBLE ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData does not have type SMF__DOUBLE", status );
return 0;
}
d = (double *) data->pntr[0];
/* Check for QUALITY */
qual = smf_select_qualpntr( data, NULL, status );
if( !qual ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL quality supplied", status);
return 0;
}
/* Calculate data dimensions */
smf_get_dims( data, NULL, NULL, NULL, NULL, &ndata, &bstride,
&tstride, status );
if( *status == SAI__OK ) {
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many samples to process in each worker thread. */
sampstep = ndata/nw;
if( sampstep == 0 ) sampstep = 1;
/* Allocate job data for threads, and store the range of samples to be
processed by each one. Ensure that the last thread picks up any
left-over samples. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->d1 = iw*sampstep;
if( iw < nw - 1 ) {
pdata->d2 = pdata->d1 + sampstep - 1;
} else {
pdata->d2 = ndata - 1 ;
}
/* Store other values common to all jobs. */
pdata->qual = qual;
pdata->d = d;
pdata->showbad = showbad;
pdata->bstride = bstride;
pdata->tstride = tstride;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_check_quality, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Accumulate the results from all the worker threads. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
nbad += pdata->nbad;
nnan += pdata->nnan;
ninf += pdata->ninf;
nqualincon += pdata->nqualincon;
}
/* Free the job data. */
job_data = astFree( job_data );
}
}
if (nbad > 0) {
msgOutiff( MSG__VERB, "", "Quality inconsistency found: %zu DATA/QUAL, %zu NaN, %zu Inf",
status, nqualincon, nnan, ninf );
}
return nbad;
}
size_t smf_clean_pca( ThrWorkForce *wf, smfData *data, size_t t_first,
size_t t_last, double thresh, size_t ncomp,
smfData **components, smfData **amplitudes,
int flagbad, int sub, AstKeyMap *keymap,
smf_qual_t mask, int *status ){
double *amp=NULL; /* matrix of components amplitudes for each bolo */
size_t abstride; /* bolo stride in amp array */
size_t acompstride; /* component stride in amp array */
size_t bstride; /* bolo stride */
double *comp=NULL; /* data cube of components */
size_t ccompstride; /* component stride in comp array */
size_t ctstride; /* time stride in comp array */
gsl_matrix *cov=NULL; /* bolo-bolo covariance matrix */
size_t i; /* Loop counter */
int ii; /* Loop counter */
size_t j; /* Loop counter */
smfPCAData *job_data=NULL;/* job data */
size_t k; /* Loop counter */
size_t *goodbolo=NULL; /* Indices of the good bolometers for analysis */
dim_t nbolo; /* number of bolos */
dim_t ndata; /* number of samples in data */
size_t ngoodbolo; /* number good bolos = number principal components */
dim_t ntslice; /* number of time slices */
int nw; /* total available worker threads */
smfPCAData *pdata=NULL; /* Pointer to job data */
smf_qual_t *qua=NULL; /* Pointer to quality array */
gsl_vector *s=NULL; /* singular values for SVD */
size_t bstep; /* Bolo step size for job division */
size_t step; /* step size for job division */
size_t tlen; /* Length of the time-series used for PCA */
size_t tstride; /* time slice stride */
gsl_vector *work=NULL; /* workspace for SVD */
if (*status != SAI__OK) return 0;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Check for NULL smfData pointer */
if( !data || !data->pntr[0]) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, NULL data supplied", status );
return 0;
}
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride, &tstride,
status );
if( data->ndims != 3 ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, smfData should be 3-dimensional",
status );
return 0;
}
if( data->dtype != SMF__DOUBLE ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, smfData should be double precision",
status );
return 0;
}
if( ntslice <= 2 ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": fewer than 2 time slices!", status );
goto CLEANUP;
}
/* If the range of time slices has not been specified, us the total
range excluding padding and apodizing. */
qua = smf_select_qualpntr( data, 0, status );
if( !t_last ) {
if( qua ) {
smf_get_goodrange( qua, ntslice, tstride, (SMF__Q_PAD | SMF__Q_APOD),
&t_first, &t_last, status );
} else {
t_last = ntslice-1;
}
}
if( t_last > (ntslice-1) ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": t_last is set past the last time slice!",
status );
goto CLEANUP;
}
if( (t_last < t_first) || ( (t_last - t_first) < 1 ) ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": t_last - t_first must be > 1", status );
goto CLEANUP;
}
tlen = t_last - t_first + 1;
if( flagbad && (tlen != ntslice ) ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": flagbad unsupported if t_first/last do not span full data",
status );
goto CLEANUP;
}
if( qua ) {
/* If quality supplied, identify good bolometers */
ngoodbolo = 0;
for( i=0; i<nbolo; i++ ) {
if( !(qua[i*bstride]&SMF__Q_BADB) ) {
ngoodbolo++;
}
}
/* Now remember which were the good bolometers */
goodbolo = astCalloc( ngoodbolo, sizeof(*goodbolo) );
ngoodbolo = 0;
for( i=0; i<nbolo; i++ ) {
if( !(qua[i*bstride]&SMF__Q_BADB) ) {
goodbolo[ngoodbolo] = i;
ngoodbolo++;
}
}
} else {
/* Otherwise assume all bolometers are good */
ngoodbolo = nbolo;
goodbolo = astCalloc( ngoodbolo, sizeof(*goodbolo) );
for( i=0; i<ngoodbolo; i++ ) {
goodbolo[i] = i;
}
}
if( ngoodbolo <= 2 ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": fewer than 2 working bolometers!", status );
goto CLEANUP;
}
/* Fill bad values and values flagged via "mask" (except entirely bad
bolometers) with interpolated data values. */
mask &= ~SMF__Q_BADB;
smf_fillgaps( wf, data, mask, status );
/* Allocate arrays */
amp = astCalloc( nbolo*ngoodbolo, sizeof(*amp) );
comp = astCalloc( ngoodbolo*tlen, sizeof(*comp) );
cov = gsl_matrix_alloc( ngoodbolo, ngoodbolo );
s = gsl_vector_alloc( ngoodbolo );
work = gsl_vector_alloc( ngoodbolo );
/* These strides will make comp time-ordered */
ccompstride = 1;
ctstride = ngoodbolo;
/* These strides will also make amp look time-ordered (sort-of: the time
axis is now the component number */
abstride = 1;
acompstride = nbolo;
/* Allocate job data for threads */
job_data = astCalloc( nw, sizeof(*job_data) );
/* Set up the division of labour for threads: independent blocks of time */
if( nw > (int) tlen ) {
step = 1;
} else {
step = tlen/nw;
}
if( nw > (int) ngoodbolo ) {
bstep = 1;
} else {
bstep = ngoodbolo/nw;
}
for( ii=0; (*status==SAI__OK)&&(ii<nw); ii++ ) {
pdata = job_data + ii;
/* Blocks of time slices */
pdata->t1 = ii*step + t_first;
pdata->t2 = (ii+1)*step + t_first - 1;
/* Blocks of bolometers. */
pdata->b1 = ii*bstep;
pdata->b2 = (ii+1)*bstep - 1;
/* Ensure that the last thread picks up any left-over tslices */
if( (ii==(nw-1)) ) {
pdata->t2 = t_first + tlen - 1;
pdata->b2 = ngoodbolo - 1;
}
/* initialize work data */
pdata->amp = NULL;
pdata->abstride = abstride;
pdata->acompstride = acompstride;
pdata->bstride = bstride;
pdata->comp = comp;
pdata->cov = NULL;
pdata->covwork = NULL;
pdata->ccompstride = ccompstride;
pdata->ctstride = ctstride;
pdata->data = data;
pdata->goodbolo = NULL;
pdata->ijob = -1;
pdata->nbolo = nbolo;
pdata->ngoodbolo = ngoodbolo;
pdata->t_first = t_first;
pdata->t_last = t_last;
pdata->tlen = tlen;
pdata->operation = 0;
pdata->tstride = tstride;
/* Each thread will accumulate the projection of its own portion of
the time-series. We'll add them to the master amp at the end */
pdata->amp = astCalloc( nbolo*ngoodbolo, sizeof(*(pdata->amp)) );
/* Each thread will accumulate sums of x, y, and x*y for each bolo when
calculating the covariance matrix */
pdata->covwork = astCalloc( ngoodbolo*ngoodbolo,
sizeof(*(pdata->covwork)) );
/* each thread gets its own copy of the goodbolo lookup table */
pdata->goodbolo = astCalloc( ngoodbolo, sizeof(*(pdata->goodbolo)) );
if( *status == SAI__OK ) {
memcpy( pdata->goodbolo, goodbolo,
ngoodbolo*sizeof(*(pdata->goodbolo)) );
}
}
if( *status == SAI__OK ) {
/* Remove the mean from each gap-filled bolometer time stream ---------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME ": removing bolometer means...",
status );
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
pdata->operation = -1;
thrAddJob( wf, 0, pdata, smfPCAParallel, 0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Measure the covariance matrix using parallel code ---------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME
": measuring bolo-bolo covariance matrix...", status );
/* Set up the jobs to calculate sums for each time block and submit */
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
pdata->operation = 0;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfPCAParallel,
0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* We now have to add together all of the sums from each thread and
normalize */
if( *status == SAI__OK ) {
for( i=0; i<ngoodbolo; i++ ) {
for( j=i; j<ngoodbolo; j++ ) {
double c;
double *covwork=NULL;
double sum_xy;
sum_xy = 0;
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
covwork = pdata->covwork;
sum_xy += covwork[ i + j*ngoodbolo ];
}
c = sum_xy / ((double)tlen-1);
gsl_matrix_set( cov, i, j, c );
gsl_matrix_set( cov, j, i, c );
}
}
}
}
/* Factor cov = u s v^T, noting that the SVD routine calculates v^T in
in-place of cov. --------------------------------------------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME
": perfoming singular value decomposition...", status );
smf_svd( wf, ngoodbolo, cov->data, s->data, NULL, 10*VAL__EPSD,
1, status );
if( CHECK ) {
double check=0;
for( i=0; i<ngoodbolo; i++ ) {
for( j=0; j<ngoodbolo; j++ ) {
check += gsl_matrix_get( cov, j, i );
}
}
printf("--- check inverted: %lf\n", check);
}
/* Calculate normalized eigenvectors with parallel code --------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME
": calculating statistically-independent components...", status );
/* The above calculation tells us what linear combinations of the original
bolometer time series will give us the statistically independent new
set of basis vectors (components), which we then normalize by their RMS. */
/* Set up the jobs to calculate sums for each time block and submit */
if( *status == SAI__OK ) {
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
pdata->cov = cov;
pdata->operation = 1;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfPCAParallel,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Then normalize. Some of the components may have zero amplitude and
so cannot be used (i.e. we are trying to use more components than
there is evidence for in the data). So we check for zero sigma. In
fact, we check for silly small sigma, not just zero sigma. Any
component for which the sigma is less than 1E-10 of the log-mean
sigma is excluded. */
{
double *sigmas = astMalloc( ngoodbolo*sizeof( *sigmas ) );
double check = 0;
double s1 = 0.0;
int s2 = 0;
int nlow = 0;
for( i=0; (*status==SAI__OK)&&(i<ngoodbolo); i++ ) {
double sigma;
smf_stats1D( comp + i*ccompstride, ctstride, tlen, NULL, 0,
0, NULL, &sigma, NULL, NULL, status );
/* Apparently we need this to get the normalization right */
sigma *= sqrt((double) tlen);
if( *status == SAI__OK ) {
if( sigma > 0.0 ) {
for( k=0; k<tlen; k++ ) {
comp[i*ccompstride + k*ctstride] /= sigma;
sigmas[ i ] = sigma;
s1 += log10( sigma );
s2++;
}
} else {
for( k=0; k<tlen; k++ ) {
comp[i*ccompstride + k*ctstride] = VAL__BADD;
sigmas[ i ] = VAL__BADD;
}
nlow++;
}
}
}
/* Exclude any components that have a silly small standard deviation
(less that 1E-10 of the logmean of all components). Any with zero
standard deviation will already have been excluded. */
if( s2 > 0 ) {
double logmean = s1/s2;
for( i=0; i<ngoodbolo; i++ ) {
if( sigmas[ i ] != VAL__BADD && sigmas[ i ] < 1E-10*logmean ) {
for( k=0; k<tlen; k++ ) {
comp[i*ccompstride + k*ctstride] = VAL__BADD;
nlow++;
}
}
}
}
msgOutiff( MSG__DEBUG, "", FUNC_NAME ": rejecting %d (out of %zu) components"
" because they are too weak to normalise", status, nlow, ngoodbolo );
for( i=0; i<ngoodbolo*tlen; i++ ) {
if( comp[i] != VAL__BADD ) check += comp[i];
}
sigmas = astFree( sigmas );
//printf("--- check component: %lf\n", check);
}
/* Now project the data along each of these normalized basis vectors
to figure out the amplitudes of the components in each bolometer
time series. ------------------------------------------------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME
": calculating component amplitudes in each bolo...", status );
/* Set up the jobs */
if( *status == SAI__OK ) {
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
pdata->operation = 2;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfPCAParallel,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Add all of the amp arrays together from the threads */
if( *status == SAI__OK ) {
size_t index;
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
for( i=0; i<ngoodbolo; i++ ) { /* Loop over good bolo */
for( j=0; j<ngoodbolo; j++ ) { /* Loop over component */
index = goodbolo[i]*abstride + j*acompstride;
amp[index] += pdata->amp[index];
}
}
}
}
if( CHECK ){
double check=0;
for( i=0; i<nbolo*ngoodbolo; i++ ) {
check += amp[i];
}
printf("--- check combined amp: %lf\n", check);
}
if( CHECK ){
double check=0;
for( i=0; i<ngoodbolo*tlen; i++ ) {
if( comp[i] != VAL__BADD ) check += comp[i];
}
printf("--- check component A: %lf\n", check);
}
/* Check to see if the amplitudes are mostly negative or positive. If
mostly negative, flip the sign of both the component and amplitudes */
if( *status == SAI__OK ) {
double total;
for( j=0; j<ngoodbolo; j++ ) { /* loop over component */
total = 0;
for( i=0; i<ngoodbolo; i++ ) { /* loop over bolometer */
total += amp[goodbolo[i]*abstride + j*acompstride];
}
/* Are most amplitudes negative for this component? */
if( total < 0 ) {
/* Flip sign of the amplitude */
for( i=0; i<ngoodbolo; i++ ) { /* loop over bolometer */
amp[goodbolo[i]*abstride + j*acompstride] =
-amp[goodbolo[i]*abstride + j*acompstride];
}
/* Flip sign of the component */
for( k=0; k<tlen; k++ ) {
if( comp[j*ccompstride + k*ctstride] != VAL__BADD ) {
comp[j*ccompstride + k*ctstride] *= -1;
}
}
}
}
}
/* Finally, copy the master amp array back into the workspace for
each thread */
if( *status == SAI__OK ) {
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
memcpy( pdata->amp, amp, sizeof(*(pdata->amp))*nbolo*ngoodbolo );
}
}
if( CHECK ){
double check=0;
for( i=0; i<ngoodbolo*tlen; i++ ) {
if( comp[i] != VAL__BADD ) check += comp[i];
}
printf("--- check component B: %lf\n", check);
}
/* Flag outlier bolometers if requested ------------------------------------*/
if( (*status==SAI__OK) && flagbad ) {
smfArray *data_array=NULL;
smfArray *gain_array=NULL;
smfGroup *gain_group=NULL;
AstKeyMap *kmap=NULL; /* Local keymap */
AstObject *obj=NULL; /* Used to avoid compiler warnings */
double *template=NULL;
smf_qual_t * smf_qual_map( ThrWorkForce *wf, int indf, const char mode[],
smf_qfam_t *family, size_t *nmap, int * status ) {
size_t i; /* Loop counter */
int itemp = 0; /* temporary int */
smf_qfam_t lfamily = SMF__QFAM_NULL; /* Local quality family */
size_t nout; /* Number of elements mapped */
size_t numqn = 0; /* number of quality names */
IRQLocs *qlocs = NULL;/* IRQ Quality */
unsigned char *qmap; /* pointer to mapped unsigned bytes */
void *qpntr[1]; /* Somewhere to put the mapped pointer */
smf_qual_t *retval = NULL; /* Returned pointer */
int there; /* Does the NDF Have a Quality component? */
char xname[DAT__SZNAM+1]; /* Name of extension holding quality names */
SmfQualMapData *job_data = NULL;
SmfQualMapData *pdata;
int nw;
size_t step;
int iw;
if (*status != SAI__OK) return retval;
/* Ensure jobs submitted to the workforce within this function are
handled separately to any jobs submitted earlier (or later) by any
other function. */
thrBeginJobContext( wf, status );
/* how many elements do we need */
ndfSize( indf, &itemp, status );
nout = itemp;
if (nmap) *nmap = nout;
/* malloc the QUALITY buffer. Initialise to zero to simplify logic
below. It is difficult to determine in advance which case can use
initialisation. */
retval = astCalloc( nout, sizeof(*retval) );
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many elements to process in each worker thread. */
step = nout/nw;
if( step == 0 ) step = 1;
/* Allocate job data for threads, and store common values. Ensure that the
last thread picks up any left-over elements. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->i1 = iw*step;
if( iw < nw - 1 ) {
pdata->i2 = pdata->i1 + step - 1;
} else {
pdata->i2 = nout - 1 ;
}
pdata->retval = retval;
}
}
/* If the NDF has no QUality component, return the buffer filled with
zeros. */
ndfState( indf, "QUALITY", &there, status );
if( there ) {
/* READ and UPDATE mode require that the QUALITY is processed
and copied before being returned. WRITE mode means that the
buffer contains no information to copy yet. WRITE/ZERO
and WRITE/BAD also require that we do not do any quality
handling */
if ( strncmp(mode, "WRITE",5) == 0 ) {
/* WRITE and WRITE/ZERO are actually treated the same way
because we always initialise */
if ( strcmp( mode, "WRITE/BAD") == 0 ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 1;
thrAddJob( wf, 0, pdata, smf1_qual_map, 0, NULL, status );
}
thrWait( wf, status );
}
/* unmap the NDF buffer and return the pointer */
if (family) *family = lfamily;
return retval;
}
/* Map the quality component (we always need to do this) */
ndfMap( indf, "QUALITY", "_UBYTE", mode, &qpntr[0], &itemp, status );
qmap = qpntr[0];
/* Need to find out what quality names are in play so we
can work out which family to translate them to */
irqFind( indf, &qlocs, xname, status );
numqn = irqNumqn( qlocs, status );
if ( *status == IRQ__NOQNI || numqn == 0) {
/* do not have any names defined so we have no choice
in copying the values directly out the file */
if (*status != SAI__OK) errAnnul( status );
/* simple copy with type conversion */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->qmap = qmap;
pdata->operation = 2;
thrAddJob( wf, 0, pdata, smf1_qual_map, 0, NULL, status );
}
thrWait( wf, status );
} else {
IRQcntxt contxt = 0;
int ndfqtosmf[NDFBITS]; /* NDF bit (arr index) and SMURF alternative */
int ndfqtoval[NDFBITS]; /* NDF bit (arr index) and corresponding Qual value */
int ndfqval[NDFBITS]; /* Bit values for NDF quality */
int identity = 1; /* Is this a simple identity map? */
/* prefill the mapping with bit to bit mapping */
for (i=0; i<NDFBITS; i++) {
ndfqtosmf[i] = i;
ndfqtoval[i] = BIT_TO_VAL(i);
ndfqval[i] = ndfqtoval[i];
}
/* Now translate each name to a bit */
for (i = 0; i < numqn && *status == SAI__OK; i++) {
char qname[IRQ__SZQNM+1];
char commnt[IRQ__SZCOM+1];
int fixed;
int value;
int bit;
int done;
smf_qual_t qval;
smf_qfam_t tmpfam = 0;
irqNxtqn( qlocs, &contxt, qname, &fixed, &value, &bit,
commnt,sizeof(commnt), &done, status );
bit--; /* IRQ starts at 1 */
/* Now convert the quality name to a quality value
and convert that to a bit. These should all be
less than 9 bits because they are in the NDF file. */
qval = smf_qual_str_to_val( qname, &tmpfam, status );
if (*status == SMF__BADQNM ) {
/* annul status and just copy this bit from the file
to SMURF without change. This might result in a clash
of bits but we either do that or drop out the loop
and assume everything is broken */
if (*status != SAI__OK) errAnnul(status);
ndfqtosmf[bit] = bit;
ndfqtoval[bit] = BIT_TO_VAL(bit);
} else if( *status == SAI__OK ){
if (lfamily == SMF__QFAM_NULL) {
lfamily = tmpfam;
} else if (lfamily != tmpfam) {
msgOutif(MSG__QUIET, "",
"WARNING: Quality names in file come from different families",
status );
}
ndfqtosmf[bit] = smf_qual_to_bit( qval, status );
ndfqtoval[bit] = qval;
/* not a 1 to 1 bit translation */
if (bit != ndfqtosmf[bit]) identity = 0;
}
}
/* Now copy from the file and translate the bits. If this is an
identity mapping or we do not know the family then we go quick. */
if (*status == SAI__OK) {
if ( (identity && lfamily != SMF__QFAM_TCOMP) || lfamily == SMF__QFAM_NULL) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->qmap = qmap;
pdata->ndfqval = ndfqval;
pdata->lfamily = lfamily;
pdata->ndfqtoval = ndfqtoval;
pdata->ndfqval = ndfqval;
pdata->operation = 2;
thrAddJob( wf, 0, pdata, smf1_qual_map, 0, NULL, status );
}
thrWait( wf, status );
} else {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->qmap = qmap;
pdata->ndfqval = ndfqval;
pdata->lfamily = lfamily;
pdata->ndfqtoval = ndfqtoval;
pdata->ndfqval = ndfqval;
pdata->operation = 3;
thrAddJob( wf, 0, pdata, smf1_qual_map, 0, NULL, status );
}
thrWait( wf, status );
/* we have uncompressed */
if (lfamily == SMF__QFAM_TCOMP) lfamily = SMF__QFAM_TSERIES;
}
}
}
/* Free quality */
irqRlse( &qlocs, status );
/* no longer need the mapped data */
ndfUnmap( indf, "QUALITY", status );
}
/* End the Thr job context */
thrEndJobContext( wf, status );
/* Free other resources. */
job_data = astFree( job_data );
if (family) *family = lfamily;
return retval;
}
void smf_snrmask( ThrWorkForce *wf, int abssnr, unsigned char *oldmask,
const double *map, const double *mapvar,
const dim_t *dims, double snr_hi, double snr_lo,
unsigned char *mask, int *status ){
/* Local Variables: */
const double *pm = NULL;
const double *pv = NULL;
dim_t i;
dim_t j;
double snr;
int *cindex = NULL;
int *ps = NULL;
int *psn = NULL;
int *table = NULL;
int iass;
int iclean;
int iclump;
int ineb;
int itemp;
int itop1;
int itop2;
int iworker;
int neb_offset[ 4 ];
int nworker;
int ok;
int rowstep;
int top;
smfSnrMaskJobData *job_data = NULL;
smfSnrMaskJobData *pdata = NULL;
unsigned char *maskold = NULL;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Save a copy of the old mask, if supplied. Doing it now, means that the
old and new mask pointers can be the same. */
if( oldmask ) maskold = astStore( NULL, oldmask,
sizeof(*oldmask)*dims[0]*dims[1] );
/* Allocate an array to hold a clump index for every map pixel. Initialise
it to hold zeros. */
cindex = astCalloc( dims[ 0 ]*dims[ 1 ], sizeof( *cindex ) );
/* Initialise the index to assign to the next clump of pixels found above
the lower SNR limit. Note, no clump is given an index of zero. */
top = 1;
/* Initialise the pointer to the table holding associated clump indices.
The first element is unused, so set it to a safe value of zero (i.e.
"no clump"). */
table = astCalloc( top, sizeof( *table ) );
/* Set up the vector offsets to the three neighbouring pixels in the lower
row, and the left hand neighbour in the current row. */
neb_offset[ 0 ] = -1; /* Left neighbour in current row */
neb_offset[ 1 ] = -dims[ 0 ] - 1; /* Left neighbour in lower row */
neb_offset[ 2 ] = -dims[ 0 ]; /* Central neighbour in lower row */
neb_offset[ 3 ] = -dims[ 0 ] + 1; /* Right neighbour in lower row */
/* Loop round the map, looking for pixels that are above the lower SNR
limit. Within this loop we store a positive clump index for each pixel that
is above the lower SNR limit. Each clump of contiguous pixel above the limit
has a separate clump index. If two clumps touch each other, we associate
their indices together using a table to indicate that they are part of the
same physical clump. */
pm = map;
pv = mapvar;
ps = cindex;
for( j = 0; j < dims[ 1 ] && *status == SAI__OK; j++ ) {
for( i = 0; i < dims[ 0 ]; i++, pm++, pv++, ps++ ) {
/* Get the SNR value. */
if( mapvar ) {
if( *pm != VAL__BADD && *pv != VAL__BADD && *pv > 0.0 ){
snr = *pm / sqrt( *pv );
} else {
snr = VAL__BADD;
}
} else {
snr = *pm;
}
/* If source can be negative as well as positive, use the absolute SNR in
the following check. */
if( abssnr && snr != VAL__BADD ) snr = fabs( snr );
/* Check the SNR is good and above the lower limit. */
if( snr != VAL__BADD && snr > snr_lo ){
/* The three neighbouring pixels on row (j-1), and the left hand
neighbouring pixel on row j, have already been checked on earlier
passes round this loop. Check each of these four pixels in turn to see
if they were flagged as being above the lower SNR limit. */
itop1 = 0;
for( ineb = 0; ineb < 4; ineb++ ) {
/* Get a pointer to the neighbouring clump index value, checking it is not off
the edge of the array. */
if( ineb == 0 ) {
ok = ( i > 0 );
} else if( ineb == 1 ) {
ok = ( i > 0 && j > 0 );
} else if( ineb == 2 ) {
ok = ( j > 0 );
} else {
ok = ( i < dims[ 0 ] - 1 && j > 0 );
}
if( ok ) {
psn = ps + neb_offset[ ineb ];
/* If this neighbour is flagged as above the lower SNR limit (i.e. has a
positive clump index), and the current pixel has not yet been assigned to
an existing clump, assign the neighbour's clump index to the current pixel. */
if( *psn > 0 ) {
if( *ps == 0 ) {
*ps = *psn;
/* Find the clump index at the top of the tree containing the neighbouring pixel. */
itop1 = *psn;
while( table[ itop1 ] ) itop1 = table[ itop1 ];
/* If this neighbour is flagged as above the lower SNR limit, but the
current pixel has already been assigned to an existing clump, the current
pixel is adjacent to both clumps and so joins them into one. So record that
this neighbours clump index should be associated with the clump index of
the current pixel. */
} else {
/* We need to check first that the two clump indices are not already part
of the same tree of associated clumps. Without this we could produce
loops in the tree. Find the clump indices at the top of the tree
containing the neighbouring pixel. */
itop2 = *psn;
while( table[ itop2 ] ) itop2 = table[ itop2 ];
/* If the two clumps are not in the same tree, indicate that the pixel
index at the top of the tree for the neighbouring pixels clump index is
associated with the central pixel's clump index. */
if( itop1 != itop2 ) table[ itop2 ] = *ps;
}
}
}
}
/* If the current pixel has no neighbours that are above the lower SNR
limit, we start a new clump for the current pixel. */
if( *ps == 0 ) {
/* Assign the next clump index to the current pixel, and then increment
the next clump index. Report an error if we have reached the max
allowable clump index value. */
if( top == INT_MAX ) {
*status = SAI__ERROR;
errRep( "", "smf_snrmask: Too many low-SNR clumps found.",
status );
break;
}
*ps = top++;
/* Extend the table that holds the associations between clumps. This
table has one element for each clump index (plus an unused element at the
start for the unused clump index "0"). The value stored in this table
for a given clump index is the index of another clump with which the
first clump should be associated. If two clumps are associated it
indicates that they are part of the same physical clump. Associations
form a tree structure. A value of zero in this table indicates that
the clump is either unassociated with any other clump, or is at the head
of a tree of associated clumps. */
table = astGrow( table, top, sizeof( *table ) );
if( *status != SAI__OK ) break;
table[ *ps ] = 0;
}
}
}
}
/* We now loop round the map again, this time looking for pixels that are
above the higher SNR limit. */
pm = map;
pv = mapvar;
ps = cindex;
for( j = 0; j < dims[ 1 ]; j++ ) {
for( i = 0; i < dims[ 0 ]; i++, pm++, pv++, ps++ ) {
/* Get the SNR value. */
if( mapvar ) {
if( *pm != VAL__BADD && *pv != VAL__BADD && *pv > 0.0 ){
snr = *pm / sqrt( *pv );
} else {
snr = VAL__BADD;
}
} else {
snr = *pm;
}
/* If source can be negative as well as positive, use the absolute SNR. */
if( abssnr && snr != VAL__BADD ) snr = fabs( snr );
/* Check the SNR is good and above the upper limit. */
if( snr != VAL__BADD && snr > snr_hi ){
/* Since this pixel is above the higher SNR limit, it must also be above
the lower SNR Limit, and so will have a non-zero clump index. We flag that
this clump contains "source" pixels by storing a value of -1 for it in the
clump association table. First record the original value for later use. */
iass = table[ *ps ];
table[ *ps ] = -1;
/* If this clump index is associated with another clump (i.e. had a non-zero
value in the clump association table), the two clumps adjoins each other.
So indicate that the second clump also contains "source" pixels by
changing its table value to -1. Enter a loop to do this all the way up
to the top of the association tree. Note, this is not necessarily all
adjoining clumps, since we have only gone "up" the tree - there may be
other adjoining clumps lower down the tree. */
while( iass > 0 ) {
itemp = table[ iass ];
table[ iass ] = -1;
iass = itemp;
}
}
}
}
/* Now check all cumps to see if they adjoin a "source" clump. Note, no
clumps are given the index zero, so we skip the first element of the
table. */
for( iclump = 1; iclump < top; iclump++ ) {
iass = table[ iclump ];
/* Work up the tree of neighbouring clumps until we find a clump that has
an index of 0 or -1. If 0, it means that we have reached the top of
the tree without finding a "source" clump. If -1 it means we have
reached a source clump. */
while( iass > 0 ) {
iass = table[ iass ];
}
/* If we have found a source clump, then all clumps above it in the tree
should already be set to -1. We now walk up the tree from the current
clump until we reach the source clump, marking all intermediate clumps
as source clumps by setting them to -1 in the table. */
if( iass < 0 ) {
iass = iclump;
while( iass > 0 ) {
itemp = table[ iass ];
table[ iass ] = -1;
iass = itemp;
}
/* If no source clump was found, mark all intermediate clumps as
non-source by setting theem to zero in the table. This may give us a
little extra speed (maybe) since subsequent walks will terminate
sooner. */
} else {
iass = iclump;
while( iass > 0 ) {
itemp = table[ iass ];
table[ iass ] = 0;
iass = itemp;
}
}
}
/* One last pass, to store the final mask values. We can multi-thread
this bit. Create structures used to pass information to the worker
threads. If we have more threads than rows, we will process one row
in each thread and so we can reduce the number of threads used to
equal the number of rows. */
nworker = wf ? wf->nworker : 1;
if( nworker > (int) dims[ 1 ] ) nworker = dims[ 1 ];
job_data = astMalloc( nworker*sizeof( *job_data ) );
/* Check we can de-reference the job data pointer safely. */
if( *status == SAI__OK ) {
/* Decide how many rows to process in each thread. */
rowstep = dims[ 1 ]/nworker;
if( rowstep == 0 ) rowstep = 1;
/* Set up the information needed by each thread, */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->operation = 1;
pdata->cindex = cindex;
pdata->jlo = iworker*rowstep;
if( iworker == nworker - 1 ) {
pdata->jhi = dims[ 1 ] - 1;
} else {
pdata->jhi = pdata->jlo + rowstep - 1;
}
pdata->rowlen = dims[ 0 ];
pdata->mask = mask;
pdata->table = table;
/* Pass the job to the workforce for execution. */
thrAddJob( wf, 0, pdata, smf1_snrmask_job, 0, NULL, status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
/* Now clean up the very crinkly edges of the mask. Also, the mask may
contain small holes which need to be cleaned. Clean it NCLEAN times. */
for( iclean = 0; iclean < NCLEAN; iclean++ ) {
/* Clean the mask, putting the cleaned mask into "cindex" array. We
exclude pixels in the first and last rows since they do not have a
complete set of neighbours (each worker thread also ignores the first
and last pixel in each row for the same reason). Decide how many rows
to process in each thread. */
rowstep = ( dims[ 1 ] - 2 )/nworker;
if( rowstep == 0 ) rowstep = 1;
/* Modify the information needed by each thread, */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->operation = 2;
pdata->jlo = iworker*rowstep + 1;
if( iworker == nworker - 1 ) {
pdata->jhi = dims[ 1 ] - 2;
} else {
pdata->jhi = pdata->jlo + rowstep - 1;
}
/* Pass the job to the workforce for execution. */
thrAddJob( wf, 0, pdata, smf1_snrmask_job, 0, NULL, status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
/* Transfer the new mask from the "cindex" array back to the "mask" array.
Add in any source pixels from the old mask if required. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->maskold = maskold;
pdata->operation = 3;
thrAddJob( wf, 0, pdata, smf1_snrmask_job, 0, NULL, status );
}
thrWait( wf, status );
/* If an old mask was supplied, ensure any source pixels in the old mask
are also source pixels in the new mask. */
if( oldmask ) {
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->maskold = maskold;
pdata->operation = 4;
thrAddJob( wf, 0, pdata, smf1_snrmask_job, 0, NULL, status );
}
thrWait( wf, status );
}
}
}
/* Free resources. */
job_data = astFree( job_data );
maskold = astFree( maskold );
table = astFree( table );
cindex = astFree( cindex );
}
void smf_fit_poly( ThrWorkForce *wf, smfData *data, const size_t order,
int remove, double *poly, int *status) {
/* Local variables */
size_t bstride; /* bolo strides */
int i; /* Loop counter */
smfFitPolyData *job_data=NULL;/* Array of job data for each thread */
dim_t nbolo=0; /* Number of bolometers */
int njobs=0; /* Number of jobs to be processed */
dim_t ntslice = 0; /* Number of time slices */
int nw; /* Number of worker threads */
smfFitPolyData *pdata=NULL; /* Pointer to job data */
const smf_qual_t *qual; /* pointer to the quality array */
size_t step; /* step size for dividing up work */
size_t tstride; /* time strides */
/* Check status */
if (*status != SAI__OK) return;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Should check data type for double */
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status)) return;
if ( smf_history_check( data, FUNC_NAME, status) ) {
msgSetc("F", FUNC_NAME);
msgOutif(MSG__VERB," ",
"^F has already been run on these data, returning to caller",
status);
return;
}
/* Get the dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &bstride,
&tstride, status);
/* Return with error if there is no QUALITY component */
qual = smf_select_cqualpntr( data, NULL, status );
if( !qual && (*status == SAI__OK) ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Data doesn't have a QUALITY component.", status );
return;
}
/* Return with error if order is greater than the number of data
points */
if ( order >= ntslice ) {
if ( *status == SAI__OK) {
msgSeti("O",order);
msgSeti("NF",ntslice);
*status = SAI__ERROR;
errRep( FUNC_NAME, "Requested polynomial order, ^O, greater than or "
"equal to the number of points, ^NF. Unable to fit polynomial.",
status );
}
return;
}
/* Set up the job data */
if( nw > (int) nbolo ) {
step = 1;
} else {
step = nbolo/nw;
if( !step ) {
step = 1;
}
}
job_data = astCalloc( nw, sizeof(*job_data) );
for( i=0; (*status==SAI__OK)&&i<nw; i++ ) {
pdata = job_data + i;
pdata->b1 = i*step;
pdata->b2 = (i+1)*step-1;
/* if b1 is greater than the number of bolometers, we've run out of jobs */
if( pdata->b1 >= nbolo ) {
break;
}
/* increase the jobs counter */
njobs++;
/* Ensure that the last thread picks up any left-over bolometers */
if( (i==(nw-1)) && (pdata->b1<(nbolo-1)) ) {
pdata->b2=nbolo-1;
}
pdata->ijob = -1; /* Flag job as ready to start */
pdata->bstride = bstride;
pdata->indata = data->pntr[0];
pdata->isTordered = data->isTordered;
pdata->nbolo = nbolo;
pdata->ntslice = ntslice;
pdata->order = order;
pdata->poly = poly;
pdata->qual = qual;
pdata->remove = remove;
pdata->tstride = tstride;
}
/* Submit jobs to fit polynomial baselines to block of bolos */
thrBeginJobContext( wf, status );
for( i=0; (*status==SAI__OK)&&i<njobs; i++ ) {
pdata = job_data + i;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfFitPolyPar, 0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
thrEndJobContext( wf, status );
/* Free local resources. */
job_data = astFree( job_data );
}
void findback( int *status ){
/*
*+
* Name:
* FINDBACK
* Purpose:
* Estimate the background in an NDF by removing small scale structure.
* Language:
* C
* Type of Module:
* ADAM A-task
* Synopsis:
* void findback( int *status );
* Description:
* This application uses spatial filtering to remove structure with a
* scale size less than a specified size from a 1, 2, or 3 dimensional
* NDF, thus producing an estimate of the local background within the NDF.
*
* The algorithm proceeds as follows. A filtered form of the input data
* is first produced by replacing every input pixel by the minimum of
* the input values within a rectangular box centred on the pixel.
* This filtered data is then filtered again, using a filter that
* replaces every pixel value by the maximum value in a box centred on
* the pixel. This produces an estimate of the lower envelope of the data,
* but usually contains unacceptable sharp edges. In addition, this
* filtered data has a tendency to hug the lower envelope of the
* noise, thus under-estimating the true background of the noise-free
* data. The first problem is minimised by smoothing the background
* estimate using a filter that replaces every pixel value by the mean
* of the values in a box centred on the pixel. The second problem
* is minimised by estimating the difference between the input data
* and the background estimate within regions well removed from any
* bright areas. This difference is then extrapolated into the bright
* source regions and used as a correction to the background estimate.
* Specifically, the residuals between the input data and the initial
* background estimate are first formed, and residuals which are more
* than three times the RMS noise are set bad. The remaining residuals
* are smoothed with a mean filter. This smoothing will replace a lot
* of the bad values rejected above, but may not remove them all. Any
* remaining bad values are estimated by linear interpolation between
* the nearest good values along the first axis. The interpolated
* residuals are then smoothed again using a mean filter, to get a
* surface representing the bias in the initial background estimate.
* This surface is finally added onto the initial background estimate
* to obtain the output NDF.
* Usage:
* findback in out box
* ADAM Parameters:
* BOX() = _INTEGER (Read)
* The dimensions of each of the filters, in pixels. Each value
* should be odd (if an even value is supplied, the next higher odd
* value will be used). The number of values supplied should not
* exceed the number of significant (i.e. more than one element)
* pixel axes in the input array. If any trailing values of 1 are
* supplied, then each pixel value on the corresponding axes
* will be fitted independently of its neighbours. For instance,
* if the data array is 3-dimensional, and the third BOX value is 1,
* then each x-y plane will be fitted independently of the neighbouring
* planes. If the NDF has more than 1 pixel axis but only 1 value is
* supplied, then the same value will be used for the both the first
* and second pixel axes (a value of 1 will be assumed for the third
* axis if the input array is 3-dimensional).
* MSG_FILTER = _CHAR (Read)
* Controls the amount of diagnostic information reported. This is the
* standard messaging level. The default messaging level is NORM (2).
* A value of NONE or 0 will suppress all screen output. VERB (3) will
* indicate progress through the various stages of the algorithm. [NORM]
* IN = NDF (Read)
* The input NDF.
* RMS = _DOUBLE (Read)
* Specifies a value to use as the global RMS noise level in the
* supplied data array. The suggested default value is the square root
* of the mean of the values in the input NDF's Variance component.
* If the NDF has no Variance component, the suggested default
* is based on the differences between neighbouring pixel values,
* measured over the entire input NDF. If multiple slices within the
* NDF are to be processed independently (see parameter BOX), it
* may be more appropriate for a separate default RMS to be calculated
* for each slice. This will normally be the case if the noise could
* be different in each of the slices. In such cases a null (!) can
* be supplied for the RMS parameter, which forces a separate
* default RMS value to be found and used for each slice. Any
* pixel-to-pixel correlation in the noise can result in these
* defaults being too low.
* SUB = _LOGICAL (Read)
* If a TRUE value is supplied, the output NDF will contain the
* difference between the supplied input data and the estimated
* background. If a FALSE value is supplied, the output NDF will
* contain the estimated background itself. [FALSE]
* OUT = NDF (Write)
* The output NDF containing either the estimated background, or the
* background-subtracted input data, as specified by parameter SUB.
* Notes:
* - Smoothing cubes in 3 dimensions can be very slow.
* Copyright:
* Copyright (C) 2009 Science and Technology Facilities Council.
* Copyright (C) 2006, 2007 Particle Physics & Astronomy Research Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* TIMJ: Tim Jenness (JAC, Hawaii)
* {enter_new_authors_here}
* History:
* 13-SEP-2006 (DSB):
* Original version.
* 19-MAR-2007 (DSB):
* - Added parameters SUB and RMS.
* - Fix bug that left the output NDF uninitialised if ILEVEL is set
* non-zero.
* - Use generic data type handling as in FINDCLUMPS.
* 14-JAN-2009 (TIMJ):
* Use MERS for message filtering.
* 29-JUL-2009 (TIMJ):
* Rename ILEVEL to MSG_FILTER
* 17-MAY-2011 (DSB):
* Use sqrt rather than sqrtf when calculating RMS.
* 12-SEP-2011 (DSB):
* Process slices in separate threads.
* {enter_further_changes_here}
*-
*/
/* Local Variables: */
CupidFindback0Data *job_data; /* Pointer to data for all jobs */
CupidFindback0Data *pdata; /* Pointer to data for current job */
Grp *grp; /* GRP identifier for configuration settings */
ThrWorkForce *wf = NULL; /* Pool of persistent worker threads */
char dtype[ 21 ]; /* HDS data type for output NDF */
char itype[ 21 ]; /* HDS data type to use when processing */
double *ipv; /* Pointer to Variance array */
double *pd1; /* Pointer to double precision input data */
double *pd2; /* Pointer to double precision output data */
double rms; /* Global rms error in data */
double sum; /* Sum of variances */
float *pf1; /* Pointer to single precision input data */
float *pf2; /* Pointer to single precision output data */
int *old_status; /* Pointer to original status value */
int box[ 3 ]; /* Dimensions of each cell in pixels */
int dim[ NDF__MXDIM ]; /* Dimensions of each NDF pixel axis */
int el; /* Number of elements mapped */
int i; /* Loop count */
int indf1; /* Identifier for input NDF */
int indf2; /* Identifier for output NDF */
int islice; /* Slice index */
int iystep; /* Index of slice in ydirection */
int izstep; /* Index of slice in z direction */
int lbnd[ NDF__MXDIM ]; /* Lower pixel bounds of slice */
int n; /* Number of values summed in "sum" */
int ndim; /* Total number of pixel axes in NDF */
int newalg; /* Use experimental algorithm variations? */
int nsdim; /* Number of significant pixel axes in NDF */
int nslice; /* Number of slices to process */
int nval; /* Number of values supplied */
int nystep; /* Number of independent y slices */
int nzstep; /* Number of slices in z direction */
int sdim[ 3 ]; /* Dimensions of each significant NDF axis */
int slice_dim[ 3 ]; /* Dimensions of each significant slice axis */
int slice_lbnd[ 3 ]; /* Lower bounds of each significant slice axis */
int slice_size; /* Number of pixels in each slice */
int state; /* Parameter state */
int sub; /* Output the background-subtracted input data? */
int type; /* Integer identifier for data type */
int ubnd[ NDF__MXDIM ]; /* Upper pixel bounds of slice */
int var; /* Does i/p NDF have a Variance component? */
size_t size; /* Size of GRP group */
void *ipd1; /* Pointer to input Data array */
void *ipd2; /* Pointer to output Data array */
void *ipdin; /* Pointer to input Data array */
void *ipdout; /* Pointer to output Data array */
/* Abort if an error has already occurred. */
if( *status != SAI__OK ) return;
/* Start an NDF context */
ndfBegin();
/* Record the existing AST status pointer, and ensure AST uses the supplied
status pointer instead. */
old_status = astWatch( status );
/* Get an identifier for the input NDF. We use NDG (via kpg1_Rgndf)
instead of calling ndfAssoc directly since NDF/HDS has problems with
file names containing spaces, which NDG does not have. */
kpg1Rgndf( "IN", 1, 1, "", &grp, &size, status );
ndgNdfas( grp, 1, "READ", &indf1, status );
grpDelet( &grp, status );
/* Get the pixel index bounds of the input NDF. */
ndfBound( indf1, NDF__MXDIM, lbnd, ubnd, &ndim, status );
/* Identify and count the number of significant axes (i.e. axes spanning
more than 1 pixel). Also record their dimensions. */
nsdim = 0;
for( i = 0; i < ndim; i++ ) {
dim[ i ] = ubnd[ i ] - lbnd[ i ] + 1;
if( dim[ i ] > 1 ) sdim[ nsdim++ ] = dim[ i ];
}
/* If there are too many significant axes, report an error. */
if( nsdim > 3 && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", indf1 );
msgSeti( "NS", nsdim );
errRep( "", "The NDF '^N' has ^NS significant pixel axes, but this"
"application requires 1, 2 or 3.", status );
}
/* Ensure we have 3 values in sdim (pad with trailings 1's if required). */
if( nsdim < 3 ) sdim[ 2 ] = 1;
if( nsdim < 2 ) sdim[ 1 ] = 1;
/* See if the output is to contain the background-subtracted data, or the
background estimate itself. */
parGet0l( "SUB", &sub, status );
/* Create the output by propagating everything except the Data and
(if we are outputting the background itself) Variance arrays. */
if( sub ) {
ndfProp( indf1, "UNITS,AXIS,WCS,QUALITY,VARIANCE", "OUT", &indf2,
status );
} else {
ndfProp( indf1, "UNITS,AXIS,WCS,QUALITY", "OUT", &indf2, status );
}
msgBlankif( MSG__VERB, status );
/* Get the dimensions of each of the filters, in pixels. If only one
value is supplied, duplicate it as the second value if the second axis
is significant. If fewer than 3 values were supplied, use 1 for the 3rd
value (whether or not it is significant). This results in each plane
being fitted independently of the adjacent planes by default. */
parGet1i( "BOX", nsdim, box, &nval, status );
if( *status != SAI__OK ) goto L999;
if( nval < 2 ) box[ 1 ] = ( nsdim > 1 ) ? box[ 0 ] : 1;
if( nval < 3 ) box[ 2 ] = 1;
/* Ensure box sizes are odd. */
box[ 0 ] = 2*( box[ 0 ] / 2 ) + 1;
box[ 1 ] = 2*( box[ 1 ] / 2 ) + 1;
box[ 2 ] = 2*( box[ 2 ] / 2 ) + 1;
msgOutiff( MSG__VERB, "", "Using box sizes [%d,%d,%d].", status,
box[0], box[1], box[2]);
/* If any trailing axes have a cell size of 1, then we apply the algorithm
independently to every pixel index on the trailing axes. First of all
set things up assuming that there are no trailing axes with cell size
of 1. */
nystep = 1;
nzstep = 1;
slice_dim[ 0 ] = sdim[ 0 ];
slice_dim[ 1 ] = sdim[ 1 ];
slice_dim[ 2 ] = sdim[ 2 ];
slice_lbnd[ 0 ] = lbnd[ 0 ];
slice_lbnd[ 1 ] = lbnd[ 1 ];
slice_lbnd[ 2 ] = lbnd[ 2 ];
/* If the 3rd pixel axis has a cell size of 1, arrange that each slice
contains a single plane. */
if( box[ 2 ] == 1 ) {
nzstep = sdim[ 2 ];
slice_dim[ 2 ] = 1;
/* If the 2nd pixel axis also has a cell size of 1, arrange that each slice
contains a single row. */
if( box[ 1 ] == 1 ) {
nystep = sdim[ 1 ];
slice_dim[ 1 ] = 1;
}
}
/* Determine the number of pixels in each independent slice. */
slice_size = slice_dim[ 0 ]*slice_dim[ 1 ]*slice_dim[ 2 ];
/* Decide what numeric data type to use, and set the output NDF data type. */
ndfMtype( "_REAL,_DOUBLE", indf1, indf1, "Data,Variance", itype,
20, dtype, 20, status );
if( !strcmp( itype, "_DOUBLE" ) ) {
type = CUPID__DOUBLE;
} else {
type = CUPID__FLOAT;
}
ndfStype( dtype, indf2, "Data,Variance", status );
/* Map the input and output arrays. */
ndfMap( indf1, "Data", itype, "READ", &ipdin, &el, status );
ndfMap( indf2, "Data", itype, "WRITE", &ipdout, &el, status );
/* If the rms value is supplied on the command, there is no need to
calculate a default value. */
parState( "RMS", &state, status );
if( state == PAR__GROUND ) {
/* Calculate the default RMS value. If the NDF has a Variance component
it is the square root of the mean Variance value. Otherwise, it is found
by looking at differences between adjacent pixel values in the Data
component. */
ndfState( indf1, "VARIANCE", &var, status );
if( *status == SAI__OK && var ) {
ndfMap( indf1, "VARIANCE", "_DOUBLE", "READ", (void *) &ipv, &el, status );
sum = 0.0;
n = 0;
for( i = 0; i < el; i++ ) {
if( ipv[ i ] != VAL__BADD ) {
sum += ipv[ i ];
n++;
}
}
if( n > 0 ) {
rms = sqrt( sum/n );
} else {
*status = SAI__ERROR;
errRep( "", "The supplied data contains insufficient "
"good Variance values to continue.", status );
}
} else {
ipv = NULL;
rms = cupidRms( type, ipdin, el, sdim[ 0 ], status );
}
/* Set the default RMS noise level. */
parDef0d( "RMS", rms, status );
}
/* Abort if an error has occurred. */
if( *status != SAI__OK ) goto L999;
/* Get the RMS noise level. */
parGet0d( "RMS", &rms, status );
/* Annul the error and use an RMS value of VAL__BAD if a null parameter
value was supplied. This causes an independent default noise estimate to
be used for each slice of the base NDF. */
if( *status == PAR__NULL ) {
errAnnul( status );
rms = VAL__BADD;
}
/* See if any experimental algorithm variations are to be used. */
parGet0l( "NEWALG", &newalg, status );
/* Create a pool of worker threads. */
wf = thrCreateWorkforce( thrGetNThread( "CUPID_THREADS", status ), status );
/* Get memory to hold a description of each job passed to a worker. There
is one job for each slice. */
nslice = nystep*nzstep;
job_data = astMalloc( nslice*sizeof( *job_data ) );
if( *status == SAI__OK ) {
/* Loop round all slices to be processed. */
ipd1 = ipdin;
ipd2 = ipdout;
islice = 0;
pdata = job_data;
for( izstep = 0; izstep < nzstep ; izstep++ ) {
slice_lbnd[ 1 ] = lbnd[ 1 ];
for( iystep = 0; iystep < nystep; iystep++, islice++,pdata++ ) {
/* Store the information needed by the function (cupidFindback0) that
does the work in a thread. */
pdata->islice = islice;
pdata->nslice = nslice;
pdata->type = type;
pdata->ndim = ndim;
pdata->box[ 0 ] = box[ 0 ];
pdata->box[ 1 ] = box[ 1 ];
pdata->box[ 2 ] = box[ 2 ];
pdata->rms = rms;
pdata->ipd1 = ipd1;
pdata->ipd2 = ipd2;
pdata->slice_dim[ 0 ] = slice_dim[ 0 ];
pdata->slice_lbnd[ 0 ] = slice_lbnd[ 0 ];
pdata->slice_dim[ 1 ] = slice_dim[ 1 ];
pdata->slice_lbnd[ 1 ] = slice_lbnd[ 1 ];
pdata->slice_dim[ 2 ] = slice_dim[ 2 ];
pdata->slice_lbnd[ 2 ] = slice_lbnd[ 2 ];
pdata->newalg = newalg;
pdata->slice_size = slice_size;
/* Submit a job to the workforce to process the current slice. */
thrAddJob( wf, 0, pdata, cupidFindback0, 0, NULL, status );
/* Update pointers to the start of the next slice in the input and output
arrays. */
if( type == CUPID__FLOAT ) {
ipd1 = ( (float *) ipd1 ) + slice_size;
ipd2 = ( (float *) ipd2 ) + slice_size;
} else {
ipd1 = ( (double *) ipd1 ) + slice_size;
ipd2 = ( (double *) ipd2 ) + slice_size;
}
/* Increment the lower bound on the 2nd pixel axis. */
slice_lbnd[ 1 ]++;
}
/* Increment the lower bound on the 3rd pixel axis. */
slice_lbnd[ 2 ]++;
}
/* Wait until all jobs have finished. */
thrWait( wf, status );
}
/* The output currently holds the background estimate. If the user has
requested that the output should hold the background-subtracted input
data, then do the arithmetic now. */
if( sub && *status == SAI__OK ) {
if( type == CUPID__FLOAT ) {
pf1 = (float *) ipdin;
pf2 = (float *) ipdout;
for( i = 0; i < el; i++, pf1++, pf2++ ) {
if( *pf1 != VAL__BADR && *pf2 != VAL__BADR ) {
*pf2 = *pf1 - *pf2;
} else {
*pf2 = VAL__BADR;
}
}
} else {
pd1 = (double *) ipdin;
pd2 = (double *) ipdout;
for( i = 0; i < el; i++, pd1++, pd2++ ) {
if( *pd1 != VAL__BADD && *pd2 != VAL__BADD ) {
*pd2 = *pd1 - *pd2;
} else {
*pd2 = VAL__BADD;
}
}
}
}
/* Tidy up */
L999:;
msgBlankif( MSG__VERB, status );
/* Free workspace. */
job_data = astFree( job_data );
wf = thrDestroyWorkforce( wf );
/* Reinstate the original AST inherited status value. */
astWatch( old_status );
/* End the NDF context */
ndfEnd( status );
/* If an error has occurred, issue another error report identifying the
program which has failed (i.e. this one). */
if( *status != SAI__OK ) {
errRep( "FINDBACK_ERR", "FINDBACK: Failed to find the background "
"of an NDF.", status );
}
}
int smf_correct_extinction(ThrWorkForce *wf, smfData *data, smf_tausrc *thetausrc, smf_extmeth method,
AstKeyMap * extpars, double tau, double *allextcorr,
double **wvmtaucache, int *status) {
/* Local variables */
int allquick = 0; /* Is the extinction for all bolometers the same? */
double amstart = VAL__BADD; /* Airmass at start */
double amend = VAL__BADD; /* Airmass at end */
double elstart = VAL__BADD; /* Elevation at start (radians) */
double elend = VAL__BADD;/* Elevation at end (radians) */
smfHead *hdr = NULL; /* Pointer to full header struct */
double *indata = NULL; /* Pointer to data array */
int isTordered; /* data order of input data */
int lbnd[2]; /* Lower bound */
size_t ndims; /* Number of dimensions in input data */
dim_t nframes = 0; /* Number of frames */
dim_t npts = 0; /* Number of data points */
dim_t nx = 0; /* # pixels in x-direction */
dim_t ny = 0; /* # pixels in y-direction */
smf_tausrc tausrc; /* Local copy of tausrc value */
int ubnd[2]; /* Upper bound */
double *vardata = NULL; /* Pointer to variance array */
double * wvmtau = NULL; /* WVM tau (smoothed or not) for these data */
int nw; /* Number of worker threads */
int iw; /* Thread index */
SmfCorrectExtinctionData *job_data = NULL; /* Array of job descriptions */
SmfCorrectExtinctionData *pdata; /* Pointer to next job description */
size_t framestep; /* Number of frames per thread */
/* Check status */
if (*status != SAI__OK) return allquick;
/* If no correction requested, return */
if( method==SMF__EXTMETH_NONE ) {
msgOutif(MSG__VERB, "", FUNC_NAME ": Extinction method=none, returning",
status );
return allquick;
}
if ( ! thetausrc ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Must supply a thetausrc argument. Possible programming error.",
status );
return allquick;
}
/* Use a local value for tausrc as we update it in this routine. In particular,
CSOFIT becomes WVMRAW and this would be confusing to the caller */
tausrc = *thetausrc;
/* If no opacity monitor specified generate bad status */
if( tausrc==SMF__TAUSRC_NULL ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No source of opacity information could be determined",
status );
return allquick;
}
if( smf_history_check( data, FUNC_NAME, status) ) {
/* If caller not requesting allextcorr fail here */
if( !allextcorr ) {
msgSetc("F", FUNC_NAME);
msgOutif(MSG__VERB," ",
"^F has already been run on these data, returning to caller",
status);
return allquick;
}
}
/* Acquire the data order */
isTordered = data->isTordered;
/* make sure we have a header */
hdr = data->hdr;
if( hdr == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Input data has no header", status);
return allquick;
}
/* Do we have 2-D image data? */
ndims = data->ndims;
if (ndims == 2) {
nframes = 1;
nx = (data->dims)[0];
ny = (data->dims)[1];
npts = nx*ny;
} else {
/* this routine will also check for dimensionality */
smf_get_dims( data, &nx, &ny, &npts, &nframes, NULL, NULL, NULL, status );
}
/* Tell user we're correcting for extinction */
msgOutif(MSG__VERB," ",
"Correcting for extinction.", status);
/* Should check data type for double if not allextcorr case */
if( !allextcorr ) {
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status)) return allquick;
}
/* Check that we're not trying to use the WVM for 2-D data */
if ( ndims == 2 && tausrc == SMF__TAUSRC_WVMRAW ) {
if ( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Method WVMRaw can not be used on 2-D image data", status );
return allquick;
}
} else if (ndims == 2 && tausrc == SMF__TAUSRC_CSOFIT ) {
/* This is CSOTAU mode with the value calculated from the fits. We have to either
calculate the value here based on the FITS headers or we have to ensure that
when this mode triggers we've been given the fallback tau derived in this manner.
Revisit this as the code develops (we do not want to be reading fits multiple times).
*/
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Method CSOFIT not yet supported on 2-D image data", status );
return allquick;
}
} else if (ndims < 2 || ndims > 3) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRepf( FUNC_NAME, "Can not extinction correct data with %zd dimension(s)", status,
ndims );
return allquick;
}
}
/* if we are WVMRAW, CSOFIT or AUTO and we have a cache we should always use it since
we assume it was filled in properly the previous time. */
if (wvmtaucache && *wvmtaucache &&
(tausrc == SMF__TAUSRC_WVMRAW ||
tausrc == SMF__TAUSRC_AUTO ||
tausrc == SMF__TAUSRC_CSOFIT)) {
wvmtau = *wvmtaucache;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using cached high resolution data for extinction correction of ^FILE",
status);
tausrc = SMF__TAUSRC_WVMRAW; /* We are now WVMRAW as we have the data */
/* Assume that we only do not know the provenance if in AUTO mode */
if (tausrc == SMF__TAUSRC_AUTO) *thetausrc = SMF__TAUSRC_CACHED;
}
if (!wvmtau && tausrc == SMF__TAUSRC_WVMRAW) {
size_t ntotaltau = 0;
size_t ngoodtau = 0;
smf_calc_smoothedwvm( wf, NULL, data, extpars, &wvmtau, &ntotaltau,
&ngoodtau, status );
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using WVM mode for extinction correction of ^FILE"
" %.0f %% of WVM data are present", status,
(double)(100.0*(double)ngoodtau/(double)ntotaltau) );
}
if (*status == SAI__OK && tausrc == SMF__TAUSRC_CSOFIT) {
/* Calculate the fit but we can use the same cache that WVM uses */
size_t nframes = 0;
smf_calc_csofit( data, extpars, &wvmtau, &nframes, status );
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__QUIET, "", "Using CSO fits for extinction correction of ^FILE",
status );
/* Rebrand as WVM data from this point on */
tausrc = SMF__TAUSRC_WVMRAW;
}
/* AUTO mode logic */
/*
* Default position is to use WVM data but we have two caveats
*
* 1. Was this observation done during a period where the WVM has been flagged as unreliable?
* 2. If the WVM is nominally okay, do we have sufficient good data during this period?
*
* If the WVM should not be used we fallback to seeing if we have a fit available for this
* night from the CSO data.
*
* If we do not have a reliable WVM or CSO fit then we fallback to using a fixed CSO number
* from the header.
*
* This final fallback position is unfortunate as it is highly likely that this is not a reliable
* number if we have fits for every night of observing (we have no information on whether a missing
* fit indicates the CSO was too unstable to use or whether it means we simply haven't got to it
* yet).
*
*/
/* Check auto mode */
if (tausrc == SMF__TAUSRC_AUTO && *status == SAI__OK) {
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
if (ndims == 2) {
/* have to use CSO mode */
tausrc = SMF__TAUSRC_CSOTAU;
*thetausrc = tausrc;
} else if (ndims == 3) {
/* We have already done the cache test so not needed here */
/* Is the WVM nominally stable for this night? */
if (smf_is_wvm_usable( data->hdr, status ) ) {
/* Calculate the WVM tau data and see if we have enough good data */
size_t ngoodtau = 0;
size_t ntotaltau = 0;
double percentgood = 0.0;
smf_calc_smoothedwvm( wf, NULL, data, extpars, &wvmtau, &ntotaltau,
&ngoodtau, status );
percentgood = 100.0 * ((double)ngoodtau / (double)ntotaltau);
if ( percentgood > 80.0) {
tausrc = SMF__TAUSRC_WVMRAW;
msgOutiff( MSG__VERB, "", "Selecting WVM mode for extinction correction of ^FILE."
" %.0f %% of WVM data are present", status, percentgood );
*thetausrc = tausrc;
} else {
tausrc = SMF__TAUSRC_AUTO; /* keep it AUTO (a no-op but make it clear) */
if (wvmtau) wvmtau = astFree( wvmtau );
}
}
/* at this point we either have WVM data handled or we still think we are AUTO.
Do a CSO FIT check */
if (tausrc == SMF__TAUSRC_AUTO && *status == SAI__OK) {
size_t nframes = 0;
smf_calc_csofit( data, extpars, &wvmtau, &nframes, status );
if (*status == SAI__OK) {
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__QUIET, "", "Using CSO fits for extinction correction of ^FILE",
status );
/* Rebrand as WVM data from this point on */
tausrc = SMF__TAUSRC_WVMRAW;
*thetausrc = SMF__TAUSRC_CSOFIT;
} else if (*status == SMF__BADFIT) {
/* No fit, carry on. */
errAnnul( status );
}
}
/* At this point if we are not WVMRAW then we have a serious issue. It means that
WVM was unusable and we did not have a good CSO fit. We should not continue at this
point as to continue implies that we know what we should do. The user should decide
how much they trust the opacity for the night. There has to be a reason why there
is no CSO fit for the night. */
if (*status == SAI__OK && tausrc != SMF__TAUSRC_WVMRAW) {
*status = SAI__ERROR;
errRep("", "Unable to determine opacity data for this observation. Both WVM and CSO fits failed. Please investigate and if necessary use CSO mode explicitly but proceed with caution.", status );
}
}
}
/* If we have a CSO Tau then convert it to the current filter. This will also
convert bad values to a value derived from the header if appropriate. */
if ( tausrc == SMF__TAUSRC_CSOTAU ) {
tau = smf_cso2filt_tau( hdr, tau, extpars, status );
/* The tau source is now a real tau */
tausrc = SMF__TAUSRC_TAU;
}
/* Find the airmass range for this data */
smf_find_airmass_interval( hdr, &amstart, &amend, &elstart, &elend, status );
if (*status == SAI__OK && (amstart == VAL__BADD || amend == VAL__BADD)) {
*status = SAI__ERROR;
errRep( "", "No good airmass values found in JCMTSTATE structure for these data",
status );
}
/* if we are not doing WVM correction but are in adaptive mode we can determine
whether or not we will have to use full or single mode just by looking at the
airmass data. */
if (ndims == 3 && tausrc != SMF__TAUSRC_WVMRAW && method == SMF__EXTMETH_ADAPT) {
/* first and last is a good approximation given that most SCUBA-2 files will only
be a minute duration. */
double refel;
double refam;
/* only need to examine the largest airmass */
if (amstart > amend) {
refam = amstart;
refel = elstart;
} else {
refam = amend;
refel = elend;
}
/* and choose a correction method */
if (is_large_delta_atau( refam, refel, tau, status) ) {
method = SMF__EXTMETH_FULL;
msgOutiff(MSG__DEBUG, " ",
"Adaptive extinction algorithm selected per-bolometer airmass value "
"per time slice (am=%g, tau=%g)", status, refam, tau);
} else {
msgOutiff(MSG__DEBUG, " ",
"Adaptive extinction algorithm selected single airmass value per time slice"
" (am=%g, tau=%g)", status, refam, tau);
method = SMF__EXTMETH_SINGLE;
}
}
/* Assign pointer to input data array if status is good */
if ( *status == SAI__OK ) {
indata = (data->pntr)[0];
vardata = (data->pntr)[1];
}
/* Jump to the cleanup section if status is bad by this point
since we need to free memory */
if (*status != SAI__OK) goto CLEANUP;
/* Array bounds for astTranGrid call */
lbnd[0] = 1;
lbnd[1] = 1;
ubnd[0] = nx;
ubnd[1] = ny;
/* Unlock the AST objects in the smfData so that the worker threads can
lock them. */
smf_lock_data( data, 0, status );
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many frames to process in each worker thread. */
framestep = nframes/nw;
if( framestep == 0 ) {
framestep = 1;
nw = nframes;
}
/* Allocate job data for threads, and store the range of frames to be
processed by each one. Ensure that the last thread picks up any
left-over frames. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->f1 = iw*framestep;
if( iw < nw - 1 ) {
pdata->f2 = pdata->f1 + framestep - 1;
} else {
pdata->f2 = nframes - 1 ;
}
pdata->nframes = nframes;
pdata->npts = npts;
pdata->allextcorr = allextcorr;
pdata->indata = indata;
pdata->tau = tau;
pdata->vardata = vardata;
pdata->wvmtau = wvmtau;
pdata->amstart = amstart;
pdata->amfirst = amstart + ( amend - amstart )*pdata->f1/( nframes - 1 );
pdata->lbnd = lbnd;
pdata->ubnd = ubnd;
pdata->isTordered = isTordered;
pdata->ndims = ndims;
pdata->data = data;
pdata->hdr = hdr;
pdata->method = method;
pdata->tausrc = tausrc;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_correct_extinction, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Record if all time slices used a single air mass. */
allquick = 1;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
if( ! pdata->allquick ) {
allquick = 0;
break;
}
}
/* Free the job data. */
job_data = astFree( job_data );
}
/* Lock the AST objects in the smfData for use by this thread. */
smf_lock_data( data, 1, status );
/* Add history entry if !allextcorr */
if( (*status == SAI__OK) && !allextcorr ) {
smf_history_add( data, FUNC_NAME, status);
}
CLEANUP:
if (wvmtaucache) {
if (!*wvmtaucache) {
*wvmtaucache = wvmtau;
}
} else {
wvmtau = astFree( wvmtau );
}
return allquick;
}
void smf_fillgaps( ThrWorkForce *wf, smfData *data,
smf_qual_t mask, int *status ) {
/* Local Variables */
const gsl_rng_type *type; /* GSL random number generator type */
dim_t bpt; /* Number of bolos per thread */
dim_t i; /* Bolometer index */
dim_t nbolo; /* Number of bolos */
dim_t ntslice; /* Number of time slices */
double *dat=NULL; /* Pointer to bolo data */
int fillpad; /* Fill PAD samples? */
size_t bstride; /* bolo stride */
size_t pend; /* Last non-PAD sample */
size_t pstart; /* First non-PAD sample */
size_t tstride; /* time slice stride */
smfFillGapsData *job_data; /* Structures holding data for worker threads */
smfFillGapsData *pdata; /* Pointer to data for next worker thread */
smf_qual_t *qua=NULL; /* Pointer to quality array */
/* Main routine */
if (*status != SAI__OK) return;
/* Check we have double precision data floating point data. */
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status )) return;
/* Pointers to data and quality */
dat = data->pntr[0];
qua = smf_select_qualpntr( data, NULL, status );
if( !qua ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No valid QUALITY array was provided", status );
return;
}
if( !dat ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData does not contain a DATA component",status);
return;
}
/* obtain data dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &bstride, &tstride,
status );
/* Determine how many bolometers to process in each thread, and create
the structures used to pass data to the threads. */
if( wf ) {
bpt = nbolo/wf->nworker;
if( wf->nworker*bpt < nbolo ) bpt++;
job_data = astMalloc( sizeof( smfFillGapsData )*wf->nworker );
} else {
bpt = nbolo;
job_data = astMalloc( sizeof( smfFillGapsData ) );
}
/* Find the indices of the first and last non-PAD sample. */
smf_get_goodrange( qua, ntslice, tstride, SMF__Q_PAD, &pstart, &pend,
status );
/* Report an error if it is too short. */
if( pend - pstart <= 2*BOX && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": length of data (%d samples) is too small "
"to fill gaps. Must have at least %d samples per bolometer.",
status, (int) ( pend - pstart ), 2*BOX + 1 );
}
/* If the supplied "mask" value includes SMF__Q_PAD, then we will be
replacing the zero-padded region at the start and end of each time series
with artificial noisey data that connects the first and last data values
smoothly. Remove SMF__Q_PAD from the mask. */
if( mask & SMF__Q_PAD ) {
mask &= ~SMF__Q_PAD;
fillpad = 1;
} else {
fillpad = 0;
}
/* Get the default GSL randim number generator type. A separate random
number generator is used for each worker thread so that the gap filling
process does not depend on the the order in which threads are
executed. */
type = gsl_rng_default;
/* Begin a job context. */
thrBeginJobContext( wf, status );
/* Loop over bolometer in groups of "bpt". */
pdata = job_data;
for( i = 0; i < nbolo; i += bpt, pdata++ ) {
/* Store information for this group in the next smfFillGapsData
structure. */
pdata->ntslice = ntslice;
pdata->dat = dat;
pdata->r = gsl_rng_alloc( type );
pdata->b1 = i;
pdata->b2 = i + bpt - 1;
pdata->pend = pend;
pdata->fillpad = fillpad;
pdata->pstart = pstart;
if( pdata->b2 >= nbolo ) pdata->b2 = nbolo - 1;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->qua = qua;
pdata->mask = mask;
/* Submit a job to the workforce to process this group of bolometers. */
(void) thrAddJob( wf, 0, pdata, smfFillGapsParallel, 0, NULL, status );
}
/* Wait until all jobs in the current job context have completed, and
then end the job context. */
thrWait( wf, status );
thrEndJobContext( wf, status );
/* Free resources. */
if( job_data ) {
pdata = job_data;
for( i = 0; i < nbolo; i += bpt, pdata++ ) {
if( pdata->r ) gsl_rng_free( pdata->r );
}
job_data = astFree( job_data );
}
}
void smf_svd( ThrWorkForce *wf, dim_t n, double *a, double *sigma,
double *u, double eps, int sort, int *status ) {
/* Local Variables */
SmfSvdData *job_data = NULL;
SmfSvdData *pdata = NULL;
dim_t *sobhigh = NULL;
dim_t *soblow = NULL;
dim_t i;
dim_t j;
dim_t k;
dim_t irow;
dim_t iter;
dim_t nbig;
dim_t nsmall;
dim_t nstep;
dim_t p;
dim_t rpb;
dim_t s;
double *aorig;
double sigold;
double delta;
int *dn = NULL;
int *up = NULL;
int converged;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* The number of threads to use. */
p = wf ? wf->nworker : 1;
/* If we have more processors than blocks in the matrix, limit the number
of processors. */
if( 4*p > n ) p = n/4;
if( p == 0 ) {
*status = SAI__ERROR;
errRepf( "", "smf_svd: Too few rows (%zu) in matrix - must "
"be no fewer than 4", status, n );
}
/* Allocate required arrays. */
sobhigh = astMalloc( 2*p*sizeof( *sobhigh ) );
soblow = astMalloc( 2*p*sizeof( *soblow ) );
up = astMalloc( p*sizeof( *up ) );
dn = astMalloc( p*sizeof( *dn ) );
job_data = astMalloc( 2*p*sizeof( *job_data ) );
aorig = u ? astStore( NULL, a, n*n*sizeof(*a) ) : NULL;
/* Check pointers can be used safely. */
if( *status == SAI__OK ) {
/* Decide on the first and last element to be processed by each thread,
for "simple" tasks. At the same time, set up jobs to find the sum
of the squares of all data values (a simple task). */
nstep = (n*n)/p;
for( i = 0; i < p; i++ ) {
pdata = job_data + i;
pdata->i1 = i*nstep;
if( i < p - 1 ){
pdata->i2 = pdata->i1 + nstep - 1;
} else {
pdata->i2 = n*n - 1;
}
pdata->a = a;
pdata->oper = 0;
thrAddJob( wf, 0, pdata, smf1_svd, 0, NULL, status );
}
thrWait( wf, status );
delta = 0.0;
for( i = 0; i < p; i++ ) {
pdata = job_data + i;
delta += pdata->delta;
}
delta *= eps;
/* Set up the "size of block" (sob) arrays: soblow holds the zero-based index
of the first matrix row in each block, and sobhigh holds the zero-based
index of the last matrix row in each block. We want 2*P blocks (i.e. twice
the number of threads). Distribute any left over rows evenly amongst the
blocks so that some blocks have n/2p rows (small blocks), and some have
n/2p+1 rows (big blocks). */
rpb = n/(2*p); /* Nominal number of rows per block */
nbig = n - 2*p*rpb; /* Number of big blocks */
nsmall = 2*p - nbig; /* Number of small blocks */
/* If we have more small blocks than big blocks, we start with "nbig" pairs
of blocks in which the first block is big and the second block is small,
and then pad the end with the surplus number of small blocks. */
if( nbig < nsmall ) {
s = 0;
irow = 0;
for( i = 0; i < nbig; i++ ) {
soblow[ s ] = irow;
irow += rpb;
sobhigh[ s ] = irow;
irow++;
s++;
soblow[ s ] = irow;
irow += rpb - 1;
sobhigh[ s ] = irow;
irow++;
s++;
}
for( i = s; i < 2*p; i++ ) {
soblow[ i ] = irow;
irow += rpb - 1;
sobhigh[ i ] = irow;
irow++;
}
/* If we have more big blocks than small blocks, we start with "nsmall" pairs
of blocks in which the first block is big and the second block is small,
and then pad the end with the surplus number of big blocks. */
} else {
s = 0;
irow = 0;
for( i = 0; i < nsmall; i++ ) {
soblow[ s ] = irow;
irow += rpb;
sobhigh[ s ] = irow;
irow++;
s++;
soblow[ s ] = irow;
irow += rpb - 1;
sobhigh[ s ] = irow;
irow++;
s++;
}
for( i = s; i < 2*p; i++ ) {
soblow[ i ] = irow;
irow += rpb;
sobhigh[ i ] = irow;
irow++;
}
}
/* Sanity check. */
if( ( irow != n || i != 2*p ) && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "smf_svd: Error setting up the SOB arrays.", status );
goto L999;
}
/* Now proceed with the "Block JRS Algorithm" algorithm, as described
in "A Block JRS Algorithm for Highly Parallel Computation of SVDs"
(Soliman, et al, DOI: 10.1007/978-3-540-75444-2_36). Note, integer
counters are one-based in the pseudo-code in the paper, but here
we use zero-based counters as is normal in C. */
for( i = 0; i < p; i++ ) {
up[ i ] = 2*i + 1;
dn[ i ] = 2*i;
}
converged = 0;
while( !converged ) {
converged = 1;
for( s = 0; s < 2*p; s++ ) {
pdata = job_data + s;
pdata->oper = 1;
pdata->soblow = soblow[s];
pdata->sobhigh = sobhigh[s];
pdata->delta = delta;
pdata->a = a;
pdata->n = n;
thrAddJob( wf, 0, pdata, smf1_svd, 0, NULL, status );
}
thrWait( wf, status );
for( s = 0; s < 2*p; s++ ) {
if( !job_data[s].converged ) converged = 0;
}
for( iter = 1; iter < 2*p; iter++ ) {
for( s = 0; s < p; s++ ) {
pdata = job_data + s;
pdata->oper = 2;
pdata->upsoblow = soblow[up[s]];
pdata->upsobhigh = sobhigh[up[s]];
pdata->dnsoblow = soblow[dn[s]];
pdata->dnsobhigh = sobhigh[dn[s]];
thrAddJob( wf, 0, pdata, smf1_svd, 0, NULL, status );
}
thrWait( wf, status );
for( s = 0; s < 2*p; s++ ) {
if( !job_data[s].converged ) converged = 0;
}
smf1_roundrobin( p, up, dn );
}
}
nstep = n/p;
for( i = 0; i < p; i++ ) {
pdata = job_data + i;
pdata->j1 = i*nstep;
if( i < p - 1 ){
pdata->j2 = pdata->j1 + nstep - 1;
} else {
pdata->j2 = n - 1;
}
pdata->sigma = sigma;
pdata->oper = 3;
thrAddJob( wf, 0, pdata, smf1_svd, 0, NULL, status );
}
thrWait( wf, status );
/* If required, sort the singular values into descending order. */
if( sort ) {
Sigma_array = sigma;
double *arowold = astMalloc( n*sizeof( *arowold ) );
int *index = astMalloc( n*sizeof( *index ) );
if( *status == SAI__OK ) {
for( i = 0; i < n; i++ ) index[ i ] = i;
qsort( index, n, sizeof(*index), smf1_compare );
for( i = 0; i < n; i++ ) {
sigold = sigma[ i ];
memcpy( arowold, a + i*n, n*sizeof(*a) );
j = i;
while( 1 ) {
k = index[ j ];
index[ j ] = j;
if( k == i ) break;
sigma[ j ] = sigma[ k ];
memcpy( a + j*n, a + k*n, n*sizeof(*a) );
j = k;
}
sigma[ j ] = sigold;
memcpy( a + j*n, arowold, n*sizeof(*a) );
}
}
index = astFree( index );
arowold = astFree( arowold );
}
/* If required, calculate the U matrix. */
if( u ) {
for( i = 0; i < p; i++ ) {
pdata = job_data + i;
pdata->u = u;
pdata->aorig = aorig;
pdata->oper = 4;
thrAddJob( wf, 0, pdata, smf1_svd, 0, NULL, status );
}
thrWait( wf, status );
}
}
/* Free resources. */
L999:
job_data = astFree( job_data );
up = astFree( up );
dn = astFree( dn );
soblow = astFree( soblow );
sobhigh = astFree( sobhigh );
aorig = astFree( aorig );
}
void * smf_dataOrder_array( ThrWorkForce *wf, void * oldbuf, smf_dtype oldtype,
smf_dtype newtype, size_t ndata, size_t ntslice,
size_t nbolo, size_t tstr1, size_t bstr1,
size_t tstr2, size_t bstr2, int inPlace,
int freeOld, int * status ) {
size_t sznew = 0; /* Size of new data type */
void * newbuf = NULL; /* Space to do the reordering */
void * retval = NULL; /* Return value with reordered buffer */
SmfDataOrderArrayData *job_data = NULL;
SmfDataOrderArrayData *pdata;
int nw;
size_t step;
int iw;
retval = oldbuf;
if (*status != SAI__OK) return retval;
if (!retval) return retval;
/* Can't do inPlace without realloc'ing if data types don't match...
so generate bad status for now */
if( (newtype != oldtype) && inPlace ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": inPlace not supported if newtype != oldtype",
status );
return retval;
}
/* For now the only data conversion that is supported is from SMF__INTEGER
to SMF__DOUBLE */
if( (oldtype!=newtype) &&
((newtype!=SMF__DOUBLE) || (oldtype!=SMF__INTEGER)) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": type conversion only supported from integer -> double",
status );
return retval;
}
/* Special case the inPlace variant with no reordering / typecasting */
if ( inPlace && tstr1 == tstr2 && bstr1 == bstr2 && oldtype == newtype ) {
return retval;
}
/* Size of data type */
sznew = smf_dtype_sz(newtype, status);
/* Allocate buffer */
newbuf = astMalloc( ndata*sznew );
if( *status == SAI__OK ) {
/* if the input and output strides are the same, and the data types are
the same, we just memcpy */
if ( tstr1 == tstr2 && bstr1 == bstr2 && oldtype == newtype ) {
memcpy( newbuf, oldbuf, sznew*ndata );
} else {
/* We currently ignore any supplied WorkForce, since using multiple
threads slows down the re-ordered rather than speeding it up - I
presume because of contention issues. I've tried ensuring that
the output array is accessed sequenctially but that does not seem
to improve anything. Leave the mult-threaded infrastructure
here, in case a better solution is found. */
wf = NULL;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many time slices to process in each worker thread. */
step = ntslice/nw;
if( ntslice == 0 ) step = 1;
/* Allocate job data for threads, and store common values. Ensure that
the last thread picks up any left-over time slices. Store the
info required by the wrker threads, then submit jobs to the work
force. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->itime1 = iw*step;
if( iw < nw - 1 ) {
pdata->itime2 = pdata->itime1 + step - 1;
} else {
pdata->itime2 = ntslice - 1 ;
}
pdata->nbolo = nbolo;
pdata->bstr1 = bstr1;
pdata->tstr1 = tstr1;
pdata->bstr2 = bstr2;
pdata->tstr2 = tstr2;
pdata->newtype = newtype;
pdata->oldtype = oldtype;
pdata->newbuf = newbuf;
pdata->oldbuf = oldbuf;
thrAddJob( wf, 0, pdata, smf1_dataOrder_array, 0, NULL, status );
}
/* Wait for the jobs to complete. */
thrWait( wf, status );
}
job_data = astFree( job_data );
}
if( inPlace ) {
/* Copy newbuf to oldbuf */
memcpy( oldbuf, newbuf, ndata*sznew );
/* Free newbuf */
newbuf = astFree( newbuf );
retval = oldbuf;
} else {
if( freeOld ) {
/* Free oldbuf */
oldbuf = astFree( oldbuf );
}
/* Set pntr to newbuf */
retval = newbuf;
}
}
return retval;
}
void smf_update_quality( ThrWorkForce *wf, smfData *data, int syncbad,
const int *badmask, smf_qual_t addqual, double badfrac,
int *status ) {
dim_t nbolo; /* Number of bolometers */
dim_t ndata; /* Number of data points */
dim_t ntslice; /* Number of time slices */
size_t bstride; /* bol stride */
size_t tstride; /* time slice stride */
smf_qual_t *qual=NULL; /* Pointer to the QUALITY array */
SmfUpdateQualityData *job_data = NULL;
SmfUpdateQualityData *pdata;
int nw;
size_t istep;
size_t bstep;
int iw;
if ( *status != SAI__OK ) return;
/* Check for QUALITY */
qual = smf_select_qualpntr( data, NULL, status );
if (!qual) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "smfData does not contain a QUALITY component",
status);
}
return;
}
/* Check for DATA */
if( !data->pntr[0] ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "smfData does not contain a DATA component", status );
return;
}
/* Check for valid badfrac */
if( (badfrac < 0) || (badfrac > 1) ) {
msgSeti( "BADFRAC", badfrac );
errRep(FUNC_NAME,
"Invalid badfrac: ^BADFRAC. Must be in range (0 -- 1).", status);
}
/* Calculate data dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride,
&tstride, status );
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many elements to process in each worker thread. */
istep = ndata/nw;
if( istep == 0 ) istep = 1;
/* Find how many bolometers to process in each worker thread. */
bstep = nbolo/nw;
if( bstep == 0 ) bstep = 1;
/* Allocate job data for threads, and store common values. Ensure that the
last thread picks up any left-over elements or bolometers. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->i1 = iw*istep;
pdata->b1 = iw*bstep;
if( iw < nw - 1 ) {
pdata->i2 = pdata->i1 + istep - 1;
pdata->b2 = pdata->b1 + bstep - 1;
} else {
pdata->i2 = ndata - 1;
pdata->b2 = nbolo - 1;
}
pdata->qual = qual;
}
}
if( *status == SAI__OK ) {
/* some pointers to the data array if needed */
double * ddata = NULL;
int * idata = NULL;
/* we will need the data array if we are checking it for bad values
or looking for bad fraction */
if (syncbad || badfrac) {
smf_select_pntr( data->pntr, data->dtype, &ddata, NULL,
&idata, NULL, status);
}
/* Synchronize SMF__Q_BADDA quality and VAL__BADD in data array */
if( syncbad ) {
if (data->dtype == SMF__DOUBLE) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 1;
pdata->ddata = ddata;
thrAddJob( wf, 0, pdata, smf1_update_quality, 0, NULL, status );
}
thrWait( wf, status );
} else if (data->dtype == SMF__INTEGER) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 2;
pdata->idata = idata;
thrAddJob( wf, 0, pdata, smf1_update_quality, 0, NULL, status );
}
thrWait( wf, status );
} else {
msgSetc( "TYP", smf_dtype_string( data, status ));
*status = SAI__ERROR;
errRep( "",FUNC_NAME " data is of unsupported type (^TYP)",
status);
return;
}
}
/* Apply badmask if available */
if( badmask || badfrac ) {
/* calculate the badfraction threshold in terms of number of bad
found so that we do not have to continually divide to calculate
the current fraction */
dim_t badthresh = ntslice;
/* special case 0 */
if (badfrac) badthresh = badfrac * (double)ntslice;
/* Submit the jobs and wait for them all to finish. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 3;
pdata->badthresh = badthresh;
pdata->addqual = addqual;
pdata->ntslice = ntslice;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->badmask = badmask;
pdata->syncbad = syncbad;
pdata->idata = idata;
pdata->ddata = ddata;
thrAddJob( wf, 0, pdata, smf1_update_quality, 0, NULL, status );
}
thrWait( wf, status );
}
}
job_data = astFree( job_data );
}
void smf_collapse( ThrWorkForce *wf, const double *array, const smf_qual_t *qua,
const smf_qual_t mask, dim_t nx, dim_t ny, dim_t nz, int axis,
double **mean, double **sigma, int **ngood, int *status ){
/* Local Variables */
dim_t bstep;
dim_t nb1;
dim_t nb2;
dim_t nbolo;
dim_t ntime;
double *wmean;
double *wsigma;
int *wngood;
int iworker;
int nworker;
size_t bstride1;
size_t bstride2;
size_t tstride;
smfCollapseJobData *job_data;
smfCollapseJobData *pdata;
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Get the dimensions of hte returned arrays, and the strides between
column elements n the 3D array. We refer to the collapse axis as the
"time" axis and the others as the "bolometer" axes. */
if( axis == 2 ) {
nb1 = nx;
nb2 = ny;
ntime = nz;
bstride1 = 1;
bstride2 = nx;
tstride = nx*ny;
} else if( axis == 1 ) {
nb1 = nx;
nb2 = nz;
ntime = ny;
bstride1 = 1;
bstride2 = nx*ny;
tstride = nx;
} else if( axis == 0 ) {
nb1 = ny;
nb2 = nz;
ntime = nx;
bstride1 = nx;
bstride2 = nx*ny;
tstride = 1;
} else {
nb1 = 0;
nb2 = 0;
ntime = 0;
bstride1 = 0;
bstride2 = 0;
tstride = 0;
*status = SAI__ERROR;
errRepf( "", "smf_collapse: Supplied axis index (%d) is illegal - must "
"be 0, 1 or 2.", status, axis );
}
/* Allocate the 2D arrays to return. */
nbolo = nb1*nb2;
if( mean && *mean ) {
wmean = *mean;
} else {
wmean = astMalloc( nbolo*sizeof( *wmean ) );
}
if( sigma && *sigma ) {
wsigma = *sigma;
} else {
wsigma = astMalloc( nbolo*sizeof( *wsigma ) );
}
if( ngood && *ngood ) {
wngood = *ngood;
} else {
wngood = astMalloc( nbolo*sizeof( *wngood ) );
}
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
/* Determine which bolometers are to be processed by which threads. */
if( *status == SAI__OK ) {
bstep = nbolo/nworker;
if( bstep < 1 ) bstep = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->b1 = iworker*bstep;
if( iworker < nworker - 1 ) {
pdata->b2 = pdata->b1 + bstep - 1;
} else {
pdata->b2 = nbolo - 1;
}
pdata->array = array;
pdata->qua = qua;
pdata->mean = wmean;
pdata->sigma = wsigma;
pdata->ngood = wngood;
pdata->nb1 = nb1;
pdata->ntime = ntime;
pdata->bstride1 = bstride1;
pdata->bstride2 = bstride2;
pdata->tstride = tstride;
pdata->mask = mask;
thrAddJob( wf, 0, pdata, smf1_collapse_job, 0, NULL, status );
}
thrWait( wf, status );
}
/* Return any requested arrays, and free the others. */
if( mean ){
if( *mean == NULL ) *mean = wmean;
} else {
wmean = astFree( wmean );
}
if( sigma ){
if( *sigma == NULL ) *sigma = wsigma;
} else {
wsigma = astFree( wsigma );
}
if( ngood ){
if( *ngood == NULL ) *ngood = wngood;
} else {
wngood = astFree( wngood );
}
}
void smf_calc_iqu( ThrWorkForce *wf, smfData *data, int block_start,
int block_end, int ipolcrd, int qplace, int uplace,
int iplace, NdgProvenance *oprov, AstFitsChan *fc,
int pasign, double paoff, double angrot, int submean,
int harmonic, int *status ){
/* Local Variables: */
AstCmpMap *cm1;
AstCmpMap *cm2;
AstFrameSet *wcs; /* WCS FrameSet for output NDFs */
AstMapping *fpmap1;
AstMapping *fpmap2;
AstMapping *oskymap;
AstMapping *totmap;
AstSkyFrame *oskyfrm;
AstWinMap *wm; /* Mapping to reverse the X GRID axis */
const JCMTState *state; /* JCMTState info for current time slice */
const char *usesys; /* Used system string */
dim_t itime; /* Time slice index */
dim_t nbolo; /* No. of bolometers */
dim_t ncol; /* No. of columns of bolometers */
dim_t nrow; /* No. of rows of bolometers */
dim_t ntime; /* Time slices to check */
dim_t ntslice; /* Number of time-slices in data */
double *ipi; /* Pointer to output I array */
double *ipiv; /* Pointer to output I variance array */
double *ipq; /* Pointer to output Q array */
double *ipqv; /* Pointer to output Q variance array */
double *ipu; /* Pointer to output U array */
double *ipuv; /* Pointer to output U variance array */
double *mean;
double ang_data[2];
double fox[2];
double foy[2];
double fpr0;
double fprinc;
double fx[2];
double fy[2];
double ina[ 2 ]; /* Bolometer coords at bottom left */
double inb[ 2 ]; /* Bolometer coords at top right */
double outa[ 2 ]; /* NDF GRID coords at bottom left */
double outb[ 2 ]; /* NDF GRID coords at top right */
int bstep; /* Bolometer step between threads */
int el; /* Number of mapped array elements */
int gotvar; /* Were any output variances created? */
int indfi; /* Identifier for NDF holding I values */
int indfq; /* Identifier for NDF holding Q values */
int indfu; /* Identifier for NDF holding Q values */
int iworker; /* Index of a worker thread */
int lbnd[ 2 ]; /* Lower pixel bounds of output NDF */
int moving;
int nworker; /* No. of worker threads */
int old; /* Data has old-style POL_ANG values? */
int tstep; /* Time slice step between threads */
int ubnd[ 2 ]; /* Upper pixel bounds of output NDF */
size_t bstride; /* Stride between adjacent bolometer values */
size_t tstride; /* Stride between adjacent time slice values */
smfCalcIQUJobData *job_data = NULL; /* Pointer to all job data */
smfCalcIQUJobData *pdata = NULL;/* Pointer to next job data */
smfHead *hdr; /* Pointer to data header this time slice */
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Convenience pointers. */
hdr = data->hdr;
/* Obtain number of time slices - will also check for 3d-ness. Also get
the dimensions of the bolometer array and the strides between adjacent
bolometer values. */
smf_get_dims( data, &nrow, &ncol, &nbolo, &ntslice, NULL, &bstride,
&tstride, status );
/* Report an error if the block of time slices extends of either end. */
if( block_start < 0 || block_end >= (int) ntslice ) {
if( *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "S", block_start );
msgSeti( "E", block_end );
msgSeti( "N", ntslice );
errRep( " ", "smf_calc_iqu: invalid block of time slices - ^S to "
"^E (^N time slices are available).", status );
}
}
/* Create the output NDFs. Each one is a 2D array with dimensions
equal to the bolometer array. */
lbnd[ 0 ] = 1;
lbnd[ 1 ] = 1;
ubnd[ 0 ] = ncol;
ubnd[ 1 ] = nrow;
ndfNew( "_DOUBLE", 2, lbnd, ubnd, &qplace, &indfq, status );
ndfNew( "_DOUBLE", 2, lbnd, ubnd, &uplace, &indfu, status );
if( iplace != NDF__NOPL ) {
ndfNew( "_DOUBLE", 2, lbnd, ubnd, &iplace, &indfi, status );
} else {
indfi = NDF__NOID;
}
/* Store any supplied provenance in all NDFs. */
if( oprov ) {
ndgWriteProv( oprov, indfq, 1, status );
ndgWriteProv( oprov, indfu, 1, status );
if( indfi != NDF__NOID ) ndgWriteProv( oprov, indfi, 1, status );
}
/* Store any supplied FITS headers in all NDFs.*/
if( fc && astGetI( fc, "NCard" ) > 0 ) {
kpgPtfts( indfq, fc, status );
kpgPtfts( indfu, fc, status );
if( indfi != NDF__NOID ) kpgPtfts( indfi, fc, status );
}
/* Store the WCS frameSet in all NDFs. First get the FrameSet for the
central time slice in the block, and set its current Frame to the
tracking frame. */
smf_tslice_ast( data, ( block_start + block_end )/2, 1, NO_FTS, status);
usesys = sc2ast_convert_system( (data->hdr->allState)[0].tcs_tr_sys,
status );
astSetC( hdr->wcs, "System", usesys );
/* Get the Mapping from focal plane coords to bolometer grid coords. This
is the same for all time slices. sc2ast ensures that frame 3 is FPLANE. */
fpmap1 = astGetMapping( hdr->wcs, 3, AST__BASE );
/* Take a copy and then reverse the X axis of the GRID Frame by remaping the
base Frame using a WinMap. This produces a pixel grid such as you would
see by looking up at the sky from underneath the array, rather than looking
down at the ground from above the array. */
wcs = astCopy( hdr->wcs );
ina[ 0 ] = 1.0;
inb[ 0 ] = ncol;
ina[ 1 ] = 1.0;
inb[ 1 ] = nrow;
outa[ 0 ] = ncol;
outb[ 0 ] = 1.0;
outa[ 1 ] = 1.0;
outb[ 1 ] = nrow;
wm = astWinMap( 2, ina, inb, outa, outb, " " );
astRemapFrame( wcs, AST__BASE, wm );
wm = astAnnul( wm );
/* Get the Mapping from output grid coords to focal plane coords. */
fpmap2 = astGetMapping( wcs, AST__BASE, 3 );
/* If the target is moving (assumed to be the case if the tracking
system is AZEL or GAPPT), make the FrameSet current Frame represent
offsets from the reference position (i.e. the moving target), and
indicate that the offset coord system should be used for alignment. */
if( !strcmp( usesys, "AZEL" ) || !strcmp( usesys, "GAPPT" ) ){
astSet( wcs, "SkyRefIs=Origin,AlignOffset=1" );
moving = 1;
} else {
moving = 0;
}
/* Store the FrameSet in the output NDFs. */
ndfPtwcs( wcs, indfq, status );
ndfPtwcs( wcs, indfu, status );
if( indfi != NDF__NOID ) ndfPtwcs( wcs, indfi, status );
/* Map the Data array in each NDF. */
ndfMap( indfq, "Data", "_DOUBLE", "WRITE", (void **) &ipq, &el, status );
ndfMap( indfu, "Data", "_DOUBLE", "WRITE", (void **) &ipu, &el, status );
if( indfi != NDF__NOID ) {
ndfMap( indfi, "Data", "_DOUBLE", "WRITE", (void **) &ipi, &el, status );
} else {
ipi = NULL;
}
/* Map the Variance array in each NDF. */
ndfMap( indfq, "Variance", "_DOUBLE", "WRITE", (void **) &ipqv, &el, status );
ndfMap( indfu, "Variance", "_DOUBLE", "WRITE", (void **) &ipuv, &el, status );
if( indfi != NDF__NOID ) {
ndfMap( indfi, "Variance", "_DOUBLE", "WRITE", (void **) &ipiv, &el, status );
} else {
ipiv = NULL;
}
/* If required, allocate memory to hold the mean bolometer value at each
time slice. */
mean = submean ? astMalloc( ntslice*sizeof( *mean ) ) : NULL;
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
/* Check the above pointers can be used safely. */
if( *status == SAI__OK ) {
/* Go through the first thousand POL_ANG values to see if they are in
units of radians (new data) or arbitrary encoder units (old data).
They are assumed to be in radians if no POL_ANG value is larger than
20. */
old = 0;
state = hdr->allState;
ntime = ( ntslice > 1000 ) ? 1000 : ntslice;
for( itime = 0; itime < ntime; itime++,state++ ) {
if( state->pol_ang > 20 ) {
old = 1;
msgOutif( MSG__VERB, ""," POL2 data contains POL_ANG values "
"in encoder units - converting to radians.", status );
break;
}
}
/* If required, find the mean bolometer value at each time slice. */
if( submean ) {
/* Determine which time-slices are to be processed by which threads. */
tstep = ntslice/nworker;
if( tstep < 1 ) tstep = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->block_start = iworker*tstep;
if( iworker < nworker - 1 ) {
pdata->block_end = pdata->block_start + tstep - 1;
} else {
pdata->block_end = ntslice - 1;
}
}
/* Store all the other info needed by the worker threads, and submit the
jobs to calculate the Q and U values in each bolo, and then wait for
them to complete. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->bstride = bstride;
pdata->dat = data->pntr[0];
pdata->nbolo = nbolo;
pdata->qua = smf_select_qualpntr( data, NULL, status );;
pdata->tstride = tstride;
pdata->mean = mean;
pdata->action = 1;
/* Pass the job to the workforce for execution. */
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_calc_iqu_job, 0, NULL,
status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
}
/* Get the Frame representing absolute sky coords in the output NDF,
and the Mapping from sky to grid in the output NDF. */
oskyfrm = astCopy( astGetFrame( wcs, AST__CURRENT ) );
astSet( oskyfrm, "SkyRefIs=Ignored" );
oskymap = astGetMapping( wcs, AST__CURRENT, AST__BASE );
wcs = astAnnul( wcs );
/* Find the first and last time slices, calculate the angle between the
focal pane Y axis at the time slice, and the focal plane Y axis in
the output NDF. For intervening time-slices, the angle is found by
linear interpolation between the extreme time slices. */
for( el = 0; el < 2; el++ ) {
/* Get the mapping from GRID coords in the input time slice to GRID
coords in the output. */
totmap = smf_rebin_totmap( data, el?ntslice-1:0, oskyfrm, oskymap,
moving, NO_FTS, status );
/* Modify it to be the Mapping from focal plane coords in the input time
slice to focal plane coords in the output. */
cm1 = astCmpMap( fpmap1, totmap, 1, " " );
cm2 = astCmpMap( cm1, fpmap2, 1, " " );
/* Use this Mapping to convert two points on the focal plane Y axis from
the input to the output. */
fx[0] = 0.0;
fy[0] = 0.0;
fx[1] = 0.0;
fy[1] = 4.0;
astTran2( cm2, 2, fx, fy, 1, fox, foy );
/* The angle from the focal plane Y axis in the output to the focal plane
Y axis in the input time slice, measured positive in sense of rotation
from Fy to Fx. */
ang_data[ el ] = atan2( fox[1]-fox[0], foy[1]-foy[0] );
/* Free resources for this time slice. */
totmap = astAnnul( totmap );
cm1 = astAnnul( cm1 );
cm2 = astAnnul( cm2 );
}
/* Annul objects. */
oskymap = astAnnul( oskymap );
oskyfrm = astAnnul( oskyfrm );
fpmap1 = astAnnul( fpmap1 );
fpmap2 = astAnnul( fpmap2 );
/* Get the constants of the linear relationship between focal plane
rotation and time slice index "fpr = fpr0 + itime*fprinc". */
fpr0 = ang_data[ 0 ];
fprinc = ( ang_data[ 1 ] - fpr0 )/( ntslice - 1 );
/* Determine which bolometers are to be processed by which threads. */
bstep = nbolo/nworker;
if( bstep < 1 ) bstep = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->b1 = iworker*bstep;
pdata->b2 = pdata->b1 + bstep - 1;
}
/* Ensure that the last thread picks up any left-over bolometers */
pdata->b2 = nbolo - 1;
/* Store all the other info needed by the worker threads, and submit the
jobs to calculate the Q and U values in each bolo, and then wait for
them to complete. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->bstride = bstride;
pdata->dat = data->pntr[0];;
pdata->nbolo = nbolo;
pdata->qua = smf_select_qualpntr( data, NULL, status );;
pdata->tstride = tstride;
pdata->allstates = hdr->allState;
pdata->ipq = ipq;
pdata->ipu = ipu;
pdata->ipi = ipi;
pdata->ipqv = ipqv;
pdata->ipuv = ipuv;
pdata->ipiv = ipiv;
pdata->ipolcrd = ipolcrd;
pdata->block_start = block_start;
pdata->block_end = block_end;
pdata->old = old;
pdata->ncol = ncol;
pdata->pasign = pasign ? +1: -1;
pdata->paoff = paoff;
pdata->angrot = angrot;
pdata->fpr0 = fpr0;
pdata->fprinc = fprinc;
pdata->angfac = harmonic/4.0;
pdata->action = 0;
pdata->mean = mean;
/* Pass the job to the workforce for execution. */
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_calc_iqu_job, 0, NULL,
status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
/* See if any thread produced non-bad variance values. */
gotvar = 0;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
if( pdata->gotvar ) gotvar = 1;
}
/* If no variances were created, erase the Variance component and tell
the user. */
ndfUnmap( indfq, "*", status );
ndfUnmap( indfu, "*", status );
if( ipi ) ndfUnmap( indfi, "*", status );
if( !gotvar ) {
ndfReset( indfq, "Variance", status );
ndfReset( indfu, "Variance", status );
if( ipi ) ndfReset( indfi, "Variance", status );
msgOut( "", "Warning: Insufficient input data to produce variances",
status );
}
}
/* Add POLANAL Frames to the WCS FrameSet in each output NDF. This Frame
is used by POLPACK to determine the reference direction of the Stokes
vectors (focal plane Y in this case, i.e. zero-based axis 1 ). */
smf_polext( indfq, 0, 0.0, "FPLANE", 1, status );
smf_polext( indfu, 0, 0.0, "FPLANE", 1, status );
if( ipi ) smf_polext( indfi, 0, 0.0, "FPLANE", 1, status );
/* Free the two output NDFs. */
ndfAnnul( &indfq, status );
ndfAnnul( &indfu, status );
if( ipi ) ndfAnnul( &indfi, status );
/* Free other resources. */
job_data = astFree( job_data );
mean = astFree( mean );
}
void smf_calc_smoothedwvm ( ThrWorkForce *wf, const smfArray * alldata,
const smfData * adata, AstKeyMap* extpars, double **wvmtau,
size_t *nelems, size_t *ngoodvals, int * status ) {
size_t i;
size_t nrelated = 0; /* Number of entries in smfArray */
size_t nframes = 0; /* Number of timeslices */
size_t ngood = 0; /* Number of elements with good tau */
double *taudata = NULL; /* Local version of WVM tau */
const smfArray * thesedata = NULL; /* Collection of smfDatas to analyse */
smfArray * tmpthesedata = NULL; /* Local version of adata in a smfArray */
if (*status != SAI__OK) return;
if (alldata && adata) {
*status = SAI__ERROR;
errRep("", "smf_calc_smoothedwvm can not be given non-NULL alldata and non-NULL adata arguments"
" (possible programming error)", status );
return;
}
if (!alldata && !adata) {
*status = SAI__ERROR;
errRep("", "smf_calc_smoothedwvm: One of alldata or adata must be non-NULL",
status);
return;
}
if (!wvmtau) {
*status = SAI__ERROR;
errRep("", "Must supply a non-NULL pointer for wvmtau argument"
" (possible programming error)", status );
return;
}
/* if we have a single smfData put it in a smfArray */
if (alldata) {
if (alldata->ndat == 0 ) {
*status = SAI__ERROR;
errRep("", "No smfDatas present in supplied smfArray for WVM smoothing"
" (possible programming error)", status );
return;
}
thesedata = alldata;
} else {
tmpthesedata = smf_create_smfArray( status );
if (tmpthesedata) {
tmpthesedata->owndata = 0; /*not owned by the smfArray */
/* we know that the smfData here will not be touched in this
function so we do the BAD thing of casting const to non-const */
smf_addto_smfArray( tmpthesedata, (smfData *)adata, status );
}
thesedata = tmpthesedata;
}
/* Check that we have headers and that the smfData are the same length */
nrelated = thesedata->ndat;
for (i = 0; i < nrelated; i++ ) {
smfData * data = (thesedata->sdata)[i];
smfHead * hdr = data->hdr;
dim_t thisframes = 0;
if ( !hdr) {
*status = SAI__ERROR;
errRepf( "", "smfData %zu has no header. Aborting WVM smoothing",
status, i );
return;
}
smf_get_dims( data, NULL, NULL, NULL, &thisframes, NULL, NULL, NULL, status );
if (!nframes) nframes = thisframes;
if (thisframes != nframes) {
*status = SAI__ERROR;
errRepf( "", "smfData %zu has different length. Aborting WVM smoothing",
status, i );
return;
}
}
/* We will need the earliest and last airmass value in order
to calculate a zenith tau */
/* As a first step, just fill the time series with calculated WVM
tau values even though we know there are about 240 fewer tau
readings in reality. This initial approach will make it easier to
use the smoothed data directly rather than having to interpolate
from the 1.2 second data back into the 200 Hz data. */
taudata = astCalloc( nframes, sizeof(*taudata) );
if (*status == SAI__OK) {
double amprev = VAL__BADD;
double steptime;
size_t maxgap;
struct timeval tv1;
struct timeval tv2;
smfCalcWvmJobData *job_data = NULL;
int nworker;
/* We need to know the steptime so we can define the max good gap
in seconds and convert it to steps*/
steptime = (thesedata->sdata)[0]->hdr->steptime;
maxgap = (size_t)( 5.0 / steptime ); /* 5 seconds is just larger than 2 WVM readings */
/* Assume all files have the same airmass information */
smf_find_airmass_interval( (thesedata->sdata)[0]->hdr, &rev, NULL, NULL, NULL, status );
smf_timerinit( &tv1, &tv2, status );
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
if (*status == SAI__OK) {
dim_t tstep;
int iworker;
smfCalcWvmJobData *pdata = NULL;
/* Get the number of time slices to process in each thread. */
if( nworker > (int) nframes ) {
tstep = 1;
} else {
tstep = nframes/nworker;
}
/* to return the same values for one thread and multiple threads
we need to break the threads on wvm sample boundaries wherever
possible. We make an initial estimate of the number of WVM measurements
by assuming one every two seconds. */
{
smfData * curdata = NULL;
size_t nwvm = 0;
double prevtime = VAL__BADD;
double curtime;
size_t *boundaries = astGrow(NULL, nframes*(size_t)(steptime/2.0), sizeof(*boundaries));
for (i=0; i<nframes; i++) {
if (!curdata) {
SELECT_DATA( thesedata, curdata, VAL__BADD, wvm_time, i );
}
if (curdata) smf_tslice_ast( curdata, i, 0, NO_FTS, status );
if ( !curdata || curdata->hdr->state->wvm_time == VAL__BADD ) {
/* Try the other datas */
SELECT_DATA( thesedata, curdata, VAL__BADD, wvm_time, i );
}
if (*status != SAI__OK) break;
if (!curdata) {
curtime = VAL__BADD;
} else {
curtime = curdata->hdr->state->wvm_time;
}
if (curtime != prevtime || nwvm == 0 ) {
/* Store the index in the boundaries array */
nwvm++;
boundaries = astGrow(boundaries, nwvm, sizeof(*boundaries));
if (!boundaries) { /* this is serious */
if (*status == SAI__OK) *status = SAI__ERROR;
errRep("", "Error allocating temporary memory for WVM calculation\n",
status );
break;
}
boundaries[nwvm-1] = i;
prevtime = curtime;
}
}
/* No point using too many threads */
if (*status == SAI__OK) {
if (nworker >= (int)nwvm) {
nworker = nwvm;
/* Allocate a measurement per thread */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->t1 = boundaries[iworker];
if (iworker+1 < nworker) pdata->t2 = boundaries[iworker+1]-1;
}
/* Ensure that the last thread picks up any left-over time slices */
pdata->t2 = nframes - 1;
} else {
/* Allocate the workers to slices of approximate size tstep */
size_t prevend = 0; /* End of previous slice */
size_t prevbnd = 0; /* Index into previous boundaries[] array selection */
for( iworker = 0; iworker < nworker; iworker++ ) {
size_t belowidx = prevend+1;
size_t aboveidx = nframes;
size_t lbnd;
size_t ubnd;
size_t j;
size_t guess;
pdata = job_data + iworker;
if (iworker == 0) { /* always start at the beginning */
pdata->t1 = 0;
} else { /* Start one after the previous block */
pdata->t1 = prevend + 1;
}
/* Now we have to find the end of this slice */
guess = (iworker*tstep) + tstep - 1;
if (guess <= pdata->t1) guess = pdata->t1 + tstep;
/* find nearest boundaries */
for (j=prevbnd; j<nwvm; j++) {
if ( boundaries[j] > guess ) {
aboveidx = boundaries[j];
ubnd = j;
if (j>0) {
belowidx = boundaries[j-1];
lbnd = j -1 ;
} else {
lbnd = 0;
}
break;
}
}
/* Choose the closest, making sure that we are not choosing t1 */
if ( (guess - belowidx < aboveidx - guess) && belowidx > pdata->t1 ) {
pdata->t2 = belowidx - 1;
prevbnd = lbnd;
} else {
pdata->t2 = aboveidx - 1;
prevbnd = ubnd;
}
prevend = pdata->t2;
if (prevend == nframes - 1 && iworker < nworker-1 ) {
/* we have run out of slices so just use fewer workers */
nworker = iworker + 1;
break;
}
}
/* Ensure that the last thread picks up any left-over time slices */
pdata->t2 = nframes - 1;
}
/* Tidy up */
boundaries = astFree( boundaries );
}
}
/* Store all the other info needed by the worker threads, and submit the
jobs to fix the steps in each bolo, and then wait for them to complete. */
for( iworker = 0; iworker < nworker; iworker++ ) {
smfArray *thrdata = NULL;
pdata = job_data + iworker;
pdata->nframes = nframes;
pdata->airmass = amprev; /* really need to get it from the start of each chunk */
pdata->taudata = taudata;
pdata->maxgap = maxgap;
/* Need to copy the smfDatas and create a new smfArray for each
thread */
thrdata = smf_create_smfArray( status );
for (i=0;i<nrelated;i++) {
smfData *tmpdata = NULL;
tmpdata = smf_deepcopy_smfData( wf, (thesedata->sdata)[i], 0, SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS |
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY, 0, 0,
status );
smf_lock_data( tmpdata, 0, status );
smf_addto_smfArray( thrdata, tmpdata, status );
}
pdata->thesedata = thrdata;
/* Need to do a deep copy of ast data and unlock them */
pdata->extpars = astCopy(extpars);
astUnlock( pdata->extpars, 1 );
/* Pass the job to the workforce for execution. */
thrAddJob( wf, THR__REPORT_JOB, pdata, smf__calc_wvm_job, 0, NULL,
status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
/* Now free the resources we allocated during job creation
and calculate the number of good values */
for( iworker = 0; iworker < nworker; iworker++ ) {
smfArray * thrdata;
pdata = job_data + iworker;
astLock( pdata->extpars, 0 );
pdata->extpars = astAnnul( pdata->extpars );
thrdata = pdata->thesedata;
for (i=0;i<thrdata->ndat;i++) {
smf_lock_data( (thrdata->sdata)[i], 1, status );
}
smf_close_related( wf, &thrdata, status );
ngood += pdata->ngood;
}
}
job_data = astFree( job_data );
msgOutiff( MSG__NORM, "", FUNC_NAME ": %f s to calculate unsmoothed WVM tau values",
status, smf_timerupdate(&tv1,&tv2,status) );
}
if (*status == SAI__OK && extpars) {
/* Read extpars to see if we need to smooth */
double smoothtime = VAL__BADD;
if (astMapGet0D( extpars, "SMOOTHWVM", &smoothtime ) ) {
if (smoothtime != VAL__BADD && smoothtime > 0.0) {
smfData * data = (thesedata->sdata)[0];
double steptime = data->hdr->steptime;
dim_t boxcar = (dim_t)( smoothtime / steptime );
msgOutiff( MSG__VERB, "",
"Smoothing WVM data with %f s tophat function",
status, smoothtime );
smf_tophat1D( taudata, nframes, boxcar, NULL, 0, 0.0, status );
/* The tophat smoothing puts a bad value at the start and end of
the time series so we replace that with the adjacent value since
the step time is much smaller than WVM readout time. If more than
one value is bad we do not try to find the good value. */
taudata[0] = taudata[1];
taudata[nframes-1] = taudata[nframes-2];
}
}
}
/* Use this to get the raw WVM output for debugging */
/*
if (*status == SAI__OK) {
smfData *data = (thesedata->sdata)[0];
smfHead *hdr = data->hdr;
printf("# IDX TAU RTS_NUM RTS_END WVM_TIME\n");
for (i=0; i<nframes;i++) {
JCMTState * state;
state = &(hdr->allState)[i];
printf("%zu %.*g %d %.*g %.*g\n", i, DBL_DIG, taudata[i], state->rts_num,
DBL_DIG, state->rts_end, DBL_DIG, state->wvm_time);
}
} */
/* Free resources */
if (tmpthesedata) smf_close_related( wf, &tmpthesedata, status );
if (*status != SAI__OK) {
if (taudata) taudata = astFree( taudata );
*nelems = 0;
*ngoodvals = 0;
} else {
*wvmtau = taudata;
*nelems = nframes;
*ngoodvals = ngood;
}
}
void smf_calc_iqu( ThrWorkForce *wf, smfData *data, int block_start,
int block_end, int ipolcrd, int qplace, int uplace,
int iplace, NdgProvenance *oprov, AstFitsChan *fc,
int pasign, double paoff, double angrot, int submean,
int *status ){
/* Local Variables: */
AstFrameSet *wcs; /* WCS FrameSet for output NDFs */
AstWinMap *wm; /* Mapping to reverse the X GRID axis */
const JCMTState *state; /* JCMTState info for current time slice */
dim_t nbolo; /* No. of bolometers */
dim_t ncol; /* No. of columns of bolometers */
dim_t nrow; /* No. of rows of bolometers */
dim_t ntslice; /* Number of time-slices in data */
double *ipi; /* Pointer to output I array */
double *ipq; /* Pointer to output Q array */
double *ipu; /* Pointer to output U array */
double ina[ 2 ]; /* Bolometer coords at bottom left */
double inb[ 2 ]; /* Bolometer coords at top right */
double outa[ 2 ]; /* NDF GRID coords at bottom left */
double outb[ 2 ]; /* NDF GRID coords at top right */
int bstep; /* Bolometer step between threads */
int el; /* Number of mapped array elements */
int indfi; /* Identifier for NDF holding I values */
int indfq; /* Identifier for NDF holding Q values */
int indfu; /* Identifier for NDF holding Q values */
int itime; /* Time slice index */
int iworker; /* Index of a worker thread */
int lbnd[ 2 ]; /* Lower pixel bounds of output NDF */
int ntime; /* Time slices to check */
int nworker; /* No. of worker threads */
int old; /* Data has old-style POL_ANG values? */
int ubnd[ 2 ]; /* Upper pixel bounds of output NDF */
size_t bstride; /* Stride between adjacent bolometer values */
size_t tstride; /* Stride between adjacent time slice values */
smfCalcIQUJobData *job_data = NULL; /* Pointer to all job data */
smfCalcIQUJobData *pdata = NULL;/* Pointer to next job data */
smfHead *hdr; /* Pointer to data header this time slice */
double *mean;
int tstep; /* Time slice step between threads */
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Convenience pointers. */
hdr = data->hdr;
/* Obtain number of time slices - will also check for 3d-ness. Also get
the dimensions of the bolometer array and the strides between adjacent
bolometer values. */
smf_get_dims( data, &nrow, &ncol, &nbolo, &ntslice, NULL, &bstride,
&tstride, status );
/* Report an error if the block of time slices extends of either end. */
if( block_start < 0 || block_end >= (int) ntslice ) {
if( *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "S", block_start );
msgSeti( "E", block_end );
msgSeti( "N", ntslice );
errRep( " ", "smf_calc_iqu: invalid block of time slices - ^S to "
"^E (^N time slices are available).", status );
}
}
/* Create the output NDFs. Each one is a 2D array with dimensions
equal to the bolometer array. */
lbnd[ 0 ] = 1;
lbnd[ 1 ] = 1;
ubnd[ 0 ] = ncol;
ubnd[ 1 ] = nrow;
ndfNew( "_DOUBLE", 2, lbnd, ubnd, &qplace, &indfq, status );
ndfNew( "_DOUBLE", 2, lbnd, ubnd, &uplace, &indfu, status );
if( iplace != NDF__NOPL ) {
ndfNew( "_DOUBLE", 2, lbnd, ubnd, &iplace, &indfi, status );
} else {
indfi = NDF__NOID;
}
/* Store any supplied provenance in all NDFs. */
if( oprov ) {
ndgWriteProv( oprov, indfq, 1, status );
ndgWriteProv( oprov, indfu, 1, status );
if( indfi != NDF__NOID ) ndgWriteProv( oprov, indfi, 1, status );
}
/* Store any supplied FITS headers in all NDFs.*/
if( fc && astGetI( fc, "NCard" ) > 0 ) {
kpgPtfts( indfq, fc, status );
kpgPtfts( indfu, fc, status );
if( indfi != NDF__NOID ) kpgPtfts( indfi, fc, status );
}
/* Store the WCS frameSet in all NDFs. First get the FrameSet for the
central time slice in the block, and set its current Frame to the
tracking frame. */
smf_tslice_ast( data, ( block_start + block_end )/2, 1, status);
astSetC( hdr->wcs, "System",
sc2ast_convert_system( (data->hdr->allState)[0].tcs_tr_sys,
status ) );
/* Take a copy and then reverse the X axis of the GRID Frame by remaping the
base Frame using a WinMap. This produces a pixel grid such as you would
see by looking up at the sky from underneath the array, rather than looking
down at the ground from above the array. */
wcs = astCopy( hdr->wcs );
ina[ 0 ] = 1.0;
inb[ 0 ] = ncol;
ina[ 1 ] = 1.0;
inb[ 1 ] = nrow;
outa[ 0 ] = ncol;
outb[ 0 ] = 1.0;
outa[ 1 ] = 1.0;
outb[ 1 ] = nrow;
wm = astWinMap( 2, ina, inb, outa, outb, " " );
astRemapFrame( wcs, AST__BASE, wm );
wm = astAnnul( wm );
/* Store the FrameSet in the output NDFs, then annull the copy. */
ndfPtwcs( wcs, indfq, status );
ndfPtwcs( wcs, indfu, status );
if( indfi != NDF__NOID ) ndfPtwcs( wcs, indfi, status );
wcs = astAnnul( wcs );
/* Map the Data array in each NDF. */
ndfMap( indfq, "Data", "_DOUBLE", "WRITE", (void **) &ipq, &el, status );
ndfMap( indfu, "Data", "_DOUBLE", "WRITE", (void **) &ipu, &el, status );
if( indfi != NDF__NOID ) {
ndfMap( indfi, "Data", "_DOUBLE", "WRITE", (void **) &ipi, &el, status );
} else {
ipi = NULL;
}
/* If required, allocate memory to hold the mean bolometer value at each
time slice. */
mean = submean ? astMalloc( ntslice*sizeof( *mean ) ) : NULL;
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
/* Check the above pointers can be used safely. */
if( *status == SAI__OK ) {
/* Go through the first thousand POL_ANG values to see if they are in
units of radians (new data) or arbitrary encoder units (old data).
They are assumed to be in radians if no POL_ANG value is larger than
20. */
old = 0;
state = hdr->allState;
ntime = ( ntslice > 1000 ) ? 1000 : ntslice;
for( itime = 0; itime < ntime; itime++,state++ ) {
if( state->pol_ang > 20 ) {
old = 1;
msgOutif( MSG__VERB, ""," POL2 data contains POL_ANG values "
"in encoder units - converting to radians.", status );
break;
}
}
/* If required, find the mean bolometer value at each time slice. */
if( submean ) {
/* Determine which time-slices are to be processed by which threads. */
tstep = ntslice/nworker;
if( tstep < 1 ) tstep = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->block_start = iworker*tstep;
if( iworker < nworker - 1 ) {
pdata->block_end = pdata->block_start + tstep - 1;
} else {
pdata->block_end = ntslice - 1;
}
}
/* Store all the other info needed by the worker threads, and submit the
jobs to calculate the Q and U values in each bolo, and then wait for
them to complete. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->bstride = bstride;
pdata->dat = data->pntr[0];
pdata->nbolo = nbolo;
pdata->qua = smf_select_qualpntr( data, NULL, status );;
pdata->tstride = tstride;
pdata->mean = mean;
pdata->action = 1;
/* Pass the job to the workforce for execution. */
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_calc_iqu_job, 0, NULL,
status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
}
/* Determine which bolometers are to be processed by which threads. */
bstep = nbolo/nworker;
if( bstep < 1 ) bstep = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->b1 = iworker*bstep;
pdata->b2 = pdata->b1 + bstep - 1;
}
/* Ensure that the last thread picks up any left-over bolometers */
pdata->b2 = nbolo - 1;
/* Store all the other info needed by the worker threads, and submit the
jobs to calculate the Q and U values in each bolo, and then wait for
them to complete. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->bstride = bstride;
pdata->dat = data->pntr[0];;
pdata->nbolo = nbolo;
pdata->qua = smf_select_qualpntr( data, NULL, status );;
pdata->tstride = tstride;
pdata->allstates = hdr->allState;
pdata->ipq = ipq;
pdata->ipu = ipu;
pdata->ipi = ipi;
pdata->ipolcrd = ipolcrd;
pdata->block_start = block_start;
pdata->block_end = block_end;
pdata->old = old;
pdata->ncol = ncol;
pdata->pasign = pasign ? +1: -1;
pdata->paoff = paoff;
pdata->angrot = angrot;
pdata->action = 0;
pdata->mean = mean;
/* Pass the job to the workforce for execution. */
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_calc_iqu_job, 0, NULL,
status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
}
/* Add POLANAL Frames to the WCS FrameSet in each output NDF. This Frame
is used by POLPACK to determine the reference direction of the Stokes
vectors (focal plane Y in this case, i.e. zero-based axis 1 ). */
smf_polext( indfq, 0, 0.0, "FPLANE", 1, status );
smf_polext( indfu, 0, 0.0, "FPLANE", 1, status );
if( ipi ) smf_polext( indfi, 0, 0.0, "FPLANE", 1, status );
/* Free the two output NDFs. */
ndfAnnul( &indfq, status );
ndfAnnul( &indfu, status );
if( ipi ) ndfAnnul( &indfi, status );
/* Free other resources. */
job_data = astFree( job_data );
mean = astFree( mean );
}
void smf_addcom( ThrWorkForce *wf, smfData *data, const double *com,
int *status ) {
/* Local Variables */
dim_t nbolo;
dim_t ntslice;
dim_t step;
int iw;
int nw;
smfAddComData *job_data;
smfAddComData *pdata;
/* Check the inherited status. */
if( *status != SAI__OK || !com ) return;
/* Check supplied smfData is time ordered (i.e. bstride=1, tstride=nbolo). */
if( !data->isTordered ) {
*status = SAI__ERROR;
errRep( " ", "smf_addcom: Supplied smfData is not time-ordered.",
status );
}
/* Note the number of time slices and bolometers. */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, NULL, NULL,
status );
/* Store the number of workers in the work force. */
nw = wf ? wf->nworker : 1;
/* Allocate job data for threads */
job_data = astMalloc( nw*sizeof(*job_data) );
if( *status == SAI__OK ) {
/* Get the number of time slices to process in each thread. */
if( nw > (int) ntslice ) {
step = 1;
} else {
step = ntslice/nw;
}
/* Set up the job data for each thread. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
/* The first and last time slice to be processed by the thread. */
pdata->t1 = iw*step;
if( iw < nw - 1 ) {
pdata->t2 = ( iw + 1 )*step - 1;
} else {
pdata->t2 = ntslice - 1;
}
/* Pointer to the first output data element for the first time slice. */
pdata->out = ( (double *) data->pntr[0] ) + pdata->t1*nbolo;
/* Pointer to the first COM data element. */
pdata->com = com + pdata->t1;
/* Number of bolometers. */
pdata->nbolo = nbolo;
}
/* Add each job to the job queue. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
(void) thrAddJob( wf, 0, pdata, smf1AddCom, 0, NULL, status );
}
/* Wait until all of the jobs have completed */
thrWait( wf, status );
}
/* Free resources. */
job_data = astFree( job_data );
}
smf_qual_t * smf_qual_unmap( ThrWorkForce *wf, int indf, smf_qfam_t family,
smf_qual_t * qual, smf_qual_t mask, int * status ) {
int canwrite = 0; /* can we write to the file? */
size_t nqbits = 0; /* Number of quality bits in this family */
SmfQualUnmapData *job_data = NULL;
SmfQualUnmapData *pdata;
int nw;
size_t step;
int iw;
if (*status != SAI__OK) goto CLEANUP;
/* do nothing if there is no quality */
if (!qual) return NULL;
/* if we do not have an NDF identifier we just free the memory */
if (indf == NDF__NOID) goto CLEANUP;
/* See if we have WRITE access to the file */
ndfIsacc( indf, "WRITE", &canwrite, status );
/* if we have WRITE access and the data were not mapped we have
to copy to the file. Also check we have a non-NULL input pointer.
If the data were mapped we still have to make sure the quality names
are stored. */
if ( canwrite && qual ) {
int highbit = -1; /* highest bit used */
size_t i;
int itemp;
int lowbit = -1; /* Lowest bit used */
size_t nout;
int nqual = 0;
void *qpntr[1];
size_t qcount[SMF__NQBITS]; /* statically allocate the largest array */
IRQLocs *qlocs;
unsigned char * qmap;
int there;
ndfMsg( "FILE", indf );
msgOutif( MSG__DEBUG, "", "Finalising quality for file ^FILE", status);
if (family == SMF__QFAM_TCOMP ) {
/* note that TCOMP is not an allowed quality because SMURF should not be
using it anywhere in a permanent way. */
*status = SAI__ERROR;
ndfMsg( "NDF", indf );
errRepf( "", "Unsupported quality family '%s' for quality unmapping of "
"file ^NDF", status, smf_qfamily_str(family,status) );
goto CLEANUP;
} else if (family == SMF__QFAM_NULL) {
/* In this case we have to assume that we just cast the quality
to UBYTE and copy it without changing anything or naming the
entries. Use a simple type conversion. */
ndfMap( indf, "QUALITY", "_UBYTE", "WRITE", &qpntr[0], &itemp, status );
qmap = qpntr[0];
nout = itemp;
for (i = 0; i<nout; i++) {
qmap[i] = qual[i];
}
ndfUnmap( indf, "QUALITY", status );
/* Turn on all quality */
ndfSbb( 255, indf, status );
/* we are finished so jump to tidy up */
goto CLEANUP;
}
/* work out how many quality items are in this family */
nqbits = smf_qfamily_count( family, status );
/* initialize qcount */
for (i=0; i<SMF__NQBITS; i++) {
qcount[i] = 0;
}
/* how many pixels in NDF (assumed to be number in quality) */
ndfSize( indf, &itemp, status );
nout = itemp;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many elements to process in each worker thread. */
step = nout/nw;
if( step == 0 ) step = 1;
/* Allocate job data for threads, and store common values. Ensure that the
last thread picks up any left-over elements. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->i1 = iw*step;
if( iw < nw - 1 ) {
pdata->i2 = pdata->i1 + step - 1;
} else {
pdata->i2 = nout - 1 ;
}
pdata->nqbits = nqbits;
pdata->qual = qual;
pdata->nout = nout;
}
}
/* Work out which bits are actually used */
if (*status == SAI__OK) {
size_t k;
/* now we try to be a bit clever. It may be a mistake since we have to
do multiple passes through "qual". First determine how many quality
bits are actually set. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 1;
thrAddJob( wf, 0, pdata, smf1_qual_unmap, 0, NULL, status );
}
thrWait( wf, status );
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
for( k=0; k<nqbits; k++ ) {
qcount[k] += pdata->qcount[k];
}
}
/* Reset the counts to zero for any bits that are not required
(i.e. are not set in "mask"). */
for( k=0; k<nqbits; k++ ) {
if( ! (mask & (1<<k)) ) qcount[k] = 0;
}
/* see how many we got */
for (k=0; k<nqbits; k++) {
if ( qcount[k] ) {
nqual++;
highbit = k;
if (lowbit < 0) lowbit = k;
}
}
}
/* for IRQ we need to ensure the SMURF extension exists so open and annul it if it is missing.
We are completely rewriting any IRQ information so we have to delete any previously existing
IRQ extension. */
irqDelet( indf, status );
ndfXstat( indf, SMURF__EXTNAME, &there, status );
if (!there) {
HDSLoc * smurfloc = NULL;
/* Create SMURF extension if it does not already exist */
ndfXnew( indf, SMURF__EXTNAME, SMURF__EXTTYPE, 0, NULL, &smurfloc, status );
if (smurfloc) datAnnul( &smurfloc, status );
}
irqNew( indf, SMURF__EXTNAME, &qlocs, status );
/* malloced so we need to map and copy over the values. IRQ
names need to be set BEFORE we copy. */
/* Map the quality component with WRITE access */
ndfMap( indf, "QUALITY", "_UBYTE", "WRITE", &qpntr[0], &itemp, status );
qmap = qpntr[0];
/* we assume the number of elements in "qual" is the same as in "qmap" */
if (*status == SAI__OK) {
size_t k;
/* if we only have 8 or fewer bits active we can just compress
by mapping them to the lower 8 bits. This will work if we also
set the IRQ quality names in the NDF. */
if (nqual == 0 ) {
/* easy */
memset( qmap, 0, nout * smf_dtype_sz( SMF__UBYTE, status ) );
} else if ( nqual <= 8 ) {
size_t curbit = 0;
/* and the quality names. Start at lowbit and go to highbit
knowing that we have shifted them down so that lowbit in qual
is bit 0 in NDF. */
for (k=lowbit; k<=(size_t)highbit; k++) {
if (qcount[k]) {
int fixed = 0; /* is bit fixed? */
const char * qdesc = NULL; /* Description of quality */
const char * qstr = NULL; /* Quality string identifier */
curbit++;
qstr = smf_qual_str( family, 1, k, &qdesc, status );
irqAddqn( qlocs, qstr, 0, qdesc, status );
irqFxbit( qlocs, qstr, curbit, &fixed, status );
}
}
/* shift them down */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 2;
pdata->qmap = qmap;
pdata->highbit = highbit;
pdata->lowbit = lowbit;
for( k=0; k<nqbits; k++ ) {
pdata->qcount[k] = qcount[k];
}
thrAddJob( wf, 0, pdata, smf1_qual_unmap, 0, NULL, status );
}
thrWait( wf, status );
} else {
size_t curbit = 0;
/* Quality names are now needed and we have to write them
all out because we have not compressed the bits in the
output quality array we've only compressed the input.
To limit the number of active bits we'd have to copy the
compressed bits to the output and then set the quality
names but IRQ does not let you do that so you would need
to run through the entire array first counting which bits
were used. */
for (k=0; k<SMF__NQBITS_TCOMP; k++) {
int fixed = 0;
const char * qdesc = NULL; /* Description of quality */
const char * qstr = NULL; /* Quality string identifier */
qstr = smf_qual_str( SMF__QFAM_TCOMP, 1, k, &qdesc, status );
/* Set the quality name */
irqAddqn( qlocs, qstr, 0, qdesc, status );
curbit++;
irqFxbit( qlocs, qstr, curbit, &fixed, status );
}
/* compress them */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 3;
pdata->qmap = qmap;
thrAddJob( wf, 0, pdata, smf1_qual_unmap, 0, NULL, status );
}
thrWait( wf, status );
}
}
/* Unmap quality */
ndfUnmap( indf, "QUALITY", status );
/* Set the badbits mask to enable all quality by default.
Do not do this for MAP quality at the moment. */
if (family != SMF__QFAM_MAP) ndfSbb( 255, indf, status );
/* release IRQ resources */
irqRlse( &qlocs, status );
}
CLEANUP:
/* Tidy up */
qual = astFree( qual );
job_data = astFree( job_data );
return NULL;
}
void smf_uncalc_iqu( ThrWorkForce *wf, smfData *data,
double *idata, double *qdata, double *udata,
int *status ){
/* Local Variables: */
const JCMTState *state; /* JCMTState info for current time slice */
dim_t nbolo; /* No. of bolometers */
dim_t ntslice; /* Number of time-slices in data */
int bstep; /* Bolometer step between threads */
int itime; /* Time slice index */
int iworker; /* Index of a worker thread */
int ntime; /* Time slices to check */
int nworker; /* No. of worker threads */
int old; /* Data has old-style POL_ANG values? */
size_t bstride; /* Stride between adjacent bolometer values */
size_t tstride; /* Stride between adjacent time slice values */
smfHead *hdr; /* Pointer to data header this time slice */
smfUncalcIQUJobData *job_data = NULL; /* Pointer to all job data */
smfUncalcIQUJobData *pdata = NULL;/* Pointer to next job data */
char headval[ 81 ]; /* FITS header value */
int ipolcrd; /* Reference direction for waveplate angles */
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Convenience pointer. */
hdr = data->hdr;
/* Check the half-waveplate and analyser were in the beam. */
headval[ 0 ] = 0;
smf_getfitss( hdr, "POLWAVIN", headval, sizeof(headval), status );
if( strcmp( headval, "Y" ) && *status == SAI__OK ) {
smf_smfFile_msg( data->file, "N", 0, "" );
*status = SAI__ERROR;
errRep( " ", "Half-waveplate was not in the beam for "
"input NDF ^N.", status );
}
headval[ 0 ] = 0;
smf_getfitss( hdr, "POLANLIN", headval, sizeof(headval), status );
if( strcmp( headval, "Y" ) && *status == SAI__OK ) {
smf_smfFile_msg( data->file, "N", 0, "" );
*status = SAI__ERROR;
errRep( " ", "Analyser was not in the beam for input "
"NDF ^N.", status );
}
/* Get the reference direction for JCMTSTATE:POL_ANG values. */
smf_getfitss( hdr, "POL_CRD", headval, sizeof(headval), status );
if( !strcmp( headval, "FPLANE" ) ) {
ipolcrd = 0;
} else if( !strcmp( headval, "AZEL" ) ) {
ipolcrd = 1;
} else if( !strcmp( headval, "TRACKING" ) ) {
ipolcrd = 2;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
smf_smfFile_msg( data->file, "N", 0, "" );
msgSetc( "V", headval );
errRep( " ", "Input NDF ^N contains unknown value "
"'^V' for FITS header 'POL_CRD'.", status );
}
/* Obtain number of time slices - will also check for 3d-ness. Also get
the dimensions of the bolometer array and the strides between adjacent
bolometer values. */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &bstride,
&tstride, status );
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
/* Check the above pointers can be used safely. */
if( *status == SAI__OK ) {
/* Go through the first thousand POL_ANG values to see if they are in
units of radians (new data) or arbitrary encoder units (old data).
They are assumed to be in radians if no POL_ANG value is larger than
20. */
old = 0;
state = hdr->allState;
ntime = ( ntslice > 1000 ) ? 1000 : ntslice;
for( itime = 0; itime < ntime; itime++,state++ ) {
if( state->pol_ang > 20 ) {
old = 1;
msgOutif( MSG__VERB, ""," POL2 data contains POL_ANG values "
"in encoder units - converting to radians.", status );
break;
}
}
/* Determine which bolometers are to be processed by which threads. */
bstep = nbolo/nworker;
if( bstep < 1 ) bstep = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->b1 = iworker*bstep;
pdata->b2 = pdata->b1 + bstep - 1;
}
/* Ensure that the last thread picks up any left-over bolometers */
pdata->b2 = nbolo - 1;
/* Store all the other info needed by the worker threads, and submit the
jobs to calculate the analysed intensity values in each bolo, and then
wait for them to complete. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->bstride = bstride;
pdata->nbolo = nbolo;
pdata->tstride = tstride;
pdata->allstates = hdr->allState;
pdata->ipi = idata;
pdata->ipq = qdata;
pdata->ipu = udata;
pdata->ipolcrd = ipolcrd;
pdata->old = old;
pdata->ntslice = ntslice;
/* Pass the job to the workforce for execution. */
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_uncalc_iqu_job, 0, NULL,
status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
}
/* Free resources. */
job_data = astFree( job_data );
}
void smurf_sc2threadtest( int *status ) {
/* Local Variables */
smfArray **res=NULL; /* array of smfArrays of test data */
smfData *data=NULL; /* Pointer to SCUBA2 data struct */
dim_t datalen; /* Number of data points */
smfFilter *filt=NULL; /* Frequency domain filter */
size_t i; /* Loop counter */
size_t j; /* Loop counter */
smfTimeChunkData *job_data=NULL; /* Array of pointers for job data */
size_t joblen; /* Number of chunks per job */
size_t k; /* Loop counter */
size_t nchunks; /* Number of chunks */
size_t nsub; /* Number of subarrays */
int nthread; /* Number of threads */
smfTimeChunkData *pdata=NULL; /* Pointer to data for single job */
int temp; /* Temporary integer */
size_t tsteps; /* How many time steps in chunk */
struct timeval tv1, tv2; /* Timers */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
double *dat=NULL;
dim_t nbolo;
dim_t ntslice;
dim_t ndata;
size_t bstride;
size_t tstride;
dim_t offset;
if (*status != SAI__OK) return;
/* Get input parameters */
parGdr0i( "NTHREAD", 1, 1, NUM__MAXI, 1, &nthread, status );
parGdr0i( "TSTEPS", 6000, 0, NUM__MAXI, 1, &temp, status );
tsteps = (size_t) temp;
parGdr0i( "NCHUNKS", 1, 1, NUM__MAXI, 1, &temp, status );
nchunks = (size_t) temp;
parGdr0i( "NSUB", 1, 1, 4, 1, &temp, status );
nsub = (size_t) temp;
msgSeti("N",nthread);
msgOut( "", TASK_NAME ": Running test with ^N threads", status );
/*** TIMER ***/
smf_timerinit( &tv1, &tv2, status );
/* Create some fake test data in the form of an array of smfArrays */
msgSeti("T",tsteps);
msgSeti("C",nchunks);
msgSeti("NS",nsub);
msgOut( "", TASK_NAME
": Creating ^NS subarrays of data with ^C chunks * ^T samples",
status );
res = astCalloc( nchunks, sizeof(*res) );
for( k=0; (*status==SAI__OK)&&(k<nchunks); k++ ) {
res[k] = smf_create_smfArray( status );
for( i=0; (*status==SAI__OK)&&(i<nsub); i++ ) {
/* Create individual smfDatas and add to array */
data = smf_create_smfData( SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS, status );
if( *status==SAI__OK ) {
data->dtype=SMF__DOUBLE;
data->ndims=3;
data->dims[0]=40;
data->dims[1]=32;
data->dims[2]=(dim_t) tsteps;
datalen=1;
data->isFFT=-1;
for( j=0; j<data->ndims; j++ ) datalen *= data->dims[j];
data->hdr->steptime = 0.005;
data->pntr[0] = astCalloc( datalen, smf_dtype_sz(data->dtype,status) );
data->qual = astCalloc( datalen, sizeof(*data->qual) );
}
smf_addto_smfArray( res[k], data, status );
}
}
/*** TIMER ***/
msgOutf( "", "** %f seconds generating data", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut( "", TASK_NAME
": Starting test 1 __parallel time: dataOrder__", status );
/* Create a pool of threads. */
wf = thrGetWorkforce( nthread, status );
/* Work out number of chunks per thread */
joblen = nchunks/nthread;
if( joblen == 0 ) joblen = 1; /* At least one chunk per thread */
/* The first test will process separate time chunks of data in
parallel, re-ordering each to bolo-ordered format. All subarrays
and an integer number of input file chunks all go into a single
thread. Start by allocating and initializing a number of
smfTimeChunkData's that hold the information required for each
thread */
job_data = astCalloc( nthread, sizeof(*job_data) );
for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) {
pdata = job_data + i;
pdata->type = 0; /* Start with a data re-order */
pdata->data = res; /* Pointer to main data array */
pdata->chunk1 = i*joblen; /* Index of first chunk for job */
pdata->nchunks = nchunks; /* Total number of time chunks in data */
pdata->ijob = -1; /* Flag job as available to do work */
/* The last thread has to pick up the remainder of chunks */
if( i==(size_t)(nthread-1) ) pdata->chunk2=nchunks-1;
else pdata->chunk2 = (i+1)*joblen-1; /* Index of last chunk for job */
/* Ensure a valid chunk range, or set to a length that we know to ignore */
if( pdata->chunk1 >= nchunks ) {
pdata->chunk1 = nchunks;
pdata->chunk2 = nchunks;
} else if( pdata->chunk2 >= nchunks ) {
pdata->chunk2 = nchunks-1;
}
if( pdata->chunk1 >= nchunks ) {
/* Nothing for this thread to do */
msgSeti( "W", i+1);
msgOutif( MSG__DEBUG, "",
"-- parallel time: skipping thread ^W, nothing to do",
status);
} else {
/* Since we know there is one job_data per thread, just submit jobs
immediately */
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Annul the bad status that we set in smfParallelTime */
if( *status == SMF__INSMP ) {
errAnnul( status );
msgOut( "", " *** Annulled SMF__INSMP set in smfParallelTime *** ",
status );
} else {
msgOut( "", " *** Flushing good status *** ", status );
errFlush( status );
}
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
/* The second test will boxcar smooth bolometers from time chunks in
parallel */
msgOut( "", TASK_NAME
": Starting test 2 __parallel time: boxcar smooth__", status );
for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) {
pdata = job_data + i;
pdata->type = 1; /* Boxcar smooth */
if( pdata->chunk1 >= nchunks ) {
/* Nothing for this thread to do */
msgSeti( "W", i+1);
msgOutif( MSG__DEBUG, "",
"-- parallel time: skipping thread ^W, nothing to do",
status);
} else {
/* Since we know there is one job_data per thread, just submit jobs
immediately */
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut( "", TASK_NAME
": *** Next 2 tests will be done twice due to FFTW planning *****",
status );
for( k=0; k<2; k++ ) {
/* The third test will FFT filter bolometers from time chunks in
parallel */
msgOut( "", TASK_NAME
": Starting test 3 __parallel time: FFT filter__", status );
for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) {
pdata = job_data + i;
pdata->type = 2; /* FFT filter */
if( pdata->chunk1 >= nchunks ) {
/* Nothing for this thread to do */
msgSeti( "W", i+1);
msgOutif( MSG__DEBUG, "",
"-- parallel time: skipping thread ^W, nothing to do",
status);
} else {
/* Since we know there is one job_data per thread, just submit jobs
immediately */
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut( "", TASK_NAME
": Starting test 4 __FFTW filter using internal threading__",
status );
for( i=0; (*status==SAI__OK)&&(i<nchunks); i++ ) {
filt = smf_create_smfFilter( res[i]->sdata[0], status );
smf_filter_ident( filt, 1, status );
for( j=0; (*status==SAI__OK)&&(j<nsub); j++ ) {
msgOutiff( MSG__DEBUG, "", " filter chunk %zu/%zu, bolo %zu/%zu",
status, i+1, nchunks, j+1, nsub );
smf_filter_execute( wf, res[i]->sdata[j], filt, 0, 0, status );
}
if( filt ) filt = smf_free_smfFilter( filt, status );
}
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
}
msgOut( "", TASK_NAME
": **************************************************************",
status );
/* Series of short single-thread array index tests */
data = res[0]->sdata[0];
dat = data->pntr[0];
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride,
&tstride, status );
msgOut("","Array index test #1: two multiplies in inner loop",status);
smf_timerinit( &tv1, &tv2, status );
for( i=0; i<nbolo; i++ ) {
for( j=0; j<ntslice; j++ ) {
dat[i*bstride + j*tstride] += 5;
}
}
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut("","Array index test #2: only index increments",status);
smf_timerinit( &tv1, &tv2, status );
for( i=0; i<nbolo*bstride; i+=bstride ) {
for( j=i; j<(i+ntslice*tstride); j+=tstride ) {
dat[j] += 5;
}
}
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut("","Array index test #3: one multiply in outer loop",status);
smf_timerinit( &tv1, &tv2, status );
offset = 0;
for( i=0; i<nbolo; i++ ) {
offset = i*bstride;
for( j=0; j<ntslice; j++ ) {
dat[offset] += 5;
offset += tstride;
}
}
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
/* Clean up */
if( res ) {
for( i=0; i<nchunks; i++ ) {
if( res[i] ) {
smf_close_related( &res[i], status );
}
}
res = astFree( res );
}
job_data = astFree( job_data );
/* Ensure that FFTW doesn't have any used memory kicking around */
fftw_cleanup();
}
/* Main entry point. */
void smf_calcmodel_smo( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel,
int flags __attribute__((unused)),
int *status) {
/* Local Variables */
size_t bstride; /* bolo stride */
dim_t boxcar = 0; /* size of boxcar smooth window */
smf_filt_t filter_type; /* The type of smoothing to perform */
size_t i; /* Loop counter */
dim_t idx=0; /* Index within subgroup */
int iworker; /* Owkrer index */
smfCalcmodelSmoJobData *job_data=NULL; /* Pointer to all job data structures */
AstKeyMap *kmap=NULL; /* Pointer to PLN-specific keys */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
double *model_data_copy=NULL; /* Copy of model_data for one bolo */
dim_t nbolo=0; /* Number of bolometers */
dim_t ndata=0; /* Total number of data points */
int notfirst=0; /* flag for delaying until after 1st iter */
dim_t ntslice=0; /* Number of time slices */
int nworker; /* No. of worker threads in supplied Workforce */
smfCalcmodelSmoJobData *pdata=NULL; /* Pointer to current data structure */
smfArray *qua=NULL; /* Pointer to QUA at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to quality data */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
int step; /* Number of bolometers per thread */
size_t tstride; /* Time slice stride in data array */
const char * typestr = NULL; /* smo.type value */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
/* Obtain pointer to sub-keymap containing PLN parameters. Something will
always be available.*/
astMapGet0A( keymap, "SMO", &kmap );
/* Are we skipping the first iteration? */
astMapGet0I(kmap, "NOTFIRST", ¬first);
if( notfirst && (flags & SMF__DIMM_FIRSTITER) ) {
msgOutif( MSG__VERB, "", FUNC_NAME
": skipping SMO this iteration", status );
return;
}
/* Get the boxcar size */
if( kmap ) smf_get_nsamp( kmap, "BOXCAR", res->sdata[0], &boxcar, status );
/* Get the type of smoothing filter to use. Anthing that is not "MEDIAN" is mean */
filter_type = SMF__FILT_MEAN;
if (astMapGet0C( kmap, "TYPE", &typestr ) ) {
if (strncasecmp( typestr, "MED", 3 ) == 0 ) {
filter_type = SMF__FILT_MEDIAN;
}
}
/* Assert bolo-ordered data */
smf_model_dataOrder( wf, dat, allmodel, chunk, SMF__RES|SMF__QUA,
0, status );
smf_get_dims( res->sdata[0], NULL, NULL, NULL, &ntslice,
&ndata, NULL, NULL, status);
model = allmodel[chunk];
msgOutiff(MSG__VERB, "",
" Calculating smoothed model using boxcar of width %" DIM_T_FMT " time slices",
status, boxcar);
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
/* Loop over index in subgrp (subarray) and put the previous iteration
of the filtered component back into the residual before calculating
and removing the new filtered component */
for( idx=0; (*status==SAI__OK)&&(idx<res->ndat); idx++ ) {
/* Obtain dimensions of the data */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice,
&ndata, &bstride, &tstride, status);
/* Get pointers to data/quality/model */
res_data = (res->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Uncomment to aid debugging */
/*
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_in",
NULL, 0, 0, MSG__VERB, 0, status );
*/
if( *status == SAI__OK ) {
/* Place last iteration back into residual if this is a smoothable section of the time series */
for (i=0; i< ndata; i++) {
if ( !(qua_data[i]&SMF__Q_FIT) && res_data[i] != VAL__BADD && model_data[i] != VAL__BADD ) {
res_data[i] += model_data[i];
}
}
}
/* Uncomment to aid debugging */
/*
smf_write_smfData( model->sdata[idx], NULL, qua_data, "model_b4",
NULL, 0, 0, MSG__VERB, 0, status );
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_b4",
NULL, 0, 0, MSG__VERB, 0, status );
*/
/* Determine which bolometers are to be processed by which threads. */
step = nbolo/nworker;
if( step < 1 ) step = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->b1 = iworker*step;
pdata->b2 = pdata->b1 + step - 1;
}
/* Ensure that the last thread picks up any left-over bolometers */
pdata->b2 = nbolo - 1;
/* Store all the other info needed by the worker threads, and submit the
jobs to apply the smoothing. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->boxcar = boxcar;
pdata->bstride = bstride;
pdata->bstride = bstride;
pdata->filter_type = filter_type;
pdata->model_data = model_data;
pdata->nbolo = nbolo;
pdata->nbolo = nbolo;
pdata->ntslice = ntslice;
pdata->ntslice = ntslice;
pdata->qua_data = qua_data;
pdata->qua_data = qua_data;
pdata->res_data = res_data;
pdata->res_data = res_data;
pdata->tstride = tstride;
pdata->tstride = tstride;
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_calcmodel_smo_job,
0, NULL, status );
}
thrWait( wf, status );
/* Uncomment to aid debugging */
/*
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_af",
NULL, 0, 0, MSG__VERB, 0, status );
smf_write_smfData( model->sdata[idx], NULL, qua_data, "model_af",
NULL, 0, 0, MSG__VERB, 0, status );
*/
}
}
/* Free work space (astFree returns without action if a NULL pointer is
supplied). */
model_data_copy = astFree( model_data_copy );
job_data = astFree( job_data );
/* Annul AST Object pointers (astAnnul reports an error if a NULL pointer
is supplied). */
if( kmap ) kmap = astAnnul( kmap );
}
int smf_correct_extinction(ThrWorkForce *wf, smfData *data, smf_tausrc tausrc, smf_extmeth method,
AstKeyMap * extpars, double tau, double *allextcorr,
double **wvmtaucache, int *status) {
/* Local variables */
int allquick = 0; /* Is the extinction for all bolometers the same? */
double amstart = VAL__BADD; /* Airmass at start */
double amend = VAL__BADD; /* Airmass at end */
double elstart = VAL__BADD; /* Elevation at start (radians) */
double elend = VAL__BADD;/* Elevation at end (radians) */
smfHead *hdr = NULL; /* Pointer to full header struct */
double *indata = NULL; /* Pointer to data array */
int isTordered; /* data order of input data */
int lbnd[2]; /* Lower bound */
size_t ndims; /* Number of dimensions in input data */
dim_t nframes = 0; /* Number of frames */
dim_t npts = 0; /* Number of data points */
dim_t nx = 0; /* # pixels in x-direction */
dim_t ny = 0; /* # pixels in y-direction */
int ubnd[2]; /* Upper bound */
double *vardata = NULL; /* Pointer to variance array */
double * wvmtau = NULL; /* WVM tau (smoothed or not) for these data */
int nw; /* Number of worker threads */
int iw; /* Thread index */
SmfCorrectExtinctionData *job_data = NULL; /* Array of job descriptions */
SmfCorrectExtinctionData *pdata; /* Pointer to next job description */
size_t framestep; /* Number of frames per thread */
/* Check status */
if (*status != SAI__OK) return allquick;
/* If no correction requested, return */
if( method==SMF__EXTMETH_NONE ) {
msgOutif(MSG__VERB, "", FUNC_NAME ": Extinction method=none, returning",
status );
return allquick;
}
/* If no opacity monitor specified generate bad status */
if( tausrc==SMF__TAUSRC_NULL ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No extinction monitor specified",
status );
return allquick;
}
if( smf_history_check( data, FUNC_NAME, status) ) {
/* If caller not requesting allextcorr fail here */
if( !allextcorr ) {
msgSetc("F", FUNC_NAME);
msgOutif(MSG__VERB," ",
"^F has already been run on these data, returning to caller",
status);
return allquick;
}
}
/* Acquire the data order */
isTordered = data->isTordered;
/* make sure we have a header */
hdr = data->hdr;
if( hdr == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Input data has no header", status);
return allquick;
}
/* Do we have 2-D image data? */
ndims = data->ndims;
if (ndims == 2) {
nframes = 1;
nx = (data->dims)[0];
ny = (data->dims)[1];
npts = nx*ny;
} else {
/* this routine will also check for dimensionality */
smf_get_dims( data, &nx, &ny, &npts, &nframes, NULL, NULL, NULL, status );
}
/* Tell user we're correcting for extinction */
msgOutif(MSG__VERB," ",
"Correcting for extinction.", status);
/* Should check data type for double if not allextcorr case */
if( !allextcorr ) {
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status)) return allquick;
}
/* Check that we're not trying to use the WVM for 2-D data */
if ( ndims == 2 && tausrc == SMF__TAUSRC_WVMRAW ) {
if ( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "Method WVMRaw can not be used on 2-D image data", status );
return allquick;
}
} else if (ndims < 2 || ndims > 3) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRepf( FUNC_NAME, "Can not extinction correct data with %zd dimension(s)", status,
ndims );
return allquick;
}
}
if (tausrc == SMF__TAUSRC_WVMRAW) {
size_t ntotaltau = 0;
size_t ngoodtau = 0;
/* calculate WVM unless we have external values */
if (wvmtaucache && *wvmtaucache) {
wvmtau = *wvmtaucache;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using cached WVM data for extinction correction of ^FILE",
status);
} else {
smf_calc_smoothedwvm( wf, NULL, data, extpars, &wvmtau, &ntotaltau,
&ngoodtau, status );
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using WVM mode for extinction correction of ^FILE"
" %.0f %% of WVM data are present", status,
(double)(100.0*(double)ngoodtau/(double)ntotaltau) );
}
}
/* Check auto mode */
if (tausrc == SMF__TAUSRC_AUTO && *status == SAI__OK) {
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
if (ndims == 2) {
/* have to use CSO mode */
tausrc = SMF__TAUSRC_CSOTAU;
} else if (ndims == 3) {
/* Calculate the WVM tau data and see if we have enough good data */
size_t ngoodtau = 0;
size_t ntotaltau = 0;
double percentgood = 0.0;
if (wvmtaucache && *wvmtaucache) {
wvmtau = *wvmtaucache;
tausrc = SMF__TAUSRC_WVMRAW;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>");
msgOutiff( MSG__VERB, "", "Using cached WVM data for extinction correction of ^FILE",
status );
} else {
smf_calc_smoothedwvm( wf, NULL, data, extpars, &wvmtau, &ntotaltau,
&ngoodtau, status );
percentgood = 100.0 * ((double)ngoodtau / (double)ntotaltau);
if ( percentgood > 80.0) {
tausrc = SMF__TAUSRC_WVMRAW;
msgOutiff( MSG__VERB, "", "Selecting WVM mode for extinction correction of ^FILE."
" %.0f %% of WVM data are present", status, percentgood );
} else {
tausrc = SMF__TAUSRC_CSOTAU;
if (wvmtau) wvmtau = astFree( wvmtau );
}
}
}
if (tausrc == SMF__TAUSRC_CSOTAU) {
msgOutiff( MSG__VERB, "", "Selecting CSO mode for extinction correction of ^FILE", status );
} else if (tausrc == SMF__TAUSRC_WVMRAW) {
/* Dealt with this above */
} else {
/* oops. Fall back position */
tausrc = SMF__TAUSRC_CSOTAU;
msgOutiff( MSG__VERB, "", "Selecting CSO mode as unexpected fallback for extinction correction of ^FILE", status );
}
}
/* If we have a CSO Tau then convert it to the current filter. This will also
convert bad values to a value derived from the header if appropriate. */
if ( tausrc == SMF__TAUSRC_CSOTAU ) {
tau = smf_cso2filt_tau( hdr, tau, extpars, status );
/* The tau source is now a real tau */
tausrc = SMF__TAUSRC_TAU;
}
/* Find the airmass range for this data */
smf_find_airmass_interval( hdr, &amstart, &amend, &elstart, &elend, status );
if (*status == SAI__OK && (amstart == VAL__BADD || amend == VAL__BADD)) {
*status = SAI__ERROR;
errRep( "", "No good airmass values found in JCMTSTATE structure for these data",
status );
}
/* if we are not doing WVM correction but are in adaptive mode we can determine
whether or not we will have to use full or single mode just by looking at the
airmass data. */
if (ndims == 3 && tausrc != SMF__TAUSRC_WVMRAW && method == SMF__EXTMETH_ADAPT) {
/* first and last is a good approximation given that most SCUBA-2 files will only
be a minute duration. */
double refel;
double refam;
/* only need to examine the largest airmass */
if (amstart > amend) {
refam = amstart;
refel = elstart;
} else {
refam = amend;
refel = elend;
}
/* and choose a correction method */
if (is_large_delta_atau( refam, refel, tau, status) ) {
method = SMF__EXTMETH_FULL;
msgOutiff(MSG__DEBUG, " ",
"Adaptive extinction algorithm selected per-bolometer airmass value "
"per time slice (am=%g, tau=%g)", status, refam, tau);
} else {
msgOutiff(MSG__DEBUG, " ",
"Adaptive extinction algorithm selected single airmass value per time slice"
" (am=%g, tau=%g)", status, refam, tau);
method = SMF__EXTMETH_SINGLE;
}
}
/* Assign pointer to input data array if status is good */
if ( *status == SAI__OK ) {
indata = (data->pntr)[0];
vardata = (data->pntr)[1];
}
/* Jump to the cleanup section if status is bad by this point
since we need to free memory */
if (*status != SAI__OK) goto CLEANUP;
/* Array bounds for astTranGrid call */
lbnd[0] = 1;
lbnd[1] = 1;
ubnd[0] = nx;
ubnd[1] = ny;
/* Unlock the AST objects in the smfData so that the worker threads can
lock them. */
smf_lock_data( data, 0, status );
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many frames to process in each worker thread. */
framestep = nframes/nw;
if( framestep == 0 ) {
framestep = 1;
nw = nframes;
}
/* Allocate job data for threads, and store the range of frames to be
processed by each one. Ensure that the last thread picks up any
left-over frames. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->f1 = iw*framestep;
if( iw < nw - 1 ) {
pdata->f2 = pdata->f1 + framestep - 1;
} else {
pdata->f2 = nframes - 1 ;
}
pdata->nframes = nframes;
pdata->npts = npts;
pdata->allextcorr = allextcorr;
pdata->indata = indata;
pdata->tau = tau;
pdata->vardata = vardata;
pdata->wvmtau = wvmtau;
pdata->amstart = amstart;
pdata->amfirst = amstart + ( amend - amstart )*pdata->f1/( nframes - 1 );
pdata->lbnd = lbnd;
pdata->ubnd = ubnd;
pdata->isTordered = isTordered;
pdata->ndims = ndims;
pdata->data = data;
pdata->hdr = hdr;
pdata->method = method;
pdata->tausrc = tausrc;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_correct_extinction, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Record if all time slices used a single air mass. */
allquick = 1;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
if( ! pdata->allquick ) {
allquick = 0;
break;
}
}
/* Free the job data. */
job_data = astFree( job_data );
}
/* Lock the AST objects in the smfData for use by this thread. */
smf_lock_data( data, 1, status );
/* Add history entry if !allextcorr */
if( (*status == SAI__OK) && !allextcorr ) {
smf_history_add( data, FUNC_NAME, status);
}
CLEANUP:
if (wvmtaucache) {
if (!*wvmtaucache) {
*wvmtaucache = wvmtau;
}
} else {
wvmtau = astFree( wvmtau );
}
return allquick;
}
