void smurf_sc2fft( int *status ) {
int avpspec=0; /* Flag for doing average power spectrum */
double avpspecthresh=0; /* Threshold noise for detectors in avpspec */
Grp * basegrp = NULL; /* Basis group for output filenames */
smfArray *bbms = NULL; /* Bad bolometer masks */
smfArray *concat=NULL; /* Pointer to a smfArray */
size_t contchunk; /* Continuous chunk counter */
smfArray *darks = NULL; /* dark frames */
int ensureflat; /* Flag for flatfielding data */
Grp *fgrp = NULL; /* Filtered group, no darks */
smfArray *flatramps = NULL;/* Flatfield ramps */
AstKeyMap *heateffmap = NULL; /* Heater efficiency data */
size_t gcount=0; /* Grp index counter */
size_t i; /* Loop counter */
smfGroup *igroup=NULL; /* smfGroup corresponding to igrp */
Grp *igrp = NULL; /* Input group of files */
int inverse=0; /* If set perform inverse transform */
int isfft=0; /* Are data fft or real space? */
dim_t maxconcat=0; /* Longest continuous chunk length in samples */
size_t ncontchunks=0; /* Number continuous chunks outside iter loop */
smfData *odata=NULL; /* Pointer to output smfData to be exported */
Grp *ogrp = NULL; /* Output group of files */
size_t outsize; /* Total number of NDF names in the output group */
int polar=0; /* Flag for FFT in polar coordinates */
int power=0; /* Flag for squaring amplitude coeffs */
size_t size; /* Number of files in input group */
smfData *tempdata=NULL; /* Temporary smfData pointer */
int weightavpspec=0; /* Flag for 1/noise^2 weighting */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
int zerobad; /* Zero VAL__BADD before taking FFT? */
/* Main routine */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get input file(s) */
kpg1Rgndf( "IN", 0, 1, "", &igrp, &size, status );
/* Filter out darks */
smf_find_science( igrp, &fgrp, 1, NULL, NULL, 1, 1, SMF__NULL, &darks,
&flatramps, &heateffmap, NULL, status );
/* input group is now the filtered group so we can use that and
free the old input group */
size = grpGrpsz( fgrp, status );
grpDelet( &igrp, status);
igrp = fgrp;
fgrp = NULL;
/* We now need to combine files from the same subarray and same sequence
to form a continuous time series */
smf_grp_related( igrp, size, 1, 0, 0, NULL, NULL, &maxconcat, NULL, &igroup,
&basegrp, NULL, status );
/* Get output file(s) */
size = grpGrpsz( basegrp, status );
if( size > 0 ) {
kpg1Wgndf( "OUT", basegrp, size, size, "More output files required...",
&ogrp, &outsize, status );
} else {
msgOutif(MSG__NORM, " ", TASK_NAME ": All supplied input frames were DARK,"
" nothing to do", status );
}
/* Get group of bolometer masks and read them into a smfArray */
smf_request_mask( "BBM", &bbms, status );
/* Obtain the number of continuous chunks and subarrays */
if( *status == SAI__OK ) {
ncontchunks = igroup->chunk[igroup->ngroups-1]+1;
}
msgOutiff( MSG__NORM, "", "Found %zu continuous chunk%s", status, ncontchunks,
(ncontchunks > 1 ? "s" : "") );
/* Are we flatfielding? */
parGet0l( "FLAT", &ensureflat, status );
/* Are we doing an inverse transform? */
parGet0l( "INVERSE", &inverse, status );
/* Are we using polar coordinates instead of cartesian for the FFT? */
parGet0l( "POLAR", &polar, status );
/* Are we going to assume amplitudes are squared? */
parGet0l( "POWER", &power, status );
/* Are we going to zero bad values first? */
parGet0l( "ZEROBAD", &zerobad, status );
/* Are we calculating the average power spectrum? */
parGet0l( "AVPSPEC", &avpspec, status );
if( avpspec ) {
power = 1;
parGet0d( "AVPSPECTHRESH", &avpspecthresh, status );
parGet0l( "WEIGHTAVPSPEC", &weightavpspec, status );
}
/* If power is true, we must be in polar form */
if( power && !polar) {
msgOutif( MSG__NORM, " ", TASK_NAME
": power spectrum requested so setting POLAR=TRUE", status );
polar = 1;
}
gcount = 1;
for( contchunk=0;(*status==SAI__OK)&&contchunk<ncontchunks; contchunk++ ) {
size_t idx;
/* Concatenate this continuous chunk but forcing a raw data read.
We will need quality. */
smf_concat_smfGroup( wf, NULL, igroup, darks, NULL, flatramps, heateffmap,
contchunk, ensureflat, 1, NULL, 0, NULL, NULL, 0, 0, 0,
&concat, NULL, status );
/* Now loop over each subarray */
/* Export concatenated data for each subarray to NDF file */
for( idx=0; (*status==SAI__OK)&&idx<concat->ndat; idx++ ) {
if( concat->sdata[idx] ) {
smfData * idata = concat->sdata[idx];
int provid = NDF__NOID;
dim_t nbolo; /* Number of detectors */
dim_t ndata; /* Number of data points */
/* Apply a mask to the quality array and data array */
smf_apply_mask( idata, bbms, SMF__BBM_QUAL|SMF__BBM_DATA, 0, status );
smf_get_dims( idata, NULL, NULL, &nbolo, NULL, &ndata, NULL, NULL,
status );
/* Check for double precision data */
if( idata->dtype != SMF__DOUBLE ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": data are not double precision.", status );
}
/* Are we zeroing VAL__BADD? */
if( (*status==SAI__OK) && zerobad ) {
double *data= (double *) idata->pntr[0];
for( i=0; i<ndata; i++ ) {
if( data[i] == VAL__BADD ) {
data[i] = 0;
}
}
}
/* Check whether we need to transform the data at all */
isfft = smf_isfft(idata,NULL,NULL,NULL,NULL,NULL,status);
if( isfft && avpspec && (*status == SAI__OK) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": to calculate average power spectrum input data cannot "
"be FFT", status );
}
if( (*status == SAI__OK) && (isfft == inverse) ) {
if( avpspec ) {
/* If calculating average power spectrum do the transforms with
smf_bolonoise so that we can also measure the noise of
each detector */
double *whitenoise=NULL;
smf_qual_t *bolomask=NULL;
double mean, sig, freqlo;
size_t ngood, newgood;
whitenoise = astCalloc( nbolo, sizeof(*whitenoise) );
bolomask = astCalloc( nbolo, sizeof(*bolomask) );
freqlo = 1. / (idata->hdr->steptime * idata->hdr->nframes);
smf_bolonoise( wf, idata, 1, freqlo, SMF__F_WHITELO,
SMF__F_WHITEHI, 1, 0, whitenoise, NULL, &odata,
status );
/* Initialize quality */
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] == VAL__BADD ) {
bolomask[i] = SMF__Q_BADB;
} else {
/* smf_bolonoise returns a variance, so take sqrt */
whitenoise[i] = sqrt(whitenoise[i]);
}
}
ngood=-1;
newgood=0;
/* Iteratively cut n-sigma noisy outlier detectors */
while( ngood != newgood ) {
ngood = newgood;
smf_stats1D( whitenoise, 1, nbolo, bolomask, 1, SMF__Q_BADB,
&mean, &sig, NULL, NULL, status );
msgOutiff( MSG__DEBUG, "", TASK_NAME
": mean=%lf sig=%lf ngood=%li\n", status,
mean, sig, ngood);
newgood=0;
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] != VAL__BADD ){
if( (whitenoise[i] - mean) > avpspecthresh *sig ) {
whitenoise[i] = VAL__BADD;
bolomask[i] = SMF__Q_BADB;
} else {
newgood++;
}
}
}
}
msgOutf( "", TASK_NAME
": Calculating average power spectrum of best %li "
" bolometers.", status, newgood);
/* If using 1/noise^2 weights, calculate 1/whitenoise^2 in-place
to avoid allocating another array */
if( weightavpspec ) {
msgOutif( MSG__VERB, "", TASK_NAME ": using 1/noise^2 weights",
status );
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] && (whitenoise[i] != VAL__BADD) ) {
whitenoise[i] = 1/(whitenoise[i]*whitenoise[i]);
}
}
}
/* Calculate the average power spectrum of good detectors */
tempdata = smf_fft_avpspec( odata, bolomask, 1, SMF__Q_BADB,
weightavpspec ? whitenoise : NULL,
status );
smf_close_file( &odata, status );
whitenoise = astFree( whitenoise );
bolomask = astFree( bolomask );
odata = tempdata;
tempdata = NULL;
/* Store the number of good bolometers */
parPut0i( "NGOOD", newgood, status );
} else {
/* Otherwise do forward/inverse transforms here as needed */
/* If inverse transform convert to cartesian representation first */
if( inverse && polar ) {
smf_fft_cart2pol( wf, idata, 1, power, status );
}
/* Tranform the data */
odata = smf_fft_data( wf, idata, NULL, inverse, 0, status );
smf_convert_bad( wf, odata, status );
if( inverse ) {
/* If output is time-domain, ensure that it is ICD bolo-ordered */
smf_dataOrder( odata, 1, status );
} else if( polar ) {
/* Store FFT of data in polar form */
smf_fft_cart2pol( wf, odata, 0, power, status );
}
}
/* open a reference input file for provenance propagation */
ndgNdfas( basegrp, gcount, "READ", &provid, status );
/* Export the data to a new file */
smf_write_smfData( odata, NULL, NULL, ogrp, gcount, provid,
MSG__VERB, 0, status );
/* Free resources */
ndfAnnul( &provid, status );
smf_close_file( &odata, status );
} else {
msgOutif( MSG__NORM, " ",
"Data are already transformed. No output will be produced",
status );
}
}
/* Update index into group */
gcount++;
}
/* Close the smfArray */
smf_close_related( &concat, status );
}
/* Write out the list of output NDF names, annulling the error if a null
parameter value is supplied. */
if( *status == SAI__OK ) {
grpList( "OUTFILES", 0, 0, NULL, ogrp, status );
if( *status == PAR__NULL ) errAnnul( status );
}
/* Tidy up after ourselves: release the resources used by the grp routines */
grpDelet( &igrp, status);
grpDelet( &ogrp, status);
if (basegrp) grpDelet( &basegrp, status );
if( igroup ) smf_close_smfGroup( &igroup, status );
if( flatramps ) smf_close_related( &flatramps, status );
if (heateffmap) heateffmap = smf_free_effmap( heateffmap, status );
if (bbms) smf_close_related( &bbms, status );
ndfEnd( status );
/* Ensure that FFTW doesn't have any used memory kicking around */
fftw_cleanup();
}
void smf_bolonoise( ThrWorkForce *wf, smfData *data, double gfrac,
size_t window, double f_low,
double f_white1, double f_white2,
int nep, size_t len, double *whitenoise, double *fratio,
smfData **fftpow,int *status ) {
double *base=NULL; /* Pointer to base coordinates of array */
size_t bstride; /* bolometer index stride */
double df=1; /* Frequency step size in Hz */
size_t i; /* Loop counter */
size_t i_low; /* Index in power spectrum to f_low */
size_t i_w1; /* Index in power spectrum to f_white1 */
size_t i_w2; /* Index in power spectrum to f_white2 */
size_t j; /* Loop counter */
size_t mingood; /* Min. required no. of good values in bolometer */
dim_t nbolo; /* Number of bolometers */
dim_t ndata; /* Number of data points */
dim_t nf=0; /* Number of frequencies */
size_t ngood; /* Number of good samples */
dim_t ntslice; /* Number of time slices */
double p_low; /* Power at f_low */
double p_white; /* Average power from f_white1 to f_white2 */
smfData *pow=NULL; /* Pointer to power spectrum data */
smf_qual_t *qua=NULL; /* Pointer to quality component */
double steptime=1; /* Length of a sample in seconds */
size_t tstride; /* time index stride */
if (*status != SAI__OK) return;
/* Check inputs */
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status )) return;
if( !data->hdr ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData has no header", status );
return;
}
/* Obtain dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride, &tstride,
status );
if( *status==SAI__OK ) {
steptime = data->hdr->steptime;
if( steptime < VAL__SMLD ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": FITS header error, STEPTIME must be > 0",
status);
} else {
/* Frequency steps in the FFT */
df = 1. / (steptime * (double) ntslice );
}
}
/* Initialize arrays */
if( whitenoise ) for(i=0; i<nbolo; i++) whitenoise[i] = VAL__BADD;
if( fratio ) for(i=0; i<nbolo; i++) fratio[i] = VAL__BADD;
/* FFT the data and convert to polar power spectral density form */
pow = smf_fft_data( wf, data, NULL, 0, len, status );
smf_convert_bad( wf, pow, status );
smf_fft_cart2pol( wf, pow, 0, 1, status );
{
dim_t fdims[2];
smf_isfft( pow, NULL, NULL, fdims, NULL, NULL, status );
if( *status == SAI__OK ) nf=fdims[0];
}
/* Check for reasonble frequencies, and integer offsets in the array */
i_low = smf_get_findex( f_low, df, nf, status );
i_w1 = smf_get_findex( f_white1, df, nf, status );
i_w2 = smf_get_findex( f_white2, df, nf, status );
/* Get the quality pointer from the smfData so that we can mask known
bad bolometer. */
qua = smf_select_qualpntr( data, NULL, status );
/* The minimum required number of good values in a bolometer. */
mingood = ( gfrac > 0.0 ) ? ntslice*gfrac : 0;
/* Loop over detectors */
for( i=0; (*status==SAI__OK)&&(i<nbolo); i++ )
if( !qua || !(qua[i*bstride]&SMF__Q_BADB) ) {
/* Pointer to start of power spectrum */
base = pow->pntr[0];
base += nf*i;
/* Smooth the power spectrum */
smf_boxcar1D( base, nf, 1, window, NULL, 0, 1, NULL, status );
/* Measure the power */
if( *status == SAI__OK ) {
p_low = base[i_low];
smf_stats1D( base+i_w1, 1, i_w2-i_w1+1, NULL, 0, 0, &p_white, NULL, NULL,
&ngood, status );
/* It's OK if bad status was generated as long as a mean was calculated */
if( *status==SMF__INSMP ) {
errAnnul( status );
/* if we had no good data there was probably a problem with SMF__Q_BADB
so we simply go to the next bolometer */
if (ngood == 0) continue;
}
/* Count the number of initially good values for the current
bolometer. */
if( (*status==SAI__OK) && qua ) {
ngood = 0;
for( j=0; j<ntslice; j++ ) {
if( qua[i*bstride + j*tstride] == 0 ) ngood++;
}
/* Set bolometer to bad if no power detected, or the number of good
values is too low. */
if( (p_low <= 0) || (p_white <= 0) || (ngood < mingood) ) {
for( j=0; j<ntslice; j++ ) {
qua[i*bstride + j*tstride] |= SMF__Q_BADB;
}
}
}
}
if( (*status==SAI__OK) && (!qua || !(qua[i*bstride]&SMF__Q_BADB)) ) {
/* Power ratio requested */
if ( fratio ) {
fratio[i] = p_low/p_white;
}
/* Store values */
if( whitenoise ) {
/* Integrate the PSD by multiplying the average white noise
level by total number of samples and the frequency spacing:
this calculates the time-domain variance (in 200 Hz SCUBA-2
samples for example) assuming this level holds at all
frequencies. */
whitenoise[i] = p_white * ntslice * df;
/* If NEP set, scale this to variance in a 1-second average by
dividing by the sampling frequency (equivalent to
multiplying by sample length). */
if( nep ) {
whitenoise[i] *= steptime;
}
}
}
}
/* Clean up if the caller does not want to take over the power spectrum */
if( pow ) {
if (fftpow) {
*fftpow = pow;
} else {
smf_close_file( &pow, status );
}
}
}
void smf_filter2d_execute( ThrWorkForce *wf, smfData *data, smfFilter *filt,
int complement, int *status ) {
double *data_i=NULL; /* Imaginary part of the transformed data */
double *data_r=NULL; /* Real part of the transformed data */
smfData *fdata=NULL; /* Transform of data */
dim_t fdims[2]={0,0}; /* Frequency dimensions */
size_t i; /* loop counter */
size_t ndims=0; /* Number of real-space dimensions */
size_t nfdata; /* Total number of frequency data points */
smfData *varfilt=NULL; /* real-space square of supplied filter for var */
AstFrameSet *wcs=NULL; /* Copy of real-space WCS */
/* Main routine */
if (*status != SAI__OK) return;
/* Check for NULL pointers */
if( !data ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfData pointer", status );
return;
}
if( !filt ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfFilter pointer", status );
return;
}
if( filt->ndims != 2 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Filter must be 2-dimensional", status );
return;
}
if( smf_isfft( data, NULL, NULL, fdims, NULL, &ndims, status ) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": FFT'd data supplied!", status );
return;
}
if( ndims != 2 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": supplied data are not a 2-d map", status );
return;
}
/* Check that the filter dimensions are appropriate for the data */
for( i=0; i<ndims; i++ ) {
if( fdims[i] != filt->fdims[i] ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME
": Filter axis %zu has length %zu, doesn't match data %zu",
status, i, filt->fdims[i], fdims[i] );
return;
}
}
/* Dimensions of the transformed data */
nfdata = 1;
for( i=0; i<ndims; i++ ) {
if( filt->fdims[i] != fdims[i] ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME
": Filter axis %zu has dim %zu doesn't match data dim %zu",
status, i, filt->fdims[i], fdims[i]);
return;
}
nfdata *= fdims[i];
}
/* Using complement of the filter? */
if( complement ) smf_filter_complement( filt, status );
/* Get a copy of the wcs of the input map */
if( data->hdr && data->hdr->wcs ) {
wcs = astCopy( data->hdr->wcs );
}
/* Transform the data */
fdata = smf_fft_data( wf, data, NULL, 0, 0, status );
/* Copy the FFT of the data if we will also be filtering the VARIANCE
since this will get us a useful container of the correct dimensions
for the squared filter */
if( data->pntr[1] ) {
varfilt = smf_deepcopy_smfData( wf, fdata, 0, SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY |
SMF__NOCREATE_FILE |
SMF__NOCREATE_DA, 0, 0, status );
}
/* Apply the frequency-domain filter. */
if( *status == SAI__OK ) {
data_r = fdata->pntr[0];
data_i = data_r + nfdata;
if( filt->isComplex ) {
double ac, bd, aPb, cPd;
for( i=0; i<nfdata; i++ ) {
ac = data_r[i] * filt->real[i];
bd = data_i[i] * filt->imag[i];
aPb = data_r[i] + data_i[i];
cPd = filt->real[i] + filt->imag[i];
}
} else {
for( i=0; i<nfdata; i++ ) {
data_r[i] *= filt->real[i];
data_i[i] *= filt->real[i];
}
}
}
/* Transform back */
smf_fft_data( wf, fdata, data, 1, 0, status );
/* Insert the copy of original real-space WCS into the smfHead since
smf_fft_data does not currently calculate it for inverse
transforms. */
if( data->hdr ) {
/* Annul current wcs if it exists */
if( data->hdr->wcs ) {
data->hdr->wcs = astAnnul( data->hdr->wcs );
}
data->hdr->wcs = wcs;
}
/* If we have a VARIANCE component we also need to smooth it. This
is slightly complicated because we have to do the equivalent of a
real-space convolution between the variance map and the
element-wise square of the real-space filter. So we first stuff
the supplied filter into a smfData (frequency space), take its
inverse to real space and square it. We then transform back to
frequency space, and run it through smf_filter2d_execute to apply
it to the VARIANCE map (which is also stuffed into its own
smfData and then copied into the correct location of the supplied
smfData when finished. */
if( (data->pntr[1]) && (*status==SAI__OK) ) {
dim_t ndata;
double *ptr=NULL;
smfData *realfilter=NULL; /* Real space smfData container for var filter */
smfData *vardata=NULL; /* smfData container for variance only */
smfFilter *vfilt=NULL; /* The var filter */
/* Copy the filter into the smfData container and transform into the
time domain. */
ptr = varfilt->pntr[0];
memcpy(ptr, filt->real, nfdata*sizeof(*ptr));
if( filt->imag) {
memcpy(ptr+nfdata, filt->imag, nfdata*sizeof(*ptr));
} else {
memset(ptr+nfdata, 0, nfdata*sizeof(*ptr));
}
realfilter = smf_fft_data( wf, varfilt, NULL, 1, 0, status );
smf_close_file( wf, &varfilt, status );
/* Square each element of the real-space filter and then transform
back to the frequency domain and stuff into a smfFilter
(vfilt). We just point the real and imaginary parts of the
smfFilter to the respective regions of the smfData to save
memory/time, but we need to be careful when freeing at the
end. */
if( *status == SAI__OK ) {
ptr = realfilter->pntr[0];
smf_get_dims( realfilter, NULL, NULL, NULL, NULL, &ndata, NULL, NULL,
status );
if( *status == SAI__OK ) {
double norm = 1. / (double) ndata;
for(i=0; i<ndata; i++) {
/* Note that we need an additional normalization of N samples */
ptr[i] *= ptr[i] * norm;
}
}
}
varfilt = smf_fft_data( wf, realfilter, NULL, 0, 0, status );
if( *status == SAI__OK ) {
ptr = varfilt->pntr[0];
vfilt = smf_create_smfFilter( data, status );
vfilt->real = ptr;
if( filt->isComplex ) {
/* Only worry about imaginary part if the original filter was
complex. */
vfilt->isComplex = 1;
vfilt->imag = ptr + nfdata;
}
}
/* Now stuff the variance array into a smfData and filter it. */
vardata = smf_deepcopy_smfData( wf, data, 0, SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY |
SMF__NOCREATE_FILE |
SMF__NOCREATE_DA, 0, 0, status );
if( *status == SAI__OK ) {
ptr = vardata->pntr[0];
memcpy( ptr, data->pntr[1], ndata*sizeof(*ptr) );
smf_filter2d_execute( wf, vardata, vfilt, 0, status );
}
/* Finally, copy the filtered variance into our output filtered smfData */
if( *status == SAI__OK ) {
ptr = data->pntr[1];
memcpy( ptr, vardata->pntr[0], ndata*sizeof(*ptr) );
}
/* Clean up */
if( realfilter ) smf_close_file( wf, &realfilter, status );
if( vardata ) smf_close_file( wf, &vardata, status );
if( vfilt ) {
vfilt->real = NULL;
vfilt->imag = NULL;
vfilt = smf_free_smfFilter( vfilt, status );
}
}
/* Return the filter to its original state if required */
if( complement == -1 ) smf_filter_complement( filt, status );
/* Clean up */
if( varfilt ) smf_close_file( wf, &varfilt, status );
if( fdata ) smf_close_file( wf, &fdata, status );
}
int smf_dataOrder( ThrWorkForce *wf, smfData *data, int isTordered,
int *status ) {
/* Local Variables */
size_t bstr1; /* bolometer index stride input */
size_t bstr2; /* bolometer index stride output */
dim_t i; /* loop counter */
int inPlace=0; /* If set change array in-place */
dim_t nbolo; /* Number of bolometers */
dim_t ndata; /* Number of data points */
dim_t newdims[3]; /* Size of each dimension new buffer */
int newlbnd[3]; /* New pixel origin */
dim_t ntslice; /* Number of time slices */
size_t tstr1; /* time index stride input */
size_t tstr2; /* time index stride output */
int waschanged = 0; /* did we chagne the order? */
int writable; /* Is the NDF writable? */
int freeold; /* Free the old array if change is not in place? */
/* Main routine */
if (*status != SAI__OK) return waschanged;
/* Check for valid isTordered */
if( (isTordered != 0) && (isTordered != 1) ) {
*status = SAI__ERROR;
msgSeti("ISTORDERED",isTordered);
errRep( "", FUNC_NAME ": Invalid isTordered (0/1): ^ISTORDERED", status);
return waschanged;
}
/* Check for a valid data */
if( !data ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL data supplied", status);
return waschanged;
}
/* If value of isTordered matches current value in smfData return */
if( data->isTordered == isTordered ) return waschanged;
/* Make sure we're looking at 3-dimensions of bolo data */
if( data->ndims != 3 ) {
*status = SMF__WDIM;
msgSeti("NDIMS",data->ndims);
errRep( "", FUNC_NAME
": Don't know how to handle ^NDIMS dimensions, should be 3.",
status);
return waschanged;
}
/* We shouldn't be trying to change the order of FFT'd data */
if( smf_isfft( data, NULL, NULL, NULL, NULL, NULL, status ) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": Possible programming error, attempting to re-order FFT of a "
"2D map!", status );
return waschanged;
}
/* we are going to change */
waschanged = 1;
/* inPlace=1 if smfData was mapped! Free the old array if it was not
stored in a file. */
freeold = 1;
if( data->file ){
if (data->file->fd ){
inPlace = 1;
freeold = 0;
/* Only change NDF values in-place if write access is abvaialble for
the NDF. */
} else if( data->file->ndfid ) {
freeold = 0;
ndfIsacc( data->file->ndfid, "WRITE", &writable, status );
if( writable ) inPlace = 1;
}
}
/* Calculate input data dimensions (before changing order) */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstr1, &tstr1,
status);
/* What will the dimensions/strides be in the newly-ordered array? */
if( isTordered ) {
newdims[0] = (data->dims)[1];
newdims[1] = (data->dims)[2];
newdims[2] = (data->dims)[0];
newlbnd[0] = (data->lbnd)[1];
newlbnd[1] = (data->lbnd)[2];
newlbnd[2] = (data->lbnd)[0];
bstr2 = 1;
tstr2 = nbolo;
} else {
newdims[0] = (data->dims)[2];
newdims[1] = (data->dims)[0];
newdims[2] = (data->dims)[1];
newlbnd[0] = (data->lbnd)[2];
newlbnd[1] = (data->lbnd)[0];
newlbnd[2] = (data->lbnd)[1];
bstr2 = ntslice;
tstr2 = 1;
}
/* Loop over elements of data->ptr and re-form arrays */
for( i=0; i<2; i++ ) {
data->pntr[i] = smf_dataOrder_array( wf, data->pntr[i], data->dtype,
data->dtype, ndata,
ntslice, nbolo, tstr1, bstr1, tstr2,
bstr2, inPlace, freeold, status );
}
/* And Quality */
data->qual = smf_dataOrder_array( wf, data->qual, SMF__QUALTYPE, SMF__QUALTYPE,
ndata, ntslice, nbolo, tstr1, bstr1, tstr2,
bstr2, inPlace, freeold, status );
/* If NDF associated with data, modify dimensions of the data */
if( data->file && (data->file->ndfid != NDF__NOID) ) {
msgOutif(MSG__DEBUG, " ", FUNC_NAME
": Warning - current implementation does not modify NDF "
"dimensions to match re-ordered data array", status);
}
/* If there is a LUT re-order it here */
data->lut = smf_dataOrder_array( wf, data->lut, SMF__INTEGER, SMF__INTEGER, ndata,
ntslice, nbolo, tstr1, bstr1, tstr2, bstr2,
inPlace, 1, status );
/* Set the new dimensions in the smfData */
if( *status == SAI__OK ) {
memcpy( data->dims, newdims, 3*sizeof(*newdims) );
memcpy( data->lbnd, newlbnd, 3*sizeof(*newlbnd) );
data->isTordered = isTordered;
}
/* Force any external quality to same ordering */
if (data->sidequal) {
int qchanged = 0;
qchanged = smf_dataOrder( wf, data->sidequal, isTordered, status );
/* and indicate if we changed anything (but not if we did not) */
if (qchanged) waschanged = qchanged;
}
/* Re-order the axes in the time-series WCS FrameSet */
if( data->hdr && data->hdr->tswcs ) {
smf_tswcsOrder( &(data->hdr->tswcs), isTordered, status );
}
/* If the re-ordering was not done in-place, then the new buffer must
have been allocated here. Set a flag so that smf_close_file knows to
deallocate the memory. */
if( ! inPlace ) data->isdyn = 1;
return waschanged;
}
smfData *smf_fft_avpspec( const smfData *pspec, smf_qual_t *quality,
size_t qstride, smf_qual_t mask, double *weights,
int *status ) {
dim_t fdims[2]; /* Lengths of frequency-space axes */
size_t i; /* Loop counter */
double *idptr=NULL; /* Pointer to input data */
double mean; /* Mean value at time slice */
dim_t nbolo=0; /* Number of detectors */
size_t ngood; /* Number of good samples */
size_t ndata; /* Total number of data points */
dim_t ntslice=0; /* Number of time slices */
dim_t nf=0; /* Number of frequencies in FFT */
double *odptr=NULL; /* Pointer to output data */
double *ovptr=NULL; /* Pointer to output variance */
dim_t rdims[2]; /* Lengths of real-space axes */
smfData *retdata=NULL; /* Returned power spectrum */
double sigma; /* RMS value at time slice */
if (*status != SAI__OK) return NULL;
/* Check for NULL pointer */
if( pspec == NULL ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData pointer is NULL", status );
return NULL;
}
/* Check that we have frequency-domain input */
if( !smf_isfft( pspec, rdims, &nbolo, fdims, NULL, NULL, status ) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Input data are not FFT!", status );
return NULL;
}
ntslice = rdims[0];
nf = fdims[0];
/* Check that we don't have a 1-d input power spectrum */
if( pspec->ndims != 4 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Input data are not 4-dimensional!", status );
return NULL;
}
idptr = pspec->pntr[0];
if( !idptr ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null input data", status );
return NULL;
}
/* Assume qstride = bolo stride of main array if not specified */
if( !qstride ) {
qstride = nf;
}
/* Create a new smfData, copying over everything except for the bolo
data itself */
retdata = smf_deepcopy_smfData( pspec, 0, SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY |
SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS, 0, 0, status );
if( *status == SAI__OK ) {
/* Allocate space for the averaged power spectrum */
retdata->ndims = 4;
retdata->dims[0] = nf;
retdata->dims[1] = 1;
retdata->dims[2] = 1;
retdata->dims[3] = 2;
retdata->dtype = SMF__DOUBLE;
ndata=1;
for( i=0; i<retdata->ndims; i++ ) {
ndata *= retdata->dims[i];
}
/* Allocate space for DATA and VARIANCE components */
retdata->pntr[0] = astCalloc( ndata, smf_dtype_sz(retdata->dtype,status) );
retdata->pntr[1] = astCalloc( ndata, smf_dtype_sz(retdata->dtype,status) );
/* Pointers to output data */
odptr = retdata->pntr[0];
ovptr = retdata->pntr[1];
/* Call the returned data tordered even though it is irrelevant */
retdata->isTordered=0;
/* Since we assumed we're in power/polar form just need to calculate
average and scatter of the amplitude coefficients. The phase
coefficients are left at their initialized values of 0. */
for( i=0; i<nf; i++ ) {
/* The bolometer stride is nf */
smf_weightstats1D( idptr+i, nf, nbolo, quality, qstride, mask,
weights, 1, &mean, &sigma, &ngood, status );
if( *status == SMF__INSMP ) {
/* If not enough samples just annul and set bad value */
errAnnul( status );
odptr[i] = VAL__BADD;
ovptr[i] = VAL__BADD;
} else {
odptr[i] = mean;
ovptr[i] = sigma*sigma;
}
}
}
return retdata;
}
void smf_grp_related( const Grp *igrp, const size_t grpsize,
const int grouping, const int checksubinst,
double maxlen_s, double *srate_maxlen,
AstKeyMap *keymap, dim_t *maxconcatlen,
dim_t *maxfilelen, smfGroup **group,
Grp **basegrp, dim_t *pad, int *status ) {
/* Local variables */
size_t *chunk=NULL; /* Array of flags for continuous chunks */
dim_t * chunklen = NULL; /* Length of continuous chunk */
size_t currentindex = 0; /* Counter */
char cwave[10]; /* String containing wavelength */
smfData *data = NULL; /* Current smfData */
double downsampscale=0; /* Angular scale downsampling size */
double downsampfreq=0; /* Target downsampling frequency */
AstKeyMap * grouped = NULL; /* Primary AstKeyMap for grouping */
size_t i; /* Loop counter for index into Grp */
int isFFT=0; /* Set if data are 4d FFT */
size_t j; /* Loop counter */
int *keepchunk=NULL; /* Flag for chunks that will be kept */
dim_t maxconcat=0; /* Longest continuous chunk length */
dim_t maxflen=0; /* Max file length in time steps */
dim_t maxlen=0; /* Maximum concat length in samples */
int maxlen_scaled=0; /* Set once maxlen has been scaled, if needed */
dim_t maxpad=0; /* Maximum padding neeed for any input file */
size_t maxrelated = 0; /* Keep track of max number of related items */
size_t *new_chunk=NULL; /* keeper chunks associated with subgroups */
dim_t *new_tlen=NULL; /* tlens for new_subgroup */
size_t ngroups = 0; /* Counter for subgroups to be stored */
size_t nkeep = 0; /* Number of chunks to keep */
dim_t * piecelen = NULL; /* Length of single file */
smf_subinst_t refsubinst; /* Subinst of first file */
size_t **subgroups = NULL; /* Array containing index arrays to parent Grp */
smf_subinst_t subinst; /* Subinst of current file */
if ( *status != SAI__OK ) return;
if( maxlen_s < 0 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": maxlen_s cannot be < 0!", status );
return;
}
/* Get downsampling parameters */
if( keymap ) {
smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, &downsampscale, &downsampfreq,
NULL, NULL, NULL, NULL, status );
if( downsampscale && downsampfreq ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": both downsampscale and downsampfreq are set",
status );
return;
}
}
/* Initialize refcwave */
refsubinst = SMF__SUBINST_NONE;
/* Loop over files in input Grp: remember Grps are indexed from 1 */
grouped = astKeyMap( "SortBy=KeyUp" );
for (i=1; i<=grpsize; i++) {
char newkey[128];
char dateobs[81];
char subarray[10];
size_t nrelated = 0;
AstKeyMap * filemap = NULL;
AstKeyMap * indexmap = NULL;
/* First step: open file and harvest metadata */
smf_open_file( NULL, igrp, i, "READ", SMF__NOCREATE_DATA, &data, status );
if (*status != SAI__OK) break;
if( i==1 ) {
isFFT = smf_isfft( data, NULL, NULL, NULL, NULL, NULL, status );
} else if( smf_isfft(data, NULL, NULL, NULL, NULL, NULL, status) != isFFT ){
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": mixture of time-series and FFT data encountered!",
status );
break;
}
/* If maxlen has not been set, do it here */
if( !maxlen && maxlen_s && data->hdr->steptime) {
maxlen = (dim_t) (maxlen_s / data->hdr->steptime );
}
/* Return srate_maxlen if requested: may want to know this number
even if maxlen_s is not set. Only calculate once, although it
gets overwritten once later if down-sampling. */
if( (i==1) && srate_maxlen && data->hdr->steptime ) {
*srate_maxlen = 1. / (double) data->hdr->steptime;
}
/* If requested check to see if we are mixing wavelengths */
if( checksubinst ) {
if( refsubinst == SMF__SUBINST_NONE ) {
refsubinst = smf_calc_subinst( data->hdr, status );
}
subinst = smf_calc_subinst( data->hdr, status );
if( subinst != refsubinst ) {
const char *refsubstr = smf_subinst_str( refsubinst, status );
const char *substr = smf_subinst_str( subinst, status );
*status = SAI__ERROR;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
msgSetc( "REFSUB", refsubstr );
msgSetc( "SUB", substr );
errRep( "", FUNC_NAME
": ^FILE uses sub-instrument ^SUB which doesn't match "
"reference ^REFSUB", status );
}
}
/* Want to form a key that will be unique for a particular subscan
We know that DATE-OBS will be set for SCUBA-2 files and be the same
for a single set. Prefix by wavelength if we are grouping by wavelength.
*/
newkey[0] = '\0';
smf_find_subarray( data->hdr, subarray, sizeof(subarray), NULL, status );
if( grouping == 1 ) {
/* Group different wavelengths separately */
smf_fits_getS( data->hdr, "WAVELEN", cwave, sizeof(cwave), status);
one_strlcat( newkey, cwave, sizeof(newkey), status );
one_strlcat( newkey, "_", sizeof(newkey), status );
}
if( grouping == 2 ) {
/* Group different subarrays separately */
one_strlcat( newkey, subarray, sizeof(newkey), status );
}
smf_fits_getS( data->hdr, "DATE-OBS", dateobs, sizeof(dateobs), status );
one_strlcat( newkey, dateobs, sizeof(newkey), status );
/* Include the dimentionality of the time series in the primary key
so that we do not end up doing something confusing like relating
a truncated file with a full length file */
if (*status == SAI__OK) {
dim_t dims[3];
char formatted[32];
smf_get_dims( data, &dims[0], &dims[1], NULL, &dims[2], NULL, NULL, NULL,
status );
sprintf(formatted, "_%" DIM_T_FMT "_%" DIM_T_FMT "_%" DIM_T_FMT, dims[0], dims[1], dims[2]);
one_strlcat( newkey, formatted, sizeof(newkey), status );
}
/* May want to read the dimensionality of the file outside of loop
so that we can compare values when storing in the keymap */
/* Now we want to create a keymap based on this key */
if (!astMapGet0A( grouped, newkey, &filemap ) ) {
int itemp = 0;
double steptime = data->hdr->steptime;
dim_t ntslice = 0;
dim_t thispad; /* Padding neeed for current input file */
filemap = astKeyMap( " " );
astMapPut0A( grouped, newkey, filemap, NULL );
/* Fill up filemap with general information on this file */
smf_find_seqcount( data->hdr, &itemp, status );
astMapPut0I( filemap, "SEQCOUNT", itemp, NULL );
smf_fits_getI( data->hdr, "NSUBSCAN", &itemp, status );
astMapPut0I( filemap, "NSUBSCAN", itemp, NULL );
/* Number of time slices */
smf_get_dims( data, NULL, NULL, NULL, &ntslice, NULL, NULL, NULL,
status );
/* Find length of down-sampled data, new steptime and maxlen */
if( (downsampscale || downsampfreq) && data->hdr && (*status==SAI__OK) ) {
double scalelen;
if( downsampscale ) {
if( data->hdr->scanvel != VAL__BADD ) {
double oldscale = steptime * data->hdr->scanvel;
scalelen = oldscale / downsampscale;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
scalelen = VAL__BADD;
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep( "", FUNC_NAME ": can't resample ^FILE because it has "
"unknown scan velocity", status );
}
} else {
if( steptime ) {
double oldsampfreq = 1./steptime;
scalelen = downsampfreq / oldsampfreq;
} else {
*status = SAI__ERROR;
scalelen = VAL__BADD;
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep( "", FUNC_NAME ": can't resample ^FILE because it has "
"unknown sample rate", status );
}
}
/* only down-sample if it will be a reasonable factor */
if( (*status==SAI__OK) && (scalelen <= SMF__DOWNSAMPLIMIT) ) {
smf_smfFile_msg(data->file, "FILE", 1, "" );
msgOutiff( MSG__VERB, "", FUNC_NAME
": will down-sample file ^FILE from %5.1lf Hz to "
"%5.1lf Hz", status, (1./steptime), (scalelen/steptime) );
ntslice = round(ntslice * scalelen);
/* If maxlen has been requested, and we have not already worked
out a scaled version (just uses the sample rates for the first
file... should be close enough -- the alternative is a 2-pass
system). */
if( !maxlen_scaled ) {
maxlen = round(maxlen*scalelen);
maxlen_scaled = 1;
msgOutiff( MSG__VERB, "", FUNC_NAME
": requested maxlen %g seconds = %" DIM_T_FMT " down-sampled "
"time-slices", status, maxlen_s, maxlen );
/* Return updated srate_maxlen for down-sampling if requested */
if( srate_maxlen ) {
*srate_maxlen = scalelen/steptime;
}
}
}
}
/* Check that an individual file is too long (we assume related
files are all the same) */
if( maxlen && (ntslice > maxlen) && *status == SAI__OK) {
*status = SAI__ERROR;
msgSeti("NTSLICE",ntslice);
msgSeti("MAXLEN",maxlen);
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep(FUNC_NAME,
"Number of time steps in file ^FILE time exceeds maximum "
"(^NTSLICE>^MAXLEN)", status);
}
/* Scaled values of ntslice and maximum length */
astMapPut0I( filemap, "NTSLICE", ntslice, NULL );
/* Work out the padding needed for this file including downsampling. */
if( keymap ) {
thispad = smf_get_padding( keymap, 0, data->hdr, VAL__BADD, status );
if( thispad > maxpad ) maxpad = thispad;
} else {
thispad = 0;
}
astMapPut0I( filemap, "PADDING", thispad, NULL );
/* Update maxflen */
if( ntslice > maxflen ) {
maxflen = ntslice;
}
/* Store OBSID or OBSIDSS depending on whether we are grouping by wavelength */
if (grouping) {
astMapPut0C( filemap, "OBSID", data->hdr->obsidss, NULL );
} else {
char obsid[81];
smf_getobsidss( data->hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status );
astMapPut0C( filemap, "OBSID", obsid, NULL );
}
}
/* Store the file index in another keymap indexed by subarray */
if ( !astMapGet0A( filemap, "GRPINDICES", &indexmap ) ) {
indexmap = astKeyMap( "SortBy=KeyUp" );
astMapPut0A( filemap, "GRPINDICES", indexmap, NULL );
}
astMapPut0I( indexmap, subarray, i, NULL );
/* Need to track the largest number of related subarrays in a single slot */
nrelated = astMapSize( indexmap );
if (nrelated > maxrelated) maxrelated = nrelated;
/* Free resources */
filemap = astAnnul( filemap );
indexmap = astAnnul( indexmap );
smf_close_file( NULL, &data, status );
}
/* We now know how many groups there are */
ngroups = astMapSize( grouped );
/* Sort out chunking. The items are sorted by date and then by wavelength.
We define a continuous chunk if it has the same OBSID, the same SEQCOUNT
and NSUBSCAN increments by one from the previous entry.
Also count number of related items in each slot.
*/
if (*status == SAI__OK) {
typedef struct { /* somewhere to store the values easily */
char obsid[81];
char related[81]; /* for concatenated subarrays */
int nsubscan;
int seqcount;
} smfCompareSeq;
smfCompareSeq current;
smfCompareSeq previous;
dim_t totlen = 0;
size_t thischunk;
/* Get the chunk flags and also store the size of the chunk */
chunk = astCalloc( ngroups, sizeof(*chunk) );
chunklen = astCalloc( ngroups, sizeof(*chunklen) );
piecelen = astCalloc( ngroups, sizeof(*piecelen) );
thischunk = 0; /* The current chunk */
for (i=0; i<ngroups; i++) {
AstKeyMap * thismap = NULL;
AstKeyMap * grpindices = NULL;
const char * tempstr = NULL;
int thistlen = 0;
size_t nsubarrays = 0;
/* Get the keymap entry for this slot */
astMapGet0A( grouped, astMapKey(grouped, i), &thismap );
/* Get info for length limits */
astMapGet0I( thismap, "NTSLICE", &thistlen );
piecelen[i] = thistlen;
if (isFFT) {
/* Never concatenate FFT data */
thismap = astAnnul(thismap);
chunk[i] = i;
chunklen[i] = thistlen;
continue;
}
/* Get indices information and retrieve the sub-instrument names
in sort order to concatenate for comparison. We only store in
a continuous chunk if we have the same subarrays for the whole
chunk. */
astMapGet0A( thismap, "GRPINDICES", &grpindices );
nsubarrays = astMapSize( grpindices );
(current.related)[0] = '\0';
for (j = 0; j < nsubarrays; j++ ) {
one_strlcat( current.related, astMapKey(grpindices, j), sizeof(current.related), status );
}
grpindices = astAnnul( grpindices );
/* Fill in the current struct */
astMapGet0I( thismap, "SEQCOUNT", &(current.seqcount) );
astMapGet0I( thismap, "NSUBSCAN", &(current.nsubscan) );
astMapGet0C( thismap, "OBSID", &tempstr );
one_strlcpy( current.obsid, tempstr, sizeof(current.obsid), status );
/* First chunk is special, else compare */
if (i == 0) {
totlen = thistlen;
} else {
if ( ( current.seqcount == previous.seqcount ) &&
( current.nsubscan - previous.nsubscan == 1 ) &&
( strcmp( current.obsid, previous.obsid ) == 0 ) &&
( strcmp( current.related, previous.related ) == 0 ) ) {
/* continuous - check length */
totlen += thistlen;
if ( maxlen && totlen > maxlen ) {
thischunk++;
totlen = thistlen; /* reset length */
} else {
/* Continuous */
}
} else {
/* discontinuity */
thischunk++;
totlen = thistlen; /* Update length of current chunk */
}
}
chunklen[thischunk] = totlen;
chunk[i] = thischunk;
memcpy( &previous, ¤t, sizeof(current) );
thismap = astAnnul( thismap );
}
}
/* Decide if we are keeping a chunk by looking at the length. */
maxconcat = 0;
nkeep = 0;
keepchunk = astMalloc( ngroups*sizeof(*keepchunk) );
for (i=0; i<ngroups; i++) {
size_t thischunk;
thischunk = chunk[i];
if ( chunklen[thischunk] < SMF__MINCHUNKSAMP ) {
/* Warning message */
msgSeti("LEN",chunklen[thischunk]);
msgSeti("MIN",SMF__MINCHUNKSAMP);
msgOut( " ", "SMF_GRP_RELATED: ignoring short chunk (^LEN<^MIN)",
status);
keepchunk[i] = 0;
} else {
keepchunk[i] = 1;
if (maxconcat < chunklen[thischunk]) maxconcat = chunklen[thischunk];
nkeep++;
}
}
/* If no useful chunks generate an error */
if( (*status==SAI__OK) && (!nkeep) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No useful chunks.", status );
goto CLEANUP;
}
/* Allocate a subgroup array of the right size and fill it. They keymap
is sorted by date (and wavelength) so we can always index into it by using
indices from the subgroup. */
subgroups = astCalloc( nkeep, sizeof(*subgroups) );
new_chunk = astCalloc( nkeep, sizeof(*new_chunk) );
new_tlen = astCalloc( nkeep, sizeof(*new_tlen) );
currentindex = 0;
for (i=0;i<ngroups;i++) {
AstKeyMap * thismap = NULL;
AstKeyMap * grpindices = NULL;
size_t nsubarrays = 0;
size_t *indices = astCalloc( maxrelated, sizeof(*indices) );
/* skip if we are dropping this chunk */
if (!keepchunk[i]) continue;
/* Get the keymap entry for this slot */
astMapGet0A( grouped, astMapKey(grouped, i), &thismap );
/* Get the indices keymap */
astMapGet0A( thismap, "GRPINDICES", &grpindices );
nsubarrays = astMapSize( grpindices );
for (j=0; j<nsubarrays; j++) {
int myindex;
astMapGet0I( grpindices, astMapKey(grpindices, j), &myindex );
indices[j] = myindex;
}
grpindices = astAnnul( grpindices );
thismap = astAnnul( thismap );
subgroups[currentindex] = indices;
new_chunk[currentindex] = chunk[i];
new_tlen[currentindex] = piecelen[i];
currentindex++;
}
/* Create the smfGroup */
*group = smf_construct_smfGroup( igrp, subgroups, new_chunk, new_tlen,
nkeep, maxrelated, 0, status );
/* Return maxfilelen if requested */
if( maxfilelen ) {
*maxfilelen = maxflen;
}
/* Return maxconcatlen if requested */
if( maxconcatlen ) {
*maxconcatlen = maxconcat;
}
/* Create a base group for output files if required */
/* Create a base group of filenames */
if (*status == SAI__OK && basegrp ) {
*basegrp = smf_grp_new( (*group)->grp, "Base Group", status );
/* Loop over time chunks */
for( i=0; (*status==SAI__OK)&&(i<(*group)->ngroups); i++ ) {
size_t idx;
/* Check for new continuous chunk */
if( i==0 || ( (*group)->chunk[i] != (*group)->chunk[i-1]) ) {
/* Loop over subarray */
for( idx=0; idx<(*group)->nrelated; idx++ ) {
size_t grpindex = (*group)->subgroups[i][idx];
if ( grpindex > 0 ) {
ndgCpsup( (*group)->grp, grpindex, *basegrp, status );
}
}
}
}
}
CLEANUP:
keepchunk = astFree( keepchunk );
chunk = astFree( chunk );
chunklen = astFree( chunklen );
piecelen = astFree( piecelen );
grouped = astAnnul( grouped );
if( *status != SAI__OK ) {
/* free the group */
if (basegrp && *basegrp) grpDelet( basegrp, status );
if (group && *group) {
smf_close_smfGroup( group, status );
} else {
/* have to clean up manually */
new_chunk = astFree( new_chunk );
new_tlen = astFree( new_tlen );
if( subgroups ) {
size_t isub;
for( isub=0; isub<nkeep; isub++ ) {
subgroups[isub] = astFree( subgroups[isub] );
}
subgroups = astFree( subgroups );
}
}
}
/* Return the maximum padding if required. */
if( pad ) *pad = maxpad;
}
void smurf_sc2filtermap( int *status ) {
Grp *fgrp = NULL; /* Output filter group */
smfFilter *filt=NULL; /* Filter */
double filt_edgehigh=0; /* High-pass filter */
double filt_edgelow=0; /* Low-pass filter */
size_t fsize; /* Number of files in fgrp */
size_t i; /* Loop (grp) counter */
smfData *idata; /* Pointer to input smfData */
Grp *igrp = NULL; /* Input group of files */
int isfft=0; /* Are data fft or real space? */
int *mask=NULL; /* Mask indicating where bad data are */
size_t ndata=0; /* Number of pixels in the map */
size_t ndims; /* Number of real space dimensions */
smfData *odata=NULL; /* Pointer to output smfData to be exported */
Grp *ogrp = NULL; /* Output group of files */
size_t outsize; /* Number of files in output group */
size_t size; /* Number of files in input group */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
smfData *wrefmap=NULL; /* Whitening reference map */
int whiten; /* Applying whitening filter? */
Grp *wgrp = NULL; /* Whitening reference map group */
size_t wsize; /* Size of wgrp */
int zerobad; /* Zero VAL__BADD before taking FFT? */
/* Main routine */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get input file(s) */
kpg1Rgndf( "IN", 0, 1, "", &igrp, &size, status );
size = grpGrpsz( igrp, status );
if (size > 0) {
int parstate=0; /* ADAM parameter state */
/* Get output file(s) */
kpg1Wgndf( "OUT", igrp, size, size, "More output files required...",
&ogrp, &outsize, status );
/* Write out the filter? */
parState( "OUTFILTER", &parstate, status );
if( parstate != PAR__GROUND ) {
kpg1Wgndf( "OUTFILTER", igrp, size, size,
"More output filter files required...",
&fgrp, &fsize, status );
}
}
/* Are we going to zero bad values first? */
parGet0l( "ZEROBAD", &zerobad, status );
/* High/low-pass filters? */
parGet0d( "FILT_EDGEHIGH", &filt_edgehigh, status );
parGet0d( "FILT_EDGELOW", &filt_edgelow, status );
/* Are we applying a spatial whitening filter? */
parGet0l( "WHITEN", &whiten, status );
if( whiten ) {
/* We also need the reference map to measure the whitening filter. We
make a deep copy of it so that we can set bad values to 0 etc. */
smfData *tempdata=NULL;
kpg1Rgndf( "whiterefmap", 0, 1, "", &wgrp, &wsize, status );
if( (*status == SAI__OK) && (wsize != 1) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": WHITEREFMAP must be a single reference map",
status );
}
smf_open_file( wgrp, 1, "READ", SMF__NOTTSERIES, &tempdata, status );
wrefmap = smf_deepcopy_smfData( tempdata, 0, 0, 0, 0, status );
smf_close_file( &tempdata, status );
/* Set VAL__BADD to zero if requested */
if( (*status==SAI__OK) && zerobad ) {
double *d=NULL;
size_t j;
ndata=1;
for( j=0; j<wrefmap->ndims; j++ ) ndata *= wrefmap->dims[j];
d = wrefmap->pntr[0];
if( d ) {
for( j=0; j<ndata; j++ ) {
if( d[j] == VAL__BADD ) {
d[j] = 0;
}
}
}
}
}
for( i=1;(*status==SAI__OK)&&i<=size; i++ ) {
smf_open_file( igrp, i, "READ", SMF__NOTTSERIES, &idata, status );
isfft = smf_isfft( idata, NULL, NULL, NULL, NULL, &ndims, status);
if( (*status==SAI__OK) && isfft ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Input data are FFT, not real-space!\n",
status );
break;
}
if( (*status==SAI__OK) && (ndims != 2) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Input data not a 2D map!\n",
status );
break;
}
/* smf_filter_execute operates in-place, so first create the output
data as a copy of the input */
odata = smf_deepcopy_smfData( idata, 0, 0, 0, 0, status );
/* Set VAL__BADD to zero if requested */
if( (*status==SAI__OK) && zerobad ) {
double *d=NULL;
size_t j, k;
ndata=1;
for( j=0; j<odata->ndims; j++ ) ndata *= odata->dims[j];
mask = astCalloc( ndata, sizeof(*mask) );
/* Do both DATA and VARIANCE */
if( *status == SAI__OK ) {
for( k=0; k<2; k++ ) {
d = odata->pntr[k];
if( d ) {
for( j=0; j<ndata; j++ ) {
if( d[j] == VAL__BADD ) {
d[j] = 0;
mask[j] = 1;
}
}
}
}
}
}
/* Measure and apply the whitening filter. We need to do this
every time because the dimensions of filt need to match idata
(not the wrefmap) and they might be different each time. We
could try to be more clever in the future if this is too slow. */
filt = smf_create_smfFilter( idata, status );
/* Set to the identity in case no whitening is applied */
msgOut( "", TASK_NAME ": initializing filter", status );
smf_filter_ident( filt, 0, status );
if( whiten ) {
msgOut( "", TASK_NAME ": whitening the filter", status );
smf_filter2d_whiten( wf, filt, wrefmap, 0, 0, 3, status );
}
if( filt_edgelow ) {
msgOutf( "", TASK_NAME ": applying low-pass at < %lg 1/arcsec", status,
filt_edgelow );
smf_filter2d_edge( filt, filt_edgelow, 1, status );
}
if( filt_edgehigh ) {
msgOutf( "", TASK_NAME ": applying high-pass at >= %lg 1/arcsec", status,
filt_edgehigh );
smf_filter2d_edge( filt, filt_edgehigh, 0, status );
}
smf_filter_execute( wf, odata, filt, 0, 0, status );
/* Set bad values from the mask */
if( mask ) {
double *d=NULL;
size_t j, k;
/* Do both DATA and VARIANCE */
for( k=0; k<2; k++ ) {
d = odata->pntr[k];
if( d ) {
for( j=0; j<ndata; j++ ) {
if( mask[j] ) {
d[j] = VAL__BADD;
}
}
}
}
}
/* Export the data to a new file */
smf_write_smfData( odata, NULL, NULL, ogrp, i, 0, MSG__NORM, status );
/* Write out filters? */
if( fgrp ) smf_write_smfFilter( filt, NULL, fgrp, i, status );
if( filt ) smf_free_smfFilter( filt, status );
}
/* Tidy up after ourselves */
if( fgrp ) grpDelet( &fgrp, status);
if( igrp ) grpDelet( &igrp, status);
if( ogrp ) grpDelet( &ogrp, status);
if( wgrp ) grpDelet( &wgrp, status );
if( odata ) smf_close_file( &odata, status );
if( wrefmap ) smf_close_file( &wrefmap, status );
if( mask ) mask = astFree( mask );
ndfEnd( status );
/* Ensure that FFTW doesn't have any used memory kicking around */
fftw_cleanup();
}
void smf_filter2d_whiten( ThrWorkForce *wf, smfFilter *filt, smfData *map,
double minfreq, double maxfreq, size_t smooth,
int *status ) {
double A; /* Amplitude 1/f component */
double B; /* exponent of 1/f component */
double df; /* Frequency spacing in ref pspec */
size_t i; /* Loop counter */
size_t j; /* Loop counter */
size_t ndims=0; /* Number of real-space dimensions */
size_t nf=0; /* Number of frequencies in ref pspec */
smfData *map_fft=NULL; /* FFT of the map */
smfData *pspec=NULL; /* Az-averaged PSPEC of map */
double *pspec_data=NULL; /* Pointer to DATA comp. of pspec */
double *smoothed_filter=NULL; /* Smoothed power spectrum */
double W; /* White noise level */
if( *status != SAI__OK ) return;
if( !filt ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfFilter supplied.", status );
return;
}
if( filt->ndims != 2 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": supplied filter is not for 2-d map.",
status );
return;
}
if( smf_isfft( map, NULL, NULL, NULL, NULL, &ndims, status ) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": supplied map is FFT instead of real space!",
status );
return;
}
/* Check for reasonable frequencies -- noting that maxfreq=0 defaults to
nyquist. */
if( maxfreq && (maxfreq < minfreq) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": maxfreq < minfreq!", status );
return;
}
/* Calculate azimuthally-averaged angular power spectrum */
map_fft = smf_fft_data( wf, map, NULL, 0, 0, status );
smf_fft_cart2pol( wf, map_fft, 0, 1, status );
pspec = smf_fft_2dazav( map_fft, &df, status );
if( *status == SAI__OK ) {
nf = pspec->dims[0];
pspec_data = pspec->pntr[0];
}
smf_close_file( &map_fft, status );
/* Fit power-law + white level model */
smf_fit_pspec( pspec_data, pspec->dims[0], 10, df, minfreq, 0.1,
maxfreq, &A, &B, &W, status );
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": P(f) = %lg*f^%lg + %lg\n", status, A, B, W );
if( smooth ) {
int whichaxis=0; /* index of higher-resolution axis */
double df_s; /* Frequency spacing of smoothed_filter */
double nf_s; /* Size of the smoothed 1-d filter */
/* Re-sample the radial power spectrum measured in the map on to
an array that corresponds to the highest-resolution (longer
real-space) dimension of the supplied filter */
if( filt->rdims[1] > filt->rdims[0] ) {
whichaxis = 1;
}
df_s = filt->df[whichaxis];
nf_s = filt->rdims[whichaxis]/2 + 1;
smoothed_filter = astMalloc( nf_s*sizeof(*smoothed_filter) );
if( *status == SAI__OK ) {
/* Nearest-neighbour... */
for( i=0; i<nf_s; i++ ) {
size_t nearest = round(i * df_s / df );
if( nearest >= nf ) nearest = nf-1;
smoothed_filter[i] = pspec_data[nearest];
}
}
/* Smooth pspec to estimate the radial power spectrum, but normalize
by the white-noise level from the fit to preserve
normalization -------------------------------------------------------- */
smf_tophat1D( smoothed_filter, nf_s, smooth, NULL, 0, 0.5, status );
/* Replace bad values (where not enough values to calculate median) with
the original values. Then normalize by the white noise level,
and take inverse square root */
if( *status == SAI__OK ) {
for( i=0; i<nf_s; i++ ) {
if( smoothed_filter[i] != VAL__BADD ) {
smoothed_filter[i] = 1./sqrt(smoothed_filter[i]/W);
} else if( i < smooth ) {
/* Filter to 0 at low frequencies where we can't estimate median */
smoothed_filter[i] = 0;
} else {
/* Close to Nyquist we set the filter gain to 1 so that it doesn't
do anything */
smoothed_filter[i] = 1;
}
}
}
/* Copy radial filter into 2d filter */
if( *status == SAI__OK ) {
size_t d; /* radial distance (FFT pixels) in smoothed filter */
double model; /* the value of the model (smoothed filter) at d */
double x; /* x- spatial frequency */
double y; /* y- spatial frequency */
for( i=0; i<filt->fdims[0]; i++ ) {
x = FFT_INDEX_TO_FREQ(i,filt->rdims[0]) * filt->df[0];
for( j=0; j<filt->fdims[1]; j++ ) {
y = FFT_INDEX_TO_FREQ(j,filt->rdims[1]) * filt->df[1];
d = (size_t) round(sqrt(x*x + y*y)/df_s);
if( d < nf_s) {
model = smoothed_filter[d];
} else {
model = 1;
}
filt->real[i + j*filt->fdims[0]] *= model;
if( filt->imag ) filt->imag[i + j*filt->fdims[0]] *= model;
}
}
}
} else {
/* --- otherwise use the smooth fitted model ---------------------------- */
if( *status == SAI__OK ) {
double d;
double model;
double x;
double y;
/* We fit a model to the power spectrum, but we want to apply its
complement to the FFT of the data. So we normalize by the
white-noise level, and take the square root of the model before
writing its complement to the filter buffer */
A = sqrt(A / W);
B = B / 2.;
for( i=0; i<filt->fdims[0]; i++ ) {
x = FFT_INDEX_TO_FREQ(i,filt->rdims[0]) * filt->df[0];
for( j=0; j<filt->fdims[1]; j++ ) {
y = FFT_INDEX_TO_FREQ(j,filt->rdims[1]) * filt->df[1];
d = sqrt(x*x + y*y);
model = 1. + A * pow(d,B);
filt->real[i + j*filt->fdims[0]] /= model;
if( filt->imag ) filt->imag[i + j*filt->fdims[0]] /= model;
}
}
}
}
/* Clean up */
if( pspec ) smf_close_file( &pspec, status );
if( smoothed_filter ) smoothed_filter = astFree( smoothed_filter );
}
