void * smf_map_or_malloc (size_t nelem, smf_dtype type, int zero, int indf,
const char * comp, int *status ) {
void *pntr[3]; /* ndfMap pointers */
int nout = 0; /* number of elements mapped */
if (*status != SAI__OK) return NULL;
/* just malloc if we do not have a file */
if ( indf == NDF__NOID) {
if( zero ) {
return astCalloc( nelem, smf_dtype_sz(type, status) );
} else {
return astMalloc( nelem*smf_dtype_sz(type, status) );
}
}
ndfMap( indf, comp, smf_dtype_str(type, status),
(zero ? "WRITE/ZERO" : "WRITE"), pntr, &nout, status);
if (nelem != (size_t)nout && *status == SAI__OK) {
ndfUnmap( indf, comp, status );
*status = SAI__ERROR;
msgSetc( "COMP", comp );
msgSeti( "ORI", nelem );
msgSeti( "NOUT", nout );
errRep(" ", "Mapping ^COMP in NDF but size differs from that listed in smfData attributes (^ORI != ^NOUT)", status);
pntr[0] = NULL;
}
return pntr[0];
}
void smf_collapse_tseries( const smfData *indata, int nclip, const float clip[],
double snrlim, int flagconst, smf_dtype dtype,
smfData **outdata,
int *status ) {
/* Per type pointers */
double *avg_d = NULL;
double *var_d = NULL;
int *avg_i = NULL;
int *var_i = NULL;
dim_t dims[2]; /* dimensions of data array */
smfHead *hdr = NULL; /* copy of header */
AstKeyMap * history = NULL; /* history */
size_t nbperel; /* Number of bytes in dtype */
size_t nelem; /* number of elements in mean image */
void *pntr[] = { NULL, NULL }; /* pointers to data */
if (*status != SAI__OK) return;
/* see if we have a 2d input (likely reduced) */
if (indata->ndims == 2 ||
(indata->ndims == 3 && (indata->dims)[2] == 1) ) {
*outdata = NULL;
return;
}
/* Trap SMF__NULL */
if (dtype == SMF__NULL) dtype = indata->dtype;
/* Get some memory of the right type - data and variance */
dims[0] = (indata->dims)[0];
dims[1] = (indata->dims)[1];
nelem = dims[0] * dims[1];
nbperel = smf_dtype_sz(dtype, status);
pntr[0] = astMalloc( nelem*nbperel );
pntr[1] = astMalloc( nelem*nbperel );
/* Assign the pointers */
smf_select_pntr( pntr, dtype, &avg_d, &var_d, &avg_i, &var_i, status );
if (*status == SAI__OK) {
dim_t i,j;
/* get statistics for each bolometer */
for (i = 0; i < dims[0]; i++) {
for (j = 0; j < dims[1]; j++) {
double mean = VAL__BADD;
double stdev = VAL__BADD;
double variance = VAL__BADD;
dim_t index;
index = ( j * dims[0] ) + i;
smf_calc_stats( indata, "b", index, 0, 0, nclip, clip,
&mean, &stdev, status );
if (flagconst && stdev == 0.0) {
mean = VAL__BADD;
stdev = VAL__BADD;
}
if (snrlim > 0 && mean != VAL__BADD && stdev != VAL__BADD
&& stdev != 0.0) {
double snr;
snr = fabs(mean/stdev);
if (snr < snrlim) {
mean = VAL__BADD;
stdev = VAL__BADD;
}
}
if (stdev != VAL__BADD) {
variance = stdev * stdev;
}
switch (dtype) {
case SMF__DOUBLE:
avg_d[index] = mean;
var_d[index] = variance;
break;
case SMF__INTEGER:
if (isnan(mean) || mean == VAL__BADD ) {
avg_i[index] = VAL__BADI;
} else {
avg_i[index] = (int)mean;
}
if (isnan(stdev) || stdev == VAL__BADD) {
var_i[index] = VAL__BADI;
} else if ( variance > (double)VAL__MAXI ) {
/* overflow. Convert to BAD */
var_i[index] = VAL__BADI;
avg_i[index] = VAL__BADI;
} else {
var_i[index] = (int)variance;
}
break;
default:
*status = SAI__ERROR;
errRep( " ", "Should be impossible to get here", status );
goto L999;
}
}
}
}
L999:
/* now create a new smfData - we need to copy the header info */
hdr = smf_deepcopy_smfHead( indata->hdr, status );
if (indata->history) history = astCopy( indata->history );
*outdata = smf_construct_smfData( NULL, NULL, hdr, NULL, NULL,
dtype, pntr, NULL, SMF__QFAM_TSERIES,
NULL, 0, 1, dims, indata->lbnd, 2, 0, 0,
NULL, history, status );
/* must free the data if outdata is null */
if (*outdata == NULL) {
pntr[0] = astFree( pntr[0] );
pntr[1] = astFree( pntr[1] );
}
return;
}
void smf_write_smfFilter( ThrWorkForce *wf, const smfFilter *filt, const char *filename,
const Grp * igrp, size_t grpindex, int *status ) {
double *d = NULL; /* Data array pointer */
smfData *data=NULL; /* smfData for output */
size_t i; /* Loop counter */
size_t nsamp; /* Number of samples in the filter */
if( *status != SAI__OK ) return;
if( !filt ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfFilter supplied.", status );
return;
}
/* In case we change the data type of smfFilters in the future try
to catch that here */
if( sizeof(filt->real) != smf_dtype_sz(SMF__DOUBLE,status) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": warning -- smfFilter seems to have changed type!",
status );
return;
}
/* We will pack the data into an array that has an extra dimension to
store the real/imaginary parts of the filter just like an FFT */
data = smf_create_smfData( 0, status );
if( *status == SAI__OK ) {
data->dtype = SMF__DOUBLE;
data->ndims = filt->ndims+1;
nsamp=1;
for( i=0; i<filt->ndims; i++ ) {
data->dims[i] = filt->fdims[i];
nsamp *= filt->fdims[i];
}
data->dims[data->ndims-1] = 2;
/* Copy the real and imaginary parts into our new array consecutively */
data->pntr[0] = astCalloc( nsamp*2, smf_dtype_sz(SMF__DOUBLE,status) );
if( *status == SAI__OK ) {
d = data->pntr[0];
memcpy( d, filt->real, nsamp*sizeof(d) );
if( filt->isComplex ) {
memcpy( d + nsamp, filt->imag, nsamp*sizeof(d) );
}
}
}
/* Write out the file */
smf_write_smfData( wf, data, NULL, filename, igrp, grpindex, 0, MSG__NORM,
0, NULL, NULL, status );
if( data ) smf_close_file( wf, &data, status );
}
void smf_flat_mergedata( const smfArray * heatframes,
const double heatval[], smfData ** bolvald,
int * status ) {
double * bolval = NULL; /* Output bolometer data */
double * bolvalvar = NULL; /* Variance */
size_t i;
size_t nheat; /* Number of input frames */
size_t numbol; /* Number of bolometers */
if (bolvald) *bolvald = NULL;
if (*status != SAI__OK) return;
if (!bolvald) {
*status = SAI__ERROR;
errRep( "", "Must provide a pointer for returned smfData"
" (possible programming error)", status );
return;
}
nheat = heatframes->ndat;
numbol = (heatframes->sdata)[0]->dims[0] *
(heatframes->sdata)[0]->dims[1];
/* Create a smfData for powref and bolref */
smf_flat_malloc( nheat, (heatframes->sdata)[0], NULL, bolvald, status );
if (*status == SAI__OK) {
bolval = (*bolvald)->pntr[0];
bolvalvar = (*bolvald)->pntr[1];
/* copy the data from the individual frames to the new smfData */
for (i = 0; i<nheat; i++) {
double * datpntr = &(bolval[ i * numbol ]);
double * varpntr = &(bolvalvar[ i * numbol ] );
memcpy( datpntr, (heatframes->sdata)[i]->pntr[0],
numbol * smf_dtype_sz( (heatframes->sdata)[i]->dtype, status ) );
/* just in case we are missing variance */
if ( (heatframes->sdata)[i]->pntr[1] ) {
memcpy( varpntr, (heatframes->sdata)[i]->pntr[1],
numbol * smf_dtype_sz( (heatframes->sdata)[i]->dtype, status ) );
} else {
size_t j;
for (j = 0; j < numbol; j++) {
varpntr[j] = VAL__BADD;
}
}
}
}
/* Get the representative jcmtstate information */
if (*status == SAI__OK) {
JCMTState * state = astMalloc( nheat*sizeof(*state) );
for (i = 0; i < nheat; i++) {
smfData * frame = (heatframes->sdata)[i];
memcpy( &(state[i]), &(frame->hdr->allState)[0], sizeof(*state) );
}
(*bolvald)->hdr->allState = state;
}
/* Store the heater values in a smfDA */
if (*status == SAI__OK) {
smfDA * da = NULL;
double * dheatval = astMalloc( nheat*sizeof(*heatval) );
memcpy( dheatval, heatval, nheat * sizeof(*heatval) );
da = smf_construct_smfDA( NULL, NULL, NULL, NULL,
SMF__FLATMETH_NULL, 0, VAL__BADD,
dheatval, nheat, status );
(*bolvald)->da = da;
}
return;
}
void * smf_dataOrder_array( void * oldbuf, smf_dtype oldtype, smf_dtype newtype,
size_t ndata, size_t ntslice, size_t nbolo,
size_t tstr1, size_t bstr1,
size_t tstr2, size_t bstr2, int inPlace,
int freeOld, int * status ) {
size_t szold = 0; /* Size of old data type */
size_t sznew = 0; /* Size of new data type */
void * newbuf = NULL; /* Space to do the reordering */
void * retval = NULL; /* Return value with reordered buffer */
retval = oldbuf;
if (*status != SAI__OK) return retval;
if (!retval) return retval;
/* Can't do inPlace without realloc'ing if data types don't match...
so generate bad status for now */
if( (newtype != oldtype) && inPlace ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": inPlace not supported if newtype != oldtype",
status );
return retval;
}
/* For now the only data conversion that is supported is from SMF__INTEGER
to SMF__DOUBLE */
if( (oldtype!=newtype) &&
((newtype!=SMF__DOUBLE) || (oldtype!=SMF__INTEGER)) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": type conversion only supported from integer -> double",
status );
return retval;
}
/* Special case the inPlace variant with no reordering / typecasting */
if ( inPlace && tstr1 == tstr2 && bstr1 == bstr2 && oldtype == newtype ) {
return retval;
}
/* Size of data type */
sznew = smf_dtype_sz(newtype, status);
szold = smf_dtype_sz(oldtype, status);
/* Allocate buffer */
newbuf = astMalloc( ndata*sznew );
if( *status == SAI__OK ) {
/* if the input and output strides are the same, and the data types are
the same, we just memcpy */
if ( tstr1 == tstr2 && bstr1 == bstr2 && oldtype == newtype ) {
memcpy( newbuf, oldbuf, sznew*ndata );
} else {
size_t j;
size_t k;
/* Loop over all of the elements and re-order the data */
switch( oldtype ) {
case SMF__INTEGER:
if( newtype == SMF__DOUBLE ) {
/* Convert integer to double */
for( j=0; j<ntslice; j++ ) {
for( k=0; k<nbolo; k++ ) {
int val = ((int *)oldbuf)[j*tstr1+k*bstr1];
if( val != VAL__BADI ) {
((double *)newbuf)[j*tstr2+k*bstr2] = val;
} else {
((double *)newbuf)[j*tstr2+k*bstr2] = VAL__BADD;
}
}
}
} else {
/* integer to integer */
for( j=0; j<ntslice; j++ ) {
for( k=0; k<nbolo; k++ ) {
((int *)newbuf)[j*tstr2+k*bstr2] =
((int *)oldbuf)[j*tstr1+k*bstr1];
}
}
}
break;
case SMF__FLOAT:
for( j=0; j<ntslice; j++ ) {
for( k=0; k<nbolo; k++ ) {
((float *)newbuf)[j*tstr2+k*bstr2] =
((float *)oldbuf)[j*tstr1+k*bstr1];
}
}
break;
case SMF__DOUBLE:
for( j=0; j<ntslice; j++ ) {
for( k=0; k<nbolo; k++ ) {
((double *)newbuf)[j*tstr2+k*bstr2] =
((double *)oldbuf)[j*tstr1+k*bstr1];
}
}
break;
case SMF__USHORT:
for( j=0; j<ntslice; j++ ) {
for( k=0; k<nbolo; k++ ) {
((unsigned short *)newbuf)[j*tstr2+k*bstr2] =
((unsigned short *)oldbuf)[j*tstr1+k*bstr1];
}
}
break;
case SMF__UBYTE:
for( j=0; j<ntslice; j++ ) {
for( k=0; k<nbolo; k++ ) {
((unsigned char *)newbuf)[j*tstr2+k*bstr2] =
((unsigned char *)oldbuf)[j*tstr1+k*bstr1];
}
}
break;
default:
msgSetc("DTYPE",smf_dtype_str(oldtype, status));
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": Don't know how to handle ^DTYPE type.", status);
}
}
if( inPlace ) {
/* Copy newbuf to oldbuf */
memcpy( oldbuf, newbuf, ndata*sznew );
/* Free newbuf */
newbuf = astFree( newbuf );
retval = oldbuf;
} else {
if( freeOld ) {
/* Free oldbuf */
oldbuf = astFree( oldbuf );
}
/* Set pntr to newbuf */
retval = newbuf;
}
}
return retval;
}
void * smf_dataOrder_array( ThrWorkForce *wf, void * oldbuf, smf_dtype oldtype,
smf_dtype newtype, size_t ndata, size_t ntslice,
size_t nbolo, size_t tstr1, size_t bstr1,
size_t tstr2, size_t bstr2, int inPlace,
int freeOld, int * status ) {
size_t sznew = 0; /* Size of new data type */
void * newbuf = NULL; /* Space to do the reordering */
void * retval = NULL; /* Return value with reordered buffer */
SmfDataOrderArrayData *job_data = NULL;
SmfDataOrderArrayData *pdata;
int nw;
size_t step;
int iw;
retval = oldbuf;
if (*status != SAI__OK) return retval;
if (!retval) return retval;
/* Can't do inPlace without realloc'ing if data types don't match...
so generate bad status for now */
if( (newtype != oldtype) && inPlace ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": inPlace not supported if newtype != oldtype",
status );
return retval;
}
/* For now the only data conversion that is supported is from SMF__INTEGER
to SMF__DOUBLE */
if( (oldtype!=newtype) &&
((newtype!=SMF__DOUBLE) || (oldtype!=SMF__INTEGER)) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": type conversion only supported from integer -> double",
status );
return retval;
}
/* Special case the inPlace variant with no reordering / typecasting */
if ( inPlace && tstr1 == tstr2 && bstr1 == bstr2 && oldtype == newtype ) {
return retval;
}
/* Size of data type */
sznew = smf_dtype_sz(newtype, status);
/* Allocate buffer */
newbuf = astMalloc( ndata*sznew );
if( *status == SAI__OK ) {
/* if the input and output strides are the same, and the data types are
the same, we just memcpy */
if ( tstr1 == tstr2 && bstr1 == bstr2 && oldtype == newtype ) {
memcpy( newbuf, oldbuf, sznew*ndata );
} else {
/* We currently ignore any supplied WorkForce, since using multiple
threads slows down the re-ordered rather than speeding it up - I
presume because of contention issues. I've tried ensuring that
the output array is accessed sequenctially but that does not seem
to improve anything. Leave the mult-threaded infrastructure
here, in case a better solution is found. */
wf = NULL;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many time slices to process in each worker thread. */
step = ntslice/nw;
if( ntslice == 0 ) step = 1;
/* Allocate job data for threads, and store common values. Ensure that
the last thread picks up any left-over time slices. Store the
info required by the wrker threads, then submit jobs to the work
force. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->itime1 = iw*step;
if( iw < nw - 1 ) {
pdata->itime2 = pdata->itime1 + step - 1;
} else {
pdata->itime2 = ntslice - 1 ;
}
pdata->nbolo = nbolo;
pdata->bstr1 = bstr1;
pdata->tstr1 = tstr1;
pdata->bstr2 = bstr2;
pdata->tstr2 = tstr2;
pdata->newtype = newtype;
pdata->oldtype = oldtype;
pdata->newbuf = newbuf;
pdata->oldbuf = oldbuf;
thrAddJob( wf, 0, pdata, smf1_dataOrder_array, 0, NULL, status );
}
/* Wait for the jobs to complete. */
thrWait( wf, status );
}
job_data = astFree( job_data );
}
if( inPlace ) {
/* Copy newbuf to oldbuf */
memcpy( oldbuf, newbuf, ndata*sznew );
/* Free newbuf */
newbuf = astFree( newbuf );
retval = oldbuf;
} else {
if( freeOld ) {
/* Free oldbuf */
oldbuf = astFree( oldbuf );
}
/* Set pntr to newbuf */
retval = newbuf;
}
}
return retval;
}
void smurf_sc2threadtest( int *status ) {
/* Local Variables */
smfArray **res=NULL; /* array of smfArrays of test data */
smfData *data=NULL; /* Pointer to SCUBA2 data struct */
dim_t datalen; /* Number of data points */
smfFilter *filt=NULL; /* Frequency domain filter */
size_t i; /* Loop counter */
size_t j; /* Loop counter */
smfTimeChunkData *job_data=NULL; /* Array of pointers for job data */
size_t joblen; /* Number of chunks per job */
size_t k; /* Loop counter */
size_t nchunks; /* Number of chunks */
size_t nsub; /* Number of subarrays */
int nthread; /* Number of threads */
smfTimeChunkData *pdata=NULL; /* Pointer to data for single job */
int temp; /* Temporary integer */
size_t tsteps; /* How many time steps in chunk */
struct timeval tv1, tv2; /* Timers */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
double *dat=NULL;
dim_t nbolo;
dim_t ntslice;
dim_t ndata;
size_t bstride;
size_t tstride;
dim_t offset;
if (*status != SAI__OK) return;
/* Get input parameters */
parGdr0i( "NTHREAD", 1, 1, NUM__MAXI, 1, &nthread, status );
parGdr0i( "TSTEPS", 6000, 0, NUM__MAXI, 1, &temp, status );
tsteps = (size_t) temp;
parGdr0i( "NCHUNKS", 1, 1, NUM__MAXI, 1, &temp, status );
nchunks = (size_t) temp;
parGdr0i( "NSUB", 1, 1, 4, 1, &temp, status );
nsub = (size_t) temp;
msgSeti("N",nthread);
msgOut( "", TASK_NAME ": Running test with ^N threads", status );
/*** TIMER ***/
smf_timerinit( &tv1, &tv2, status );
/* Create some fake test data in the form of an array of smfArrays */
msgSeti("T",tsteps);
msgSeti("C",nchunks);
msgSeti("NS",nsub);
msgOut( "", TASK_NAME
": Creating ^NS subarrays of data with ^C chunks * ^T samples",
status );
res = astCalloc( nchunks, sizeof(*res) );
for( k=0; (*status==SAI__OK)&&(k<nchunks); k++ ) {
res[k] = smf_create_smfArray( status );
for( i=0; (*status==SAI__OK)&&(i<nsub); i++ ) {
/* Create individual smfDatas and add to array */
data = smf_create_smfData( SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS, status );
if( *status==SAI__OK ) {
data->dtype=SMF__DOUBLE;
data->ndims=3;
data->dims[0]=40;
data->dims[1]=32;
data->dims[2]=(dim_t) tsteps;
datalen=1;
data->isFFT=-1;
for( j=0; j<data->ndims; j++ ) datalen *= data->dims[j];
data->hdr->steptime = 0.005;
data->pntr[0] = astCalloc( datalen, smf_dtype_sz(data->dtype,status) );
data->qual = astCalloc( datalen, sizeof(*data->qual) );
}
smf_addto_smfArray( res[k], data, status );
}
}
/*** TIMER ***/
msgOutf( "", "** %f seconds generating data", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut( "", TASK_NAME
": Starting test 1 __parallel time: dataOrder__", status );
/* Create a pool of threads. */
wf = thrGetWorkforce( nthread, status );
/* Work out number of chunks per thread */
joblen = nchunks/nthread;
if( joblen == 0 ) joblen = 1; /* At least one chunk per thread */
/* The first test will process separate time chunks of data in
parallel, re-ordering each to bolo-ordered format. All subarrays
and an integer number of input file chunks all go into a single
thread. Start by allocating and initializing a number of
smfTimeChunkData's that hold the information required for each
thread */
job_data = astCalloc( nthread, sizeof(*job_data) );
for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) {
pdata = job_data + i;
pdata->type = 0; /* Start with a data re-order */
pdata->data = res; /* Pointer to main data array */
pdata->chunk1 = i*joblen; /* Index of first chunk for job */
pdata->nchunks = nchunks; /* Total number of time chunks in data */
pdata->ijob = -1; /* Flag job as available to do work */
/* The last thread has to pick up the remainder of chunks */
if( i==(size_t)(nthread-1) ) pdata->chunk2=nchunks-1;
else pdata->chunk2 = (i+1)*joblen-1; /* Index of last chunk for job */
/* Ensure a valid chunk range, or set to a length that we know to ignore */
if( pdata->chunk1 >= nchunks ) {
pdata->chunk1 = nchunks;
pdata->chunk2 = nchunks;
} else if( pdata->chunk2 >= nchunks ) {
pdata->chunk2 = nchunks-1;
}
if( pdata->chunk1 >= nchunks ) {
/* Nothing for this thread to do */
msgSeti( "W", i+1);
msgOutif( MSG__DEBUG, "",
"-- parallel time: skipping thread ^W, nothing to do",
status);
} else {
/* Since we know there is one job_data per thread, just submit jobs
immediately */
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Annul the bad status that we set in smfParallelTime */
if( *status == SMF__INSMP ) {
errAnnul( status );
msgOut( "", " *** Annulled SMF__INSMP set in smfParallelTime *** ",
status );
} else {
msgOut( "", " *** Flushing good status *** ", status );
errFlush( status );
}
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
/* The second test will boxcar smooth bolometers from time chunks in
parallel */
msgOut( "", TASK_NAME
": Starting test 2 __parallel time: boxcar smooth__", status );
for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) {
pdata = job_data + i;
pdata->type = 1; /* Boxcar smooth */
if( pdata->chunk1 >= nchunks ) {
/* Nothing for this thread to do */
msgSeti( "W", i+1);
msgOutif( MSG__DEBUG, "",
"-- parallel time: skipping thread ^W, nothing to do",
status);
} else {
/* Since we know there is one job_data per thread, just submit jobs
immediately */
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut( "", TASK_NAME
": *** Next 2 tests will be done twice due to FFTW planning *****",
status );
for( k=0; k<2; k++ ) {
/* The third test will FFT filter bolometers from time chunks in
parallel */
msgOut( "", TASK_NAME
": Starting test 3 __parallel time: FFT filter__", status );
for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) {
pdata = job_data + i;
pdata->type = 2; /* FFT filter */
if( pdata->chunk1 >= nchunks ) {
/* Nothing for this thread to do */
msgSeti( "W", i+1);
msgOutif( MSG__DEBUG, "",
"-- parallel time: skipping thread ^W, nothing to do",
status);
} else {
/* Since we know there is one job_data per thread, just submit jobs
immediately */
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut( "", TASK_NAME
": Starting test 4 __FFTW filter using internal threading__",
status );
for( i=0; (*status==SAI__OK)&&(i<nchunks); i++ ) {
filt = smf_create_smfFilter( res[i]->sdata[0], status );
smf_filter_ident( filt, 1, status );
for( j=0; (*status==SAI__OK)&&(j<nsub); j++ ) {
msgOutiff( MSG__DEBUG, "", " filter chunk %zu/%zu, bolo %zu/%zu",
status, i+1, nchunks, j+1, nsub );
smf_filter_execute( wf, res[i]->sdata[j], filt, 0, 0, status );
}
if( filt ) filt = smf_free_smfFilter( filt, status );
}
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
}
msgOut( "", TASK_NAME
": **************************************************************",
status );
/* Series of short single-thread array index tests */
data = res[0]->sdata[0];
dat = data->pntr[0];
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride,
&tstride, status );
msgOut("","Array index test #1: two multiplies in inner loop",status);
smf_timerinit( &tv1, &tv2, status );
for( i=0; i<nbolo; i++ ) {
for( j=0; j<ntslice; j++ ) {
dat[i*bstride + j*tstride] += 5;
}
}
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut("","Array index test #2: only index increments",status);
smf_timerinit( &tv1, &tv2, status );
for( i=0; i<nbolo*bstride; i+=bstride ) {
for( j=i; j<(i+ntslice*tstride); j+=tstride ) {
dat[j] += 5;
}
}
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut("","Array index test #3: one multiply in outer loop",status);
smf_timerinit( &tv1, &tv2, status );
offset = 0;
for( i=0; i<nbolo; i++ ) {
offset = i*bstride;
for( j=0; j<ntslice; j++ ) {
dat[offset] += 5;
offset += tstride;
}
}
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
/* Clean up */
if( res ) {
for( i=0; i<nchunks; i++ ) {
if( res[i] ) {
smf_close_related( &res[i], status );
}
}
res = astFree( res );
}
job_data = astFree( job_data );
/* Ensure that FFTW doesn't have any used memory kicking around */
fftw_cleanup();
}
void smf_checkmem_dimm( dim_t maxlen, inst_t instrument, int nrelated,
smf_modeltype *modeltyps, dim_t nmodels, dim_t msize,
AstKeyMap *keymap, size_t available, dim_t maxfilelen,
size_t *necessary, int *status ) {
/* Local Variables */
int dofft=0; /* flag if we need temp space for FFTs */
dim_t i; /* Loop counter */
dim_t gain_box; /* Length of blocks for GAI/COM model */
AstKeyMap *kmap=NULL; /* Local keymap */
dim_t nblock; /* Number of blocks for GAI/COM model */
size_t ncol; /* Number of columns */
size_t ndet; /* Number of detectors each time step */
size_t ndks; /* dksquid samples in a subarray, ncol*maxlen */
size_t nrow; /* Number of rows */
size_t nsamp; /* bolo samples in a subarray, ndet*maxlen */
const char *tempstr=NULL; /* Temporary pointer to static char buffer */
size_t total = 0; /* Total bytes required */
/* Main routine */
if (*status != SAI__OK) return;
/* Check inputs */
if( maxlen < 1 ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": maxlen cannot be < 1", status);
return;
}
if( (nrelated < 1) || (nrelated > SMF__MXSMF) ) {
msgSeti("NREL",nrelated);
msgSeti("MAXREL",SMF__MXSMF);
*status = SAI__ERROR;
errRep("", FUNC_NAME ": nrelated, ^NREL, must be in the range [1,^MAXREL]",
status);
return;
}
if( modeltyps ) {
if( nmodels < 1 ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": modeltyps specified, mmodels cannot be < 1",
status);
return;
}
}
if( *status == SAI__OK ) {
/* Work out the data dimensions */
switch( instrument ) {
case INST__SCUBA2:
/* Kludgey, but at least we check SC2STORE__COL_INDEX so this will help
us catch possible future problems if order is changed */
if( SC2STORE__COL_INDEX ) {
ncol = 32;
nrow = 40;
} else {
ncol = 40;
nrow = 32;
}
ndet = ncol*nrow;
break;
default:
*status = SAI__ERROR;
errRep("", FUNC_NAME ": Invalid instrument given.", status);
}
}
/* Number of samples in a full data cube for one subarray */
nsamp = ndet*maxlen;
/* Check to see if we need to do filtering as part of
pre-processing. If so, we need an extra nsamp-sized buffer to
store the FFT. */
smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
&dofft, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
status );
/* Calculate memory usage of static model components: -------------------- */
if( *status == SAI__OK ) {
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated; /* RES */
total += nsamp*smf_dtype_sz(SMF__INTEGER,status)*nrelated; /* LUT */
total += nsamp*smf_dtype_sz(SMF__QUALTYPE,status)*nrelated; /* QUA */
}
/* Add on memory usage for the JCMTState (one per time slice per array) */
if( *status == SAI__OK ) {
total += maxlen*sizeof(JCMTState)*nrelated;
}
/* Add on space for dark squids */
ndks = ncol*maxlen;
total += ndks*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
/* Add on space for fakemap */
astMapGet0C( keymap, "FAKEMAP", &tempstr );
if( tempstr ) {
total += msize*sizeof(double);
}
/* Apply fudge factor */
total *= CHECKMEM_FUDGE;
/* Calculate memory usage of dynamic model components: ------------------- */
if( *status == SAI__OK ) {
/* Most of these will have data arrays associated with each
subarray (hence the multiplication by nrelated). An exception
is SMF__COM for which a single-common mode is used across all
subarrays. */
if( modeltyps ) {
for( i=0; i<nmodels; i++ ) {
switch( modeltyps[i] ) {
case SMF__NOI:
/* SMF__NOI also estimates the noise in each detector from the
power spectra, requiring a temporary buffer to store an FFT.
Currently we just store one variance per detector */
dofft = 1;
total += ndet*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__COM:
CHECK_MASK("COM")
total += maxlen*smf_dtype_sz(SMF__DOUBLE,status);
break;
case SMF__EXT:
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__DKS:
total += (maxlen + nrow*3)*ncol*smf_dtype_sz(SMF__DOUBLE,status) *
nrelated;
break;
case SMF__GAI:
/* Every COM.GAIN_BOX samples there are 3 planes of data
corresponding to each bolometer in the subarray. The
conversion of COM.GAIN_BOX from seconds to samples within
smf_get_nsamp assumes a sample rate of 200 Hz. Later
downsampling may result in a lower sample rate, but at least
we are erring on the conservative side by assuming 200 Hz. */
if( astMapGet0A( keymap, "COM", &kmap ) ) {
smf_get_nsamp( kmap, "GAIN_BOX", NULL, &gain_box, status );
nblock = maxlen/gain_box;
if( nblock == 0 ) nblock = 1;
total += nblock*3*nrow*ncol*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
kmap = astAnnul( kmap );
}
break;
case SMF__FLT:
/* Presently the filter temporarily transforms the entire
data cube into a second array. We therefore need to
ensure enough memory to temporarily store the data cube
twice. */
CHECK_MASK("FLT")
dofft = 1;
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__PLN:
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__SMO:
total += nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__TMP:
/* An externally supplied template. The model just stores the gain,
offset and correlation coefficient for each bolometer, similar
to SMF__GAI except with only one chunk considered */
total += 3*nrow*ncol*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__TWO:
/* two common-mode time series and coefficients for all the
detectors */
total += 2*(ndet+maxlen)*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
break;
case SMF__AST:
/* Mostly accounted for as static memory usage above, but add space
for mask if required */
CHECK_MASK("AST")
break;
default:
*status = SAI__ERROR;
errRep("", FUNC_NAME ": Invalid smf_modeltype given.", status);
}
/* Exit on bad status */
if( *status != SAI__OK ) {
i = nmodels;
}
}
}
/* Calculate temporary space here -------------------------------------- */
if( *status == SAI__OK ) {
size_t temp=0; /* current temp memory required */
size_t maxtemp=0; /* max temp memory required */
/* Some temp space required when we initially read in the
data. Normally smf_concat_smfGroup will
smf_open_and_flatfield one file at a time before copying into
the concatenated data array. If there are N time slices in a
file, smf_open_and_flatfield will usually require space both
for the raw, and double-precision flatfielded data
simultaneously. This is an upper limit, since if previously
flatfielded data are provided no extra space for the raw
data will be required. We use the supplied maxfilelen to figure
out this maximum temporary buffer size. Remember to account
for bolo data, dark squids, and JCMTState. Also, the way
smf_iteratemap is currently written, a concatenated memory-mapped
pointing LUT is initially created, and then copied to a new
malloc'd array, so we need extra temp space for the full LUT. */
temp = (ndet + ncol)*( smf_dtype_sz(SMF__INTEGER,status)+
smf_dtype_sz(SMF__DOUBLE,status) )*maxfilelen +
2*maxfilelen*sizeof(JCMTState);
temp += nsamp*smf_dtype_sz(SMF__INTEGER,status)*nrelated;
if( temp > maxtemp ) maxtemp = temp;
/* If we are doing FFTs */
if( dofft ) {
temp = nsamp*smf_dtype_sz(SMF__DOUBLE,status)*nrelated;
if( temp > maxtemp ) maxtemp = temp;
}
/* Data cleaning... seems to be a lot. Just putting in a rough
estimate of 3 extra full sized arrays for the standard
options. */
temp = 3.*nsamp*smf_dtype_sz(SMF__DOUBLE,status);
if( temp > maxtemp ) maxtemp = temp;
/* Add on the maximum chunk of temporary memory that is required */
total += maxtemp;
}
/* Set bad status if too big */
if( (*status == SAI__OK) && (total > available) ) {
*status = SMF__NOMEM;
msgSeti("REQ",total/SMF__MIB);
msgSeti("AVAIL",available/SMF__MIB);
errRep("", FUNC_NAME
": Requested memory ^REQ MiB for map exceeds available ^AVAIL MiB",
status);
}
/* Return the required space */
if( necessary ) {
*necessary = total;
}
}
}
smf_qual_t * smf_qual_unmap( ThrWorkForce *wf, int indf, smf_qfam_t family,
smf_qual_t * qual, smf_qual_t mask, int * status ) {
int canwrite = 0; /* can we write to the file? */
size_t nqbits = 0; /* Number of quality bits in this family */
SmfQualUnmapData *job_data = NULL;
SmfQualUnmapData *pdata;
int nw;
size_t step;
int iw;
if (*status != SAI__OK) goto CLEANUP;
/* do nothing if there is no quality */
if (!qual) return NULL;
/* if we do not have an NDF identifier we just free the memory */
if (indf == NDF__NOID) goto CLEANUP;
/* See if we have WRITE access to the file */
ndfIsacc( indf, "WRITE", &canwrite, status );
/* if we have WRITE access and the data were not mapped we have
to copy to the file. Also check we have a non-NULL input pointer.
If the data were mapped we still have to make sure the quality names
are stored. */
if ( canwrite && qual ) {
int highbit = -1; /* highest bit used */
size_t i;
int itemp;
int lowbit = -1; /* Lowest bit used */
size_t nout;
int nqual = 0;
void *qpntr[1];
size_t qcount[SMF__NQBITS]; /* statically allocate the largest array */
IRQLocs *qlocs;
unsigned char * qmap;
int there;
ndfMsg( "FILE", indf );
msgOutif( MSG__DEBUG, "", "Finalising quality for file ^FILE", status);
if (family == SMF__QFAM_TCOMP ) {
/* note that TCOMP is not an allowed quality because SMURF should not be
using it anywhere in a permanent way. */
*status = SAI__ERROR;
ndfMsg( "NDF", indf );
errRepf( "", "Unsupported quality family '%s' for quality unmapping of "
"file ^NDF", status, smf_qfamily_str(family,status) );
goto CLEANUP;
} else if (family == SMF__QFAM_NULL) {
/* In this case we have to assume that we just cast the quality
to UBYTE and copy it without changing anything or naming the
entries. Use a simple type conversion. */
ndfMap( indf, "QUALITY", "_UBYTE", "WRITE", &qpntr[0], &itemp, status );
qmap = qpntr[0];
nout = itemp;
for (i = 0; i<nout; i++) {
qmap[i] = qual[i];
}
ndfUnmap( indf, "QUALITY", status );
/* Turn on all quality */
ndfSbb( 255, indf, status );
/* we are finished so jump to tidy up */
goto CLEANUP;
}
/* work out how many quality items are in this family */
nqbits = smf_qfamily_count( family, status );
/* initialize qcount */
for (i=0; i<SMF__NQBITS; i++) {
qcount[i] = 0;
}
/* how many pixels in NDF (assumed to be number in quality) */
ndfSize( indf, &itemp, status );
nout = itemp;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many elements to process in each worker thread. */
step = nout/nw;
if( step == 0 ) step = 1;
/* Allocate job data for threads, and store common values. Ensure that the
last thread picks up any left-over elements. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->i1 = iw*step;
if( iw < nw - 1 ) {
pdata->i2 = pdata->i1 + step - 1;
} else {
pdata->i2 = nout - 1 ;
}
pdata->nqbits = nqbits;
pdata->qual = qual;
pdata->nout = nout;
}
}
/* Work out which bits are actually used */
if (*status == SAI__OK) {
size_t k;
/* now we try to be a bit clever. It may be a mistake since we have to
do multiple passes through "qual". First determine how many quality
bits are actually set. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 1;
thrAddJob( wf, 0, pdata, smf1_qual_unmap, 0, NULL, status );
}
thrWait( wf, status );
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
for( k=0; k<nqbits; k++ ) {
qcount[k] += pdata->qcount[k];
}
}
/* Reset the counts to zero for any bits that are not required
(i.e. are not set in "mask"). */
for( k=0; k<nqbits; k++ ) {
if( ! (mask & (1<<k)) ) qcount[k] = 0;
}
/* see how many we got */
for (k=0; k<nqbits; k++) {
if ( qcount[k] ) {
nqual++;
highbit = k;
if (lowbit < 0) lowbit = k;
}
}
}
/* for IRQ we need to ensure the SMURF extension exists so open and annul it if it is missing.
We are completely rewriting any IRQ information so we have to delete any previously existing
IRQ extension. */
irqDelet( indf, status );
ndfXstat( indf, SMURF__EXTNAME, &there, status );
if (!there) {
HDSLoc * smurfloc = NULL;
/* Create SMURF extension if it does not already exist */
ndfXnew( indf, SMURF__EXTNAME, SMURF__EXTTYPE, 0, NULL, &smurfloc, status );
if (smurfloc) datAnnul( &smurfloc, status );
}
irqNew( indf, SMURF__EXTNAME, &qlocs, status );
/* malloced so we need to map and copy over the values. IRQ
names need to be set BEFORE we copy. */
/* Map the quality component with WRITE access */
ndfMap( indf, "QUALITY", "_UBYTE", "WRITE", &qpntr[0], &itemp, status );
qmap = qpntr[0];
/* we assume the number of elements in "qual" is the same as in "qmap" */
if (*status == SAI__OK) {
size_t k;
/* if we only have 8 or fewer bits active we can just compress
by mapping them to the lower 8 bits. This will work if we also
set the IRQ quality names in the NDF. */
if (nqual == 0 ) {
/* easy */
memset( qmap, 0, nout * smf_dtype_sz( SMF__UBYTE, status ) );
} else if ( nqual <= 8 ) {
size_t curbit = 0;
/* and the quality names. Start at lowbit and go to highbit
knowing that we have shifted them down so that lowbit in qual
is bit 0 in NDF. */
for (k=lowbit; k<=(size_t)highbit; k++) {
if (qcount[k]) {
int fixed = 0; /* is bit fixed? */
const char * qdesc = NULL; /* Description of quality */
const char * qstr = NULL; /* Quality string identifier */
curbit++;
qstr = smf_qual_str( family, 1, k, &qdesc, status );
irqAddqn( qlocs, qstr, 0, qdesc, status );
irqFxbit( qlocs, qstr, curbit, &fixed, status );
}
}
/* shift them down */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 2;
pdata->qmap = qmap;
pdata->highbit = highbit;
pdata->lowbit = lowbit;
for( k=0; k<nqbits; k++ ) {
pdata->qcount[k] = qcount[k];
}
thrAddJob( wf, 0, pdata, smf1_qual_unmap, 0, NULL, status );
}
thrWait( wf, status );
} else {
size_t curbit = 0;
/* Quality names are now needed and we have to write them
all out because we have not compressed the bits in the
output quality array we've only compressed the input.
To limit the number of active bits we'd have to copy the
compressed bits to the output and then set the quality
names but IRQ does not let you do that so you would need
to run through the entire array first counting which bits
were used. */
for (k=0; k<SMF__NQBITS_TCOMP; k++) {
int fixed = 0;
const char * qdesc = NULL; /* Description of quality */
const char * qstr = NULL; /* Quality string identifier */
qstr = smf_qual_str( SMF__QFAM_TCOMP, 1, k, &qdesc, status );
/* Set the quality name */
irqAddqn( qlocs, qstr, 0, qdesc, status );
curbit++;
irqFxbit( qlocs, qstr, curbit, &fixed, status );
}
/* compress them */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 3;
pdata->qmap = qmap;
thrAddJob( wf, 0, pdata, smf1_qual_unmap, 0, NULL, status );
}
thrWait( wf, status );
}
}
/* Unmap quality */
ndfUnmap( indf, "QUALITY", status );
/* Set the badbits mask to enable all quality by default.
Do not do this for MAP quality at the moment. */
if (family != SMF__QFAM_MAP) ndfSbb( 255, indf, status );
/* release IRQ resources */
irqRlse( &qlocs, status );
}
CLEANUP:
/* Tidy up */
qual = astFree( qual );
job_data = astFree( job_data );
return NULL;
}
smfData *smf_fft_avpspec( const smfData *pspec, smf_qual_t *quality,
size_t qstride, smf_qual_t mask, double *weights,
int *status ) {
dim_t fdims[2]; /* Lengths of frequency-space axes */
size_t i; /* Loop counter */
double *idptr=NULL; /* Pointer to input data */
double mean; /* Mean value at time slice */
dim_t nbolo=0; /* Number of detectors */
size_t ngood; /* Number of good samples */
size_t ndata; /* Total number of data points */
dim_t ntslice=0; /* Number of time slices */
dim_t nf=0; /* Number of frequencies in FFT */
double *odptr=NULL; /* Pointer to output data */
double *ovptr=NULL; /* Pointer to output variance */
dim_t rdims[2]; /* Lengths of real-space axes */
smfData *retdata=NULL; /* Returned power spectrum */
double sigma; /* RMS value at time slice */
if (*status != SAI__OK) return NULL;
/* Check for NULL pointer */
if( pspec == NULL ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData pointer is NULL", status );
return NULL;
}
/* Check that we have frequency-domain input */
if( !smf_isfft( pspec, rdims, &nbolo, fdims, NULL, NULL, status ) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Input data are not FFT!", status );
return NULL;
}
ntslice = rdims[0];
nf = fdims[0];
/* Check that we don't have a 1-d input power spectrum */
if( pspec->ndims != 4 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Input data are not 4-dimensional!", status );
return NULL;
}
idptr = pspec->pntr[0];
if( !idptr ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null input data", status );
return NULL;
}
/* Assume qstride = bolo stride of main array if not specified */
if( !qstride ) {
qstride = nf;
}
/* Create a new smfData, copying over everything except for the bolo
data itself */
retdata = smf_deepcopy_smfData( pspec, 0, SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY |
SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS, 0, 0, status );
if( *status == SAI__OK ) {
/* Allocate space for the averaged power spectrum */
retdata->ndims = 4;
retdata->dims[0] = nf;
retdata->dims[1] = 1;
retdata->dims[2] = 1;
retdata->dims[3] = 2;
retdata->dtype = SMF__DOUBLE;
ndata=1;
for( i=0; i<retdata->ndims; i++ ) {
ndata *= retdata->dims[i];
}
/* Allocate space for DATA and VARIANCE components */
retdata->pntr[0] = astCalloc( ndata, smf_dtype_sz(retdata->dtype,status) );
retdata->pntr[1] = astCalloc( ndata, smf_dtype_sz(retdata->dtype,status) );
/* Pointers to output data */
odptr = retdata->pntr[0];
ovptr = retdata->pntr[1];
/* Call the returned data tordered even though it is irrelevant */
retdata->isTordered=0;
/* Since we assumed we're in power/polar form just need to calculate
average and scatter of the amplitude coefficients. The phase
coefficients are left at their initialized values of 0. */
for( i=0; i<nf; i++ ) {
/* The bolometer stride is nf */
smf_weightstats1D( idptr+i, nf, nbolo, quality, qstride, mask,
weights, 1, &mean, &sigma, &ngood, status );
if( *status == SMF__INSMP ) {
/* If not enough samples just annul and set bad value */
errAnnul( status );
odptr[i] = VAL__BADD;
ovptr[i] = VAL__BADD;
} else {
odptr[i] = mean;
ovptr[i] = sigma*sigma;
}
}
}
return retdata;
}
smf_qual_t * smf_qual_unmap( int indf, smf_qfam_t family, smf_qual_t * qual, int * status ) {
int canwrite = 0; /* can we write to the file? */
size_t nqbits = 0; /* Number of quality bits in this family */
if (*status != SAI__OK) goto CLEANUP;
/* do nothing if there is no quality */
if (!qual) return NULL;
/* if we do not have an NDF identifier we just free the memory */
if (indf == NDF__NOID) goto CLEANUP;
/* See if we have WRITE access to the file */
ndfIsacc( indf, "WRITE", &canwrite, status );
/* if we have WRITE access and the data were not mapped we have
to copy to the file. Also check we have a non-NULL input pointer.
If the data were mapped we still have to make sure the quality names
are stored. */
if ( canwrite && qual ) {
int highbit = -1; /* highest bit used */
size_t i;
int itemp;
int lowbit = -1; /* Lowest bit used */
size_t nout;
int nqual = 0;
void *qpntr[1];
size_t qcount[SMF__NQBITS]; /* statically allocate the largest array */
IRQLocs *qlocs;
unsigned char * qmap;
int there;
ndfMsg( "FILE", indf );
msgOutif( MSG__DEBUG, "", "Finalising quality for file ^FILE", status);
if (family == SMF__QFAM_TCOMP ) {
/* note that TCOMP is not an allowed quality because SMURF should not be
using it anywhere in a permanent way. */
*status = SAI__ERROR;
ndfMsg( "NDF", indf );
errRepf( "", "Unsupported quality family '%s' for quality unmapping of "
"file ^NDF", status, smf_qfamily_str(family,status) );
goto CLEANUP;
} else if (family == SMF__QFAM_NULL) {
/* In this case we have to assume that we just cast the quality
to UBYTE and copy it without changing anything or naming the
entries. Use a simple type conversion. */
ndfMap( indf, "QUALITY", "_UBYTE", "WRITE", &qpntr[0], &itemp, status );
qmap = qpntr[0];
nout = itemp;
for (i = 0; i<nout; i++) {
qmap[i] = qual[i];
}
ndfUnmap( indf, "QUALITY", status );
/* Turn on all quality */
ndfSbb( 255, indf, status );
/* we are finished so jump to tidy up */
goto CLEANUP;
}
/* work out how many quality items are in this family */
nqbits = smf_qfamily_count( family, status );
/* initialize qcount */
for (i=0; i<SMF__NQBITS; i++) {
qcount[i] = 0;
}
/* how many pixels in NDF (assumed to be number in quality) */
ndfSize( indf, &itemp, status );
nout = itemp;
/* Work out which bits are actually used */
if (*status == SAI__OK) {
size_t k;
/* now we try to be a bit clever. It may be a mistake since we have to
do multiple passes through "qual". First determine how many quality
bits are actually set. */
for (i = 0; i<nout; i++) {
/* try all the bits */
for( k=0; k<nqbits; k++ ) {
if( qual[i] & BIT_TO_VAL(k) ) {
qcount[k]++;
}
}
}
/* see how many we got */
for (k=0; k<nqbits; k++) {
if ( qcount[k] ) {
nqual++;
highbit = k;
if (lowbit < 0) lowbit = k;
}
}
}
/* for IRQ we need to ensure the SMURF extension exists so open and annul it if it is missing.
We are completely rewriting any IRQ information so we have to delete any previously existing
IRQ extension. */
irqDelet( indf, status );
ndfXstat( indf, SMURF__EXTNAME, &there, status );
if (!there) {
HDSLoc * smurfloc = NULL;
/* Create SMURF extension if it does not already exist */
ndfXnew( indf, SMURF__EXTNAME, SMURF__EXTTYPE, 0, NULL, &smurfloc, status );
if (smurfloc) datAnnul( &smurfloc, status );
}
irqNew( indf, SMURF__EXTNAME, &qlocs, status );
/* malloced so we need to map and copy over the values. IRQ
names need to be set BEFORE we copy. */
/* Map the quality component with WRITE access */
ndfMap( indf, "QUALITY", "_UBYTE", "WRITE", &qpntr[0], &itemp, status );
qmap = qpntr[0];
/* we assume the number of elements in "qual" is the same as in "qmap" */
if (*status == SAI__OK) {
size_t k;
/* if we only have 8 or fewer bits active we can just compress
by mapping them to the lower 8 bits. This will work if we also
set the IRQ quality names in the NDF. */
if (nqual == 0 ) {
/* easy */
memset( qmap, 0, nout * smf_dtype_sz( SMF__UBYTE, status ) );
} else if ( nqual <= 8 ) {
size_t curbit = 0;
/* and the quality names. Start at lowbit and go to highbit
knowing that we have shifted them down so that lowbit in qual
is bit 0 in NDF. */
for (k=lowbit; k<=(size_t)highbit; k++) {
if (qcount[k]) {
int fixed = 0; /* is bit fixed? */
const char * qdesc = NULL; /* Description of quality */
const char * qstr = NULL; /* Quality string identifier */
curbit++;
qstr = smf_qual_str( family, 1, k, &qdesc, status );
irqAddqn( qlocs, qstr, 0, qdesc, status );
irqFxbit( qlocs, qstr, curbit, &fixed, status );
}
}
/* shift them down */
for (i=0; i<nout; i++) {
curbit = 0;
qmap[i] = 0;
for (k=lowbit; k<=(size_t)highbit; k++) {
/* was this bit used by this data array? */
if (qcount[k]) {
/* was the bit set for this location? */
if ( qual[i]&BIT_TO_VAL(k)) {
qmap[i] |= BIT_TO_VAL(curbit);
}
curbit++;
}
}
}
} else {
size_t curbit = 0;
/* Quality names are now needed and we have to write them
all out because we have not compressed the bits in the
output quality array we've only compressed the input.
To limit the number of active bits we'd have to copy the
compressed bits to the output and then set the quality
names but IRQ does not let you do that so you would need
to run through the entire array first counting which bits
were used. */
for (k=0; k<SMF__NQBITS_TCOMP; k++) {
int fixed = 0;
const char * qdesc = NULL; /* Description of quality */
const char * qstr = NULL; /* Quality string identifier */
qstr = smf_qual_str( SMF__QFAM_TCOMP, 1, k, &qdesc, status );
/* Set the quality name */
irqAddqn( qlocs, qstr, 0, qdesc, status );
curbit++;
irqFxbit( qlocs, qstr, curbit, &fixed, status );
}
/* compress them */
for (i = 0; i<nout; i++) {
qmap[i] = 0;
if (qual[i]) {
if ( qual[i] & (SMF__Q_BADDA|SMF__Q_BADB|SMF__Q_NOISE) ) {
qmap[i] |= SMF__TCOMPQ_BAD;
}
if ( qual[i] & (SMF__Q_APOD|SMF__Q_PAD) ) {
qmap[i] |= SMF__TCOMPQ_ENDS;
}
if ( qual[i] & (SMF__Q_JUMP|SMF__Q_SPIKE|SMF__Q_FILT|SMF__Q_EXT|SMF__Q_LOWAP|SMF__Q_BADEF) ) {
qmap[i] |= SMF__TCOMPQ_BLIP;
}
if ( qual[i] & (SMF__Q_COM) ) {
qmap[i] |= SMF__TCOMPQ_MATCH;
}
if ( qual[i] & (SMF__Q_STAT) ) {
qmap[i] |= SMF__TCOMPQ_TEL;
}
if (qmap[i] == 0 ) {
/* something went wrong. We missed a quality bit somewhere */
msgOutiff(MSG__QUIET, "", FUNC_NAME ": Untested quality bit found"
" in position %zu with value %u", status,
i, (unsigned int)qual[i]);
}
}
}
}
}
/* Unmap quality */
ndfUnmap( indf, "QUALITY", status );
/* Set the badbits mask to enable all quality by default.
Do not do this for MAP quality at the moment. */
if (family != SMF__QFAM_MAP) ndfSbb( 255, indf, status );
/* release IRQ resources */
irqRlse( &qlocs, status );
}
CLEANUP:
/* Tidy up */
if (qual) qual = astFree( qual );
return NULL;
}
