void smf_clean_dksquid( smfData *indata, smf_qual_t mask, size_t window, smfData *model,
int calcdk, int nofit, int replacebad, int *status ) {
dim_t b; /* Bolometer index */
size_t bstride; /* Bolometer index stride */
double corr; /* Linear correlation coefficient */
double *corrbuf=NULL; /* Array of correlation coeffs all bolos this col */
int needDA=0; /* Do we need dksquids from the DA? */
int *dkgood=NULL; /* Flag for non-constant dark squid */
double *dksquid=NULL; /* Buffer for smoothed dark squid */
double *dkav=NULL; /* Buffer for average dark squid */
double firstdk; /* First value in dksquid signal */
double gain; /* Gain parameter from template fit */
double *gainbuf=NULL; /* Array of gains for all bolos in this col */
size_t i; /* Loop counter */
size_t jt1;
size_t jt2;
size_t jf1; /* Starting tslice that should be fit */
size_t jf2; /* Final tslice that should be fit */
size_t j; /* Loop counter */
size_t k; /* Loop counter */
size_t nbad=0; /* Number of new bad bolos due to bad dark squid */
dim_t nbolo; /* Number of bolometers */
dim_t ncol; /* Number of columns */
dim_t ndata; /* Number of data points */
size_t nfit; /* number of samples over good range to fit */
size_t ngood=0; /* number of good dark squids */
dim_t nrow; /* Number of rows */
size_t ntot;
dim_t ntslice; /* Number of time slices */
double offset; /* Offset parameter from template fit */
double *offsetbuf=NULL; /* Array of offsets for all bolos in this col */
int pass; /* two passes over data to get estimate of average */
smf_qual_t *qua=NULL;/* Pointer to quality array */
size_t tstride; /* Time slice index stride */
if (*status != SAI__OK) return;
/* Check for NULL smfData pointer */
if( !indata ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, smfData pointer is NULL", status );
return;
}
/* Decide if we need the DA extension or not */
if( (!model) || (model && calcdk) ) {
needDA = 1;
if( !indata->da) {
/* Check for NULL smfDA */
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, no smfDA struct in smfData",
status);
return;
} else if( !indata->da->dksquid) {
/* Check for NULL dksquid */
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, no dksquid array in smfData",
status);
return;
}
/* Assert the correct data order here */
smf_dataOrder( indata->da->dksquid, 1, status );
}
/* Check for 3-d data and get dimensions */
smf_get_dims( indata, &nrow, &ncol, &nbolo, &ntslice, &ndata, &bstride,
&tstride, status );
/* Identify the range of data that should be fit using SMF__Q_BOUND */
if( qua ) {
smf_get_goodrange( qua, ntslice, tstride, SMF__Q_BOUND, &jf1, &jf2,
status );
} else {
jf1 = 0;
jf2 = ntslice-1;
}
nfit = jf2-jf1+1;
/* Total total range only using SMF__Q_PAD */
if( qua ) {
smf_get_goodrange( qua, ntslice, tstride, SMF__Q_BOUND, &jt1, &jt2,
status );
} else {
jt1 = 0;
jt2 = ntslice-1;
}
ntot = jt2-jt1+1;
if( model ) {
/* Check for valid model dimensions if supplied */
if( model->dtype != SMF__DOUBLE ) {
msgSetc("DT", smf_dtype_str(model->dtype, status) );
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": Data type ^DT for model not supported.",
status );
return;
}
if( (model->ndims != 2) ||
(model->dims[0] != ntslice+nrow*3) ||
(model->dims[1] != ncol) ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": model has incorrect dimensions", status );
return;
}
} else {
/* Otherwise allocate space for local dksquid buffer */
dksquid = astCalloc( ntslice, sizeof(*dksquid) );
}
/* Pointer to quality */
qua = smf_select_qualpntr( indata, 0, status );
/* Two passes: in the first we calculate an average dark squid to use as
a surrogate for columns with dead dark squids. In the second we do the
actual cleaning etc. */
dkgood = astCalloc( ncol, sizeof(*dkgood) );
dkav = astCalloc( ntslice, sizeof(*dkav) );
for( pass=0; (*status==SAI__OK)&&(pass<2); pass++ ) {
/* Loop over columns */
for( i=0; (*status==SAI__OK)&&(i<ncol); i++ ) {
/* Point dksquid, gainbuf, offsetbuf and corrbuf to the right
place in model if supplied. */
if( model ) {
dksquid = model->pntr[0];
dksquid += i*(ntslice+nrow*3);
gainbuf = dksquid + ntslice;
offsetbuf = gainbuf + nrow;
corrbuf = offsetbuf + nrow;
}
/* First pass is just to copy the dark squid over to the model and replace
dead dark squids with the average */
if( pass==0 ) {
/* Copy dark squids from the DA extension into dksquid */
if( needDA && calcdk && model ) {
double *ptr = indata->da->dksquid->pntr[0];
for( j=0; j<ntslice; j++ ) {
dksquid[j] = ptr[i+ncol*j];
}
}
/* Check for a good dark squid by seeing if it ever changes */
firstdk = VAL__BADD;
for( j=jt1; j<=jt2; j++ ) {
if( dksquid[j] != VAL__BADD ) {
if( firstdk == VAL__BADD ) {
firstdk = dksquid[j];
} else if( dksquid[j] != firstdk ) {
dkgood[i] = 1;
ngood++;
/* Add good squid to average dksquid */
for( k=jt1; k<=jt2; k++ ) {
dkav[k] += dksquid[k];
}
break;
}
}
}
}
/* Second pass actually do the fitting / replace with average dksquid if
dksquid was dead */
if( pass==1 ) {
/* Do some dksquid initialization if requested */
if( (*status==SAI__OK) && needDA && calcdk && model && dkgood[i] ) {
/* Smooth the dark squid template */
smf_boxcar1D( &dksquid[jt1], ntot, window, NULL, 0, status );
}
/* Initialize fit coeffs to VAL__BADD */
if( (*status == SAI__OK) && model ) {
for( j=0; j<nrow; j++ ) {
gainbuf[j] = VAL__BADD;
offsetbuf[j] = VAL__BADD;
corrbuf[j] = VAL__BADD;
}
}
/* Loop over rows, removing the fitted dksquid template. */
for( j=0; (!nofit) && (*status==SAI__OK) && (j<nrow); j++ ) {
/* Calculate bolometer index from row/col counters */
if( SC2STORE__COL_INDEX ) {
b = i*nrow + j;
} else {
b = i + j*ncol;
}
/* If dark squid is bad, flag entire bolo as bad if it isn't already */
if( !dkgood[i] && qua && !(qua[b*bstride]&SMF__Q_BADB) ) {
nbad++;
for( k=0; k<ntslice; k++ ) {
qua[b*bstride+k*tstride] |= SMF__Q_BADB;
}
}
/* Try to fit if we think we have a good dark squid and bolo, and only
the goodrange of data (excluding padding etc.) */
if((!qua && dkgood[i]) ||
(qua && dkgood[i] && !(qua[b*bstride]&SMF__Q_BADB))) {
double *d_d;
int *d_i;
switch( indata->dtype ) {
case SMF__DOUBLE:
d_d = (double *) indata->pntr[0];
smf_templateFit1D( &d_d[b*bstride+jf1*tstride],
&qua[b*bstride+jf1*tstride], NULL, NULL,
mask, mask, nfit, tstride, &dksquid[jf1], 1,
1, &gain, &offset, &corr, status );
break;
case SMF__INTEGER:
d_i = (int *) indata->pntr[0];
smf_templateFit1I( &d_i[b*bstride+jf1*tstride],
&qua[b*bstride+jf1*tstride], NULL, NULL, mask,
mask, nfit, tstride, &dksquid[jf1], 1, 1,
&gain, &offset, &corr, status );
break;
default:
msgSetc( "DT", smf_dtype_string( indata, status ));
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": Unsupported data type for dksquid cleaning (^DT)",
status );
}
if( *status == SMF__INSMP || *status == SMF__DIVBZ ) {
int wasinsmp = (*status == SMF__INSMP ? 1 : 0 );
/* Annul SMF__INSMP as it was probably due to a bad bolometer */
errAnnul( status );
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": ROW,COL (%zu,%zu) %s", status, j, i,
(wasinsmp ? "insufficient good samples" : "division by zero" ));
/* Flag entire bolo as bad if it isn't already */
if( qua && !(qua[b*bstride]&SMF__Q_BADB) ) {
for( k=0; k<ntslice; k++ ) {
qua[b*bstride+k*tstride] |= SMF__Q_BADB;
}
}
} else {
/* Store gain and offset in model */
if( model ) {
gainbuf[j] = gain;
offsetbuf[j] = offset;
corrbuf[j] = corr;
}
if( msgFlevok( MSG__DEBUG1, status ) ) {
msgSeti( "COL", i );
msgSeti( "ROW", j );
msgSetd( "GAI", gain );
msgSetd( "OFF", offset );
msgSetd( "CORR", corr );
msgOutif( MSG__DEBUG1, "", FUNC_NAME
": ROW,COL (^ROW,^COL) GAIN,OFFSET,CORR "
"(^GAI,^OFF,^CORR)", status );
}
}
}
}
}
}
/* Re-normalize the average dark-squid here at the end of pass 0 */
if( (pass==0) && (ngood) && (*status==SAI__OK) ) {
for( j=jt1; j<=jt2; j++ ) {
dkav[j] /= ngood;
}
}
/* Replace bad bolos in model with the average? */
if( replacebad && calcdk && needDA && model && (*status==SAI__OK) ) {
for( i=0; i<ncol; i++ ) {
dksquid = model->pntr[0];
dksquid += i*(ntslice+nrow*3);
if( !dkgood[i] ) {
memcpy( dksquid, dkav, sizeof(*dksquid)*ntslice );
dkgood[i] = 1;
}
}
}
}
/* Report number of new bad bolos that were flagged */
if( !replacebad && nbad ) {
msgOutiff( MSG__VERB, "", FUNC_NAME
": %zu new bolos flagged bad due to dead DKS", status, nbad );
}
/* Free dksquid only if it was a local buffer */
if( !model && dksquid ) dksquid = astFree( dksquid );
dkgood = astFree( dkgood );
dkav = astFree( dkav );
}
void smf_update_quality( ThrWorkForce *wf, smfData *data, int syncbad,
const int *badmask, smf_qual_t addqual, double badfrac,
int *status ) {
dim_t nbolo; /* Number of bolometers */
dim_t ndata; /* Number of data points */
dim_t ntslice; /* Number of time slices */
size_t bstride; /* bol stride */
size_t tstride; /* time slice stride */
smf_qual_t *qual=NULL; /* Pointer to the QUALITY array */
SmfUpdateQualityData *job_data = NULL;
SmfUpdateQualityData *pdata;
int nw;
size_t istep;
size_t bstep;
int iw;
if ( *status != SAI__OK ) return;
/* Check for QUALITY */
qual = smf_select_qualpntr( data, NULL, status );
if (!qual) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "smfData does not contain a QUALITY component",
status);
}
return;
}
/* Check for DATA */
if( !data->pntr[0] ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "smfData does not contain a DATA component", status );
return;
}
/* Check for valid badfrac */
if( (badfrac < 0) || (badfrac > 1) ) {
msgSeti( "BADFRAC", badfrac );
errRep(FUNC_NAME,
"Invalid badfrac: ^BADFRAC. Must be in range (0 -- 1).", status);
}
/* Calculate data dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride,
&tstride, status );
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many elements to process in each worker thread. */
istep = ndata/nw;
if( istep == 0 ) istep = 1;
/* Find how many bolometers to process in each worker thread. */
bstep = nbolo/nw;
if( bstep == 0 ) bstep = 1;
/* Allocate job data for threads, and store common values. Ensure that the
last thread picks up any left-over elements or bolometers. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->i1 = iw*istep;
pdata->b1 = iw*bstep;
if( iw < nw - 1 ) {
pdata->i2 = pdata->i1 + istep - 1;
pdata->b2 = pdata->b1 + bstep - 1;
} else {
pdata->i2 = ndata - 1;
pdata->b2 = nbolo - 1;
}
pdata->qual = qual;
}
}
if( *status == SAI__OK ) {
/* some pointers to the data array if needed */
double * ddata = NULL;
int * idata = NULL;
/* we will need the data array if we are checking it for bad values
or looking for bad fraction */
if (syncbad || badfrac) {
smf_select_pntr( data->pntr, data->dtype, &ddata, NULL,
&idata, NULL, status);
}
/* Synchronize SMF__Q_BADDA quality and VAL__BADD in data array */
if( syncbad ) {
if (data->dtype == SMF__DOUBLE) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 1;
pdata->ddata = ddata;
thrAddJob( wf, 0, pdata, smf1_update_quality, 0, NULL, status );
}
thrWait( wf, status );
} else if (data->dtype == SMF__INTEGER) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 2;
pdata->idata = idata;
thrAddJob( wf, 0, pdata, smf1_update_quality, 0, NULL, status );
}
thrWait( wf, status );
} else {
msgSetc( "TYP", smf_dtype_string( data, status ));
*status = SAI__ERROR;
errRep( "",FUNC_NAME " data is of unsupported type (^TYP)",
status);
return;
}
}
/* Apply badmask if available */
if( badmask || badfrac ) {
/* calculate the badfraction threshold in terms of number of bad
found so that we do not have to continually divide to calculate
the current fraction */
dim_t badthresh = ntslice;
/* special case 0 */
if (badfrac) badthresh = badfrac * (double)ntslice;
/* Submit the jobs and wait for them all to finish. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 3;
pdata->badthresh = badthresh;
pdata->addqual = addqual;
pdata->ntslice = ntslice;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->badmask = badmask;
pdata->syncbad = syncbad;
pdata->idata = idata;
pdata->ddata = ddata;
thrAddJob( wf, 0, pdata, smf1_update_quality, 0, NULL, status );
}
thrWait( wf, status );
}
}
job_data = astFree( job_data );
}
