void smf_open_ndfname( const HDSLoc *loc, const char accmode[], const char extname[], const char state[], const char dattype[], const int ndims, const int lbnd[], const int ubnd[], const char datalabel[], const char dataunits[], const AstFrameSet* wcs, smfData **ndfdata, int *status) { /* Local variables */ void *datarr[] = { NULL, NULL }; /* Pointers for data */ int dims[NDF__MXDIM]; /* Extent of each dimension */ smf_dtype dtype; /* Data type */ int flags = 0; /* Flags for creating smfDA, smfFile and smfHead components in the output smfData */ int i; int ndat; /* Number of elements mapped in the requested NDF */ char ndfaccmode[NDF__SZMMD+1];/* Access mode to use to open the file */ int ndimsmapped; /* Number of dimensions in mapped NDF */ int ndfid; /* NDF identifier */ AstFrameSet *ndfwcs = NULL; /* Copy of input FrameSet to write to NDF */ smfFile *newfile = NULL; /* New smfFile with details of requested NDF */ int place; /* Placeholder for NDF */ int updating = 0; /* True if the extension is being updated */ /* Initialize the output smfData to NULL pointer */ *ndfdata = NULL; if ( *status != SAI__OK ) return; /* Check to see if the HDS Locator is null and retrieve the NDF id */ if ( loc == NULL ) { errRep( FUNC_NAME, "Given HDS locator is NULL", status ); return; } /* Start be assuming the requested access mode can be used for mapping and file opening */ one_strlcpy( ndfaccmode, accmode, sizeof(ndfaccmode), status ); /* Note: write access clears the contents of the NDF */ if ( strncmp( accmode, "WRITE", 5 ) == 0 ) { msgOutif(MSG__DEBUG," ", "Opening NDF with WRITE access: this will clear the current contents if the NDF exists.", status); updating = 1; /* We can have WRITE/ZERO or WRITE/BAD so we need to force WRITE into the NDF open access mode */ one_strlcpy( ndfaccmode, "WRITE", sizeof(ndfaccmode), status ); } else if ( strncmp( accmode, "UPDATE", 6) == 0) { updating = 1; } ndfOpen( loc, extname, ndfaccmode, state, &ndfid, &place, status ); if ( *status != SAI__OK ) { errRep( FUNC_NAME, "Call to ndfOpen failed: unable to obtain an NDF identifier", status ); return; } /* No placeholder => NDF exists */ if ( place != NDF__NOPL ) { /* Define properties of NDF */ ndfNew( dattype, ndims, lbnd, ubnd, &place, &ndfid, status ); if ( *status != SAI__OK ) { errRep( FUNC_NAME, "Unable to create a new NDF", status ); return; } } /* Convert the data type string to SMURF dtype */ dtype = smf_dtype_fromstring( dattype, status ); /* First step is to create an empty smfData with no extra components */ flags |= SMF__NOCREATE_DA; flags |= SMF__NOCREATE_FTS; flags |= SMF__NOCREATE_HEAD; flags |= SMF__NOCREATE_FILE; *ndfdata = smf_create_smfData( flags, status); /* Set the requested data type */ (*ndfdata)->dtype = dtype; /* OK, now map the data array */ ndfMap( ndfid, "DATA", dattype, accmode, &datarr[0], &ndat, status ); if ( *status != SAI__OK ) { errRep( FUNC_NAME, "Unable to map data array: invalid NDF identifier?", status ); } /* Retrieve dimensions of mapped array */ ndfDim( ndfid, NDF__MXDIM, dims, &ndimsmapped, status ); if ( *status != SAI__OK ) { errRep( FUNC_NAME, "Problem identifying dimensions of requested NDF", status ); } /* Consistency check */ if ( ndimsmapped != ndims ) { if ( *status == SAI__OK ) { *status = SAI__ERROR; errRep( FUNC_NAME, "Number of dimensions in new NDF not equal to number of dimensions specified", status ); } } if (*status == SAI__OK) { for (i=0; i<ndims; i++) { ((*ndfdata)->dims)[i] = dims[i]; ((*ndfdata)->lbnd)[i] = lbnd[i]; } } /* Allow for label, units and WCS to be written */ if (updating) { if (datalabel) ndfCput( datalabel, ndfid, "Label", status ); if (dataunits) ndfCput( dataunits, ndfid, "Unit", status ); if (wcs) { /* Take a copy of the input WCS and modify if necessary that before writing to the NDF */ ndfwcs = astCopy( wcs ); smf_set_moving( (AstFrame *) ndfwcs, NULL, status ); ndfPtwcs( ndfwcs, ndfid, status ); if (ndfwcs) ndfwcs = astAnnul( ndfwcs ); } } /* Create the smfFile */ newfile = smf_construct_smfFile( newfile, ndfid, 0, 0, NULL, status ); if ( *status != SAI__OK ) { errRep( FUNC_NAME, "Unable to construct new smfFile", status ); } /* And populate the new smfData */ *ndfdata = smf_construct_smfData( *ndfdata, newfile, NULL, NULL, NULL, dtype, datarr, NULL, SMF__QFAM_NULL, NULL, 0, 1, (*ndfdata)->dims, (*ndfdata)->lbnd, ndims, 0, 0, NULL, NULL, status ); }
void smf_clean_smfArray( ThrWorkForce *wf, smfArray *array, smfArray **noisemaps, smfArray **com, smfArray **gai, AstKeyMap *keymap, int *status ) { /* Local Variables */ double badfrac; /* Fraction of bad samples to flag bad bolo */ smfData *data=NULL; /* Pointer to individual smfData */ int compreprocess; /* COMmon-mode cleaning as pre-processing step */ dim_t dcfitbox; /* width of box for measuring DC steps */ int dclimcorr; /* Min number of correlated steps */ int dcmaxsteps; /* number of DC steps/min. to flag bolo bad */ dim_t dcsmooth; /* median filter width before finding DC steps */ double dcthresh; /* n-sigma threshold for primary DC steps */ int dofft; /* are we doing a freq.-domain filter? */ int dkclean; /* Flag for dark squid cleaning */ smfFilter *filt=NULL; /* Frequency domain filter */ double flagfast; /* Threshold for flagging slow slews */ double flagslow; /* Threshold for flagging slow slews */ dim_t idx; /* Index within subgroup */ size_t nflag; /* Number of elements flagged */ double noisecliphigh = 0; /* Sigma clip high-noise outlier bolos */ double noisecliplow = 0; /* Sigma clip low-noise outlier bolos */ int noiseclipprecom = 0; /* Noise clipping before common-mode cleaning? */ const char *opteff=NULL; /* Pointer to optical efficiency NDF file name*/ int opteffdiv; /* Divide data by the optical efficiencies? */ int order; /* Order of polynomial for baseline fitting */ char param[ 20 ]; /* Buffer for config parameter name */ dim_t pcalen; /* Chunk length for PCA cleaning */ double pcathresh; /* n-sigma threshold for PCA cleaning */ double spikethresh; /* Threshold for finding spikes */ dim_t spikebox=0; /* Box size for spike finder */ struct timeval tv1, tv2; /* Timers */ int whiten; /* Apply whitening filter? */ int zeropad; /* Pad with zeros? */ /* Main routine */ if (*status != SAI__OK) return; /*** TIMER ***/ smf_timerinit( &tv1, &tv2, status ); /* Check for valid inputs */ if( !array || (array->ndat < 1) ) { *status = SAI__ERROR; errRep( "", FUNC_NAME ": No data supplied", status ); } if( array->sdata[0]->ndims != 3 ) { *status = SMF__WDIM; errRepf( "", FUNC_NAME ": Supplied smfData has %zu dims, needs 3", status, data->ndims ); return; } if( !keymap ) { *status = SAI__ERROR; errRep( "", FUNC_NAME ": NULL AstKeyMap supplied", status ); return; } /* Get cleaning parameters */ smf_get_cleanpar( keymap, array->sdata[0], &badfrac, &dcfitbox, &dcmaxsteps, &dcthresh, &dcsmooth, &dclimcorr, &dkclean, NULL, &zeropad, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, &flagslow, &flagfast, &order, &spikethresh, &spikebox, &noisecliphigh, &noisecliplow, NULL, &compreprocess, &pcalen, &pcathresh, NULL, NULL, NULL, &noiseclipprecom, status ); /* Loop over subarray */ for( idx=0; (idx<array->ndat)&&(*status==SAI__OK); idx++ ) { data = array->sdata[idx]; /* Update quality by synchronizing to the data array VAL__BADD values */ msgOutif(MSG__VERB,"", FUNC_NAME ": update quality", status); smf_update_quality( data, 1, NULL, 0, badfrac, status ); /*** TIMER ***/ msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s updating quality", status, smf_timerupdate(&tv1,&tv2,status) ); /* Fix DC steps */ if( dcthresh && dcfitbox ) { msgOutiff(MSG__VERB, "", FUNC_NAME ": Flagging bolos with %lf-sigma DC steps in %" DIM_T_FMT " " "samples as bad, using %" DIM_T_FMT "-sample median filter and max %d " "DC steps per min before flagging entire bolo bad...", status, dcthresh, dcfitbox, dcsmooth, dcmaxsteps); smf_fix_steps( wf, data, dcthresh, dcsmooth, dcfitbox, dcmaxsteps, dclimcorr, 0, &nflag, NULL, NULL, status ); msgOutiff(MSG__VERB, "", FUNC_NAME": ...%zd flagged\n", status, nflag); /*** TIMER ***/ msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s fixing DC steps", status, smf_timerupdate(&tv1,&tv2,status) ); } /* Flag Spikes */ if( spikethresh ) { msgOutif(MSG__VERB," ", FUNC_NAME ": flag spikes...", status); smf_flag_spikes( wf, data, SMF__Q_FIT, spikethresh, spikebox, &nflag, status ); msgOutiff(MSG__VERB,"", FUNC_NAME ": ...found %zd", status, nflag ); /*** TIMER ***/ msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s flagging spikes", status, smf_timerupdate(&tv1,&tv2,status) ); } /* Flag periods of stationary pointing, and update scanspeed to more accurate value */ if( flagslow || flagfast ) { if( data->hdr && data->hdr->allState ) { double scanvel=0; if( flagslow ) { msgOutiff( MSG__VERB, "", FUNC_NAME ": Flagging regions with slew speeds < %.2lf arcsec/sec", status, flagslow ); } if( flagfast ) { msgOutiff( MSG__VERB, "", FUNC_NAME ": Flagging regions with slew speeds > %.2lf arcsec/sec", status, flagfast ); /* Check to see if this was a sequence type that involved motion. If not, skip this section */ if( data && data->hdr && ( (data->hdr->seqtype==SMF__TYP_SCIENCE) || (data->hdr->seqtype==SMF__TYP_POINTING) || (data->hdr->seqtype==SMF__TYP_FOCUS) || (data->hdr->seqtype==SMF__TYP_SKYDIP)) && (data->hdr->obsmode!=SMF__OBS_STARE) ) { smf_flag_slewspeed( data, flagslow, flagfast, &nflag, &scanvel, status ); msgOutiff( MSG__VERB,"", "%zu new time slices flagged", status, nflag); if( msgIflev( NULL, status ) >= MSG__VERB ) { msgOutf( "", FUNC_NAME ": mean SCANVEL=%.2lf arcsec/sec" " (was %.2lf)", status, scanvel, data->hdr->scanvel ); } data->hdr->scanvel = scanvel; /*** TIMER ***/ msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s flagging outlier slew speeds", status, smf_timerupdate(&tv1,&tv2,status) ); } else { msgOutif( MSG__VERB, "", FUNC_NAME ": not a moving sequence or missing header, " "skipping slew speed flagging", status ); } } } else { msgOutif( MSG__DEBUG, "", FUNC_NAME ": Skipping flagslow/flagfast because no header present", status ); } } /* Clean out the dark squid signal */ if( dkclean ) { msgOutif(MSG__VERB, "", FUNC_NAME ": Cleaning dark squid signals from data.", status); smf_clean_dksquid( data, 0, 100, NULL, 0, 0, 0, status ); /*** TIMER ***/ msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s DKSquid cleaning", status, smf_timerupdate(&tv1,&tv2,status) ); } /* Apply optical efficiency corrections. */ one_strlcpy( param, "OPTEFF", sizeof(param), status ); smf_find_subarray( data->hdr, param + strlen(param), sizeof(param) - strlen(param), NULL, status ); astChrCase( NULL, param, 1, 0 ); if( astMapHasKey( keymap, param ) ) { astMapGet0I( keymap, "OPTEFFDIV", &opteffdiv ); if ( astMapGet0C( keymap, param, &opteff ) ) { msgOutiff( MSG__VERB,"", FUNC_NAME ": %s bolometer values " "by factors read from NDF %s", status, opteffdiv ? "Dividing" : "Multiplying", opteff ); smf_scale_bols( wf, data, NULL, opteff, param, opteffdiv, status ); } } /* Remove baselines */ if( order >= 0 ) { msgOutiff( MSG__VERB,"", FUNC_NAME ": Fitting and removing %i-order polynomial baselines", status, order ); smf_fit_poly( wf, data, order, 1, NULL, status ); /*** TIMER ***/ msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s removing poly baseline", status, smf_timerupdate(&tv1,&tv2,status) ); } } /* Mask noisy bolos here if happening before common-mode cleaning */ if( (*status == SAI__OK) && ((noisecliphigh>0.0) || (noisecliplow>0.0)) && noiseclipprecom ) { smf__noisymask( wf, data, noisemaps, noisecliphigh, noisecliplow, zeropad, &tv1, &tv2, status ); } /* Optionally call smf_calcmodel_com to perform a subset of the following tasks as a pre-processing step: - remove the common-mode - flag outlier data using common-mode rejection - determine relative flatfields using amplitude of common-mode In order to do this we need to set up some temporary model container files so that the routine can be called properly. All of the same COMmon-mode and GAIn model parameters (e.g. com.* and gai.*) will be used here. However, in addition the "compreprocess" flag must be set for this operation to be performed. */ if( compreprocess ) { smfArray *comdata = NULL; smfGroup *comgroup = NULL; smfDIMMData dat; smfArray *gaidata = NULL; smfGroup *gaigroup = NULL; smfArray *quadata = NULL; smfData *thisqua=NULL; msgOutif(MSG__VERB," ", FUNC_NAME ": Remove common-mode", status); /* Create model containers for COM, GAI */ smf_model_create( wf, NULL, &array, NULL, NULL, NULL, NULL, NULL, 1, SMF__COM, 0, NULL, 0, NULL, NULL, &comgroup, &comdata, keymap, status ); smf_model_create( wf, NULL, &array, NULL, NULL, NULL, NULL, NULL, 1, SMF__GAI, 0, NULL, 0, NULL, NULL, &gaigroup, &gaidata, keymap, status ); /* Manually create quadata to share memory with the quality already stored in array */ quadata = smf_create_smfArray( status ); for( idx=0; (*status==SAI__OK) && (idx<array->ndat); idx++ ) { /* Create several new smfDatas, but they will all be freed properly when we close quadata */ thisqua = smf_create_smfData( SMF__NOCREATE_DA | SMF__NOCREATE_HEAD | SMF__NOCREATE_FILE, status ); /* Probably only need pntr->[0], but fill in the dimensionality information to be on the safe side */ thisqua->dtype = SMF__QUALTYPE; thisqua->ndims = array->sdata[idx]->ndims; thisqua->isTordered = array->sdata[idx]->isTordered; memcpy( thisqua->dims, array->sdata[idx]->dims, sizeof(thisqua->dims) ); memcpy( thisqua->lbnd, array->sdata[idx]->lbnd, sizeof(thisqua->lbnd) ); thisqua->pntr[0] = smf_select_qualpntr( array->sdata[idx], NULL, status ); smf_addto_smfArray( quadata, thisqua, status ); } /* Set up the smfDIMMData and call smf_calcmodel_com */ memset( &dat, 0, sizeof(dat) ); dat.res = &array; dat.gai = &gaidata; dat.qua = &quadata; dat.noi = NULL; smf_calcmodel_com( wf, &dat, 0, keymap, &comdata, SMF__DIMM_FIRSTITER, status ); /*** TIMER ***/ msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s removing common-mode", status, smf_timerupdate(&tv1,&tv2,status) ); /* Clean up and/or return values */ if( com ) { *com = comdata; } else { if( comdata ) smf_close_related( &comdata, status ); } if( gai ) { *gai = gaidata; } else { if( gaidata ) smf_close_related( &gaidata, status ); } if( comgroup ) smf_close_smfGroup( &comgroup, status ); if( gaigroup ) smf_close_smfGroup( &gaigroup, status ); /* Before closing quadata unset all the pntr[0] since this is shared memory with the quality associated with array */ if( quadata ) { for( idx=0; idx<quadata->ndat; idx++ ) { quadata->sdata[idx]->pntr[0] = NULL; } if( quadata ) smf_close_related( &quadata, status ); } } /* PCA cleaning */ if( pcathresh ) { /* Loop over subarray */ for( idx=0; (idx<array->ndat)&&(*status==SAI__OK); idx++ ) { data = array->sdata[idx]; smf_clean_pca_chunks( wf, data, pcalen, pcathresh, keymap, status ); } /*** TIMER ***/ msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s PCA cleaning", status, smf_timerupdate(&tv1,&tv2,status) ); } /* Allocate space for noisemaps if required */ if( noisemaps ) { *noisemaps = smf_create_smfArray( status ); } /* Loop over subarray */ for( idx=0; (idx<array->ndat)&&(*status==SAI__OK); idx++ ) { data = array->sdata[idx]; /* Filter the data. Note that we call smf_filter_execute to apply a per-bolometer whitening filter even if there is no explicitly requested smfFilter (in which case the smf_filter_fromkeymap call will leave the real/imaginary parts of the filter as NULL pointers and they will get ignored inside smf_filter_execute). */ filt = smf_create_smfFilter( data, status ); smf_filter_fromkeymap( filt, keymap, data->hdr, &dofft, &whiten, status ); if( (*status == SAI__OK) && dofft ) { msgOutif( MSG__VERB, "", FUNC_NAME ": frequency domain filter", status ); smf_filter_execute( wf, data, filt, 0, whiten, status ); /*** TIMER ***/ msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s filtering data", status, smf_timerupdate(&tv1,&tv2,status) ); } filt = smf_free_smfFilter( filt, status ); /* Mask noisy bolos here if happening after common-mode cleaning */ if( (*status == SAI__OK) && ((noisecliphigh>0.0) || (noisecliplow>0.0)) && !noiseclipprecom ) { smf__noisymask( wf, data, noisemaps, noisecliphigh, noisecliplow, zeropad, &tv1, &tv2, status ); } } }
void smf_write_smfFilter( ThrWorkForce *wf, const smfFilter *filt, const char *filename, const Grp * igrp, size_t grpindex, int *status ) { double *d = NULL; /* Data array pointer */ smfData *data=NULL; /* smfData for output */ size_t i; /* Loop counter */ size_t nsamp; /* Number of samples in the filter */ if( *status != SAI__OK ) return; if( !filt ) { *status = SAI__ERROR; errRep( "", FUNC_NAME ": NULL smfFilter supplied.", status ); return; } /* In case we change the data type of smfFilters in the future try to catch that here */ if( sizeof(filt->real) != smf_dtype_sz(SMF__DOUBLE,status) ) { *status = SAI__ERROR; errRep( "", FUNC_NAME ": warning -- smfFilter seems to have changed type!", status ); return; } /* We will pack the data into an array that has an extra dimension to store the real/imaginary parts of the filter just like an FFT */ data = smf_create_smfData( 0, status ); if( *status == SAI__OK ) { data->dtype = SMF__DOUBLE; data->ndims = filt->ndims+1; nsamp=1; for( i=0; i<filt->ndims; i++ ) { data->dims[i] = filt->fdims[i]; nsamp *= filt->fdims[i]; } data->dims[data->ndims-1] = 2; /* Copy the real and imaginary parts into our new array consecutively */ data->pntr[0] = astCalloc( nsamp*2, smf_dtype_sz(SMF__DOUBLE,status) ); if( *status == SAI__OK ) { d = data->pntr[0]; memcpy( d, filt->real, nsamp*sizeof(d) ); if( filt->isComplex ) { memcpy( d + nsamp, filt->imag, nsamp*sizeof(d) ); } } } /* Write out the file */ smf_write_smfData( wf, data, NULL, filename, igrp, grpindex, 0, MSG__NORM, 0, NULL, NULL, status ); if( data ) smf_close_file( wf, &data, status ); }
void smurf_sc2threadtest( int *status ) { /* Local Variables */ smfArray **res=NULL; /* array of smfArrays of test data */ smfData *data=NULL; /* Pointer to SCUBA2 data struct */ dim_t datalen; /* Number of data points */ smfFilter *filt=NULL; /* Frequency domain filter */ size_t i; /* Loop counter */ size_t j; /* Loop counter */ smfTimeChunkData *job_data=NULL; /* Array of pointers for job data */ size_t joblen; /* Number of chunks per job */ size_t k; /* Loop counter */ size_t nchunks; /* Number of chunks */ size_t nsub; /* Number of subarrays */ int nthread; /* Number of threads */ smfTimeChunkData *pdata=NULL; /* Pointer to data for single job */ int temp; /* Temporary integer */ size_t tsteps; /* How many time steps in chunk */ struct timeval tv1, tv2; /* Timers */ ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */ double *dat=NULL; dim_t nbolo; dim_t ntslice; dim_t ndata; size_t bstride; size_t tstride; dim_t offset; if (*status != SAI__OK) return; /* Get input parameters */ parGdr0i( "NTHREAD", 1, 1, NUM__MAXI, 1, &nthread, status ); parGdr0i( "TSTEPS", 6000, 0, NUM__MAXI, 1, &temp, status ); tsteps = (size_t) temp; parGdr0i( "NCHUNKS", 1, 1, NUM__MAXI, 1, &temp, status ); nchunks = (size_t) temp; parGdr0i( "NSUB", 1, 1, 4, 1, &temp, status ); nsub = (size_t) temp; msgSeti("N",nthread); msgOut( "", TASK_NAME ": Running test with ^N threads", status ); /*** TIMER ***/ smf_timerinit( &tv1, &tv2, status ); /* Create some fake test data in the form of an array of smfArrays */ msgSeti("T",tsteps); msgSeti("C",nchunks); msgSeti("NS",nsub); msgOut( "", TASK_NAME ": Creating ^NS subarrays of data with ^C chunks * ^T samples", status ); res = astCalloc( nchunks, sizeof(*res) ); for( k=0; (*status==SAI__OK)&&(k<nchunks); k++ ) { res[k] = smf_create_smfArray( status ); for( i=0; (*status==SAI__OK)&&(i<nsub); i++ ) { /* Create individual smfDatas and add to array */ data = smf_create_smfData( SMF__NOCREATE_FILE | SMF__NOCREATE_DA | SMF__NOCREATE_FTS, status ); if( *status==SAI__OK ) { data->dtype=SMF__DOUBLE; data->ndims=3; data->dims[0]=40; data->dims[1]=32; data->dims[2]=(dim_t) tsteps; datalen=1; data->isFFT=-1; for( j=0; j<data->ndims; j++ ) datalen *= data->dims[j]; data->hdr->steptime = 0.005; data->pntr[0] = astCalloc( datalen, smf_dtype_sz(data->dtype,status) ); data->qual = astCalloc( datalen, sizeof(*data->qual) ); } smf_addto_smfArray( res[k], data, status ); } } /*** TIMER ***/ msgOutf( "", "** %f seconds generating data", status, smf_timerupdate(&tv1,&tv2,status) ); msgOut( "", TASK_NAME ": Starting test 1 __parallel time: dataOrder__", status ); /* Create a pool of threads. */ wf = thrGetWorkforce( nthread, status ); /* Work out number of chunks per thread */ joblen = nchunks/nthread; if( joblen == 0 ) joblen = 1; /* At least one chunk per thread */ /* The first test will process separate time chunks of data in parallel, re-ordering each to bolo-ordered format. All subarrays and an integer number of input file chunks all go into a single thread. Start by allocating and initializing a number of smfTimeChunkData's that hold the information required for each thread */ job_data = astCalloc( nthread, sizeof(*job_data) ); for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) { pdata = job_data + i; pdata->type = 0; /* Start with a data re-order */ pdata->data = res; /* Pointer to main data array */ pdata->chunk1 = i*joblen; /* Index of first chunk for job */ pdata->nchunks = nchunks; /* Total number of time chunks in data */ pdata->ijob = -1; /* Flag job as available to do work */ /* The last thread has to pick up the remainder of chunks */ if( i==(size_t)(nthread-1) ) pdata->chunk2=nchunks-1; else pdata->chunk2 = (i+1)*joblen-1; /* Index of last chunk for job */ /* Ensure a valid chunk range, or set to a length that we know to ignore */ if( pdata->chunk1 >= nchunks ) { pdata->chunk1 = nchunks; pdata->chunk2 = nchunks; } else if( pdata->chunk2 >= nchunks ) { pdata->chunk2 = nchunks-1; } if( pdata->chunk1 >= nchunks ) { /* Nothing for this thread to do */ msgSeti( "W", i+1); msgOutif( MSG__DEBUG, "", "-- parallel time: skipping thread ^W, nothing to do", status); } else { /* Since we know there is one job_data per thread, just submit jobs immediately */ pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime, 0, NULL, status ); } } /* Wait until all of the submitted jobs have completed */ thrWait( wf, status ); /* Annul the bad status that we set in smfParallelTime */ if( *status == SMF__INSMP ) { errAnnul( status ); msgOut( "", " *** Annulled SMF__INSMP set in smfParallelTime *** ", status ); } else { msgOut( "", " *** Flushing good status *** ", status ); errFlush( status ); } /*** TIMER ***/ msgOutf( "", "** %f seconds to complete test", status, smf_timerupdate(&tv1,&tv2,status) ); /* The second test will boxcar smooth bolometers from time chunks in parallel */ msgOut( "", TASK_NAME ": Starting test 2 __parallel time: boxcar smooth__", status ); for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) { pdata = job_data + i; pdata->type = 1; /* Boxcar smooth */ if( pdata->chunk1 >= nchunks ) { /* Nothing for this thread to do */ msgSeti( "W", i+1); msgOutif( MSG__DEBUG, "", "-- parallel time: skipping thread ^W, nothing to do", status); } else { /* Since we know there is one job_data per thread, just submit jobs immediately */ pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime, 0, NULL, status ); } } /* Wait until all of the submitted jobs have completed */ thrWait( wf, status ); /*** TIMER ***/ msgOutf( "", "** %f seconds to complete test", status, smf_timerupdate(&tv1,&tv2,status) ); msgOut( "", TASK_NAME ": *** Next 2 tests will be done twice due to FFTW planning *****", status ); for( k=0; k<2; k++ ) { /* The third test will FFT filter bolometers from time chunks in parallel */ msgOut( "", TASK_NAME ": Starting test 3 __parallel time: FFT filter__", status ); for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) { pdata = job_data + i; pdata->type = 2; /* FFT filter */ if( pdata->chunk1 >= nchunks ) { /* Nothing for this thread to do */ msgSeti( "W", i+1); msgOutif( MSG__DEBUG, "", "-- parallel time: skipping thread ^W, nothing to do", status); } else { /* Since we know there is one job_data per thread, just submit jobs immediately */ pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime, 0, NULL, status ); } } /* Wait until all of the submitted jobs have completed */ thrWait( wf, status ); /*** TIMER ***/ msgOutf( "", "** %f seconds to complete test", status, smf_timerupdate(&tv1,&tv2,status) ); msgOut( "", TASK_NAME ": Starting test 4 __FFTW filter using internal threading__", status ); for( i=0; (*status==SAI__OK)&&(i<nchunks); i++ ) { filt = smf_create_smfFilter( res[i]->sdata[0], status ); smf_filter_ident( filt, 1, status ); for( j=0; (*status==SAI__OK)&&(j<nsub); j++ ) { msgOutiff( MSG__DEBUG, "", " filter chunk %zu/%zu, bolo %zu/%zu", status, i+1, nchunks, j+1, nsub ); smf_filter_execute( wf, res[i]->sdata[j], filt, 0, 0, status ); } if( filt ) filt = smf_free_smfFilter( filt, status ); } /*** TIMER ***/ msgOutf( "", "** %f seconds to complete test", status, smf_timerupdate(&tv1,&tv2,status) ); } msgOut( "", TASK_NAME ": **************************************************************", status ); /* Series of short single-thread array index tests */ data = res[0]->sdata[0]; dat = data->pntr[0]; smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride, &tstride, status ); msgOut("","Array index test #1: two multiplies in inner loop",status); smf_timerinit( &tv1, &tv2, status ); for( i=0; i<nbolo; i++ ) { for( j=0; j<ntslice; j++ ) { dat[i*bstride + j*tstride] += 5; } } msgOutf( "", "** %f seconds to complete test", status, smf_timerupdate(&tv1,&tv2,status) ); msgOut("","Array index test #2: only index increments",status); smf_timerinit( &tv1, &tv2, status ); for( i=0; i<nbolo*bstride; i+=bstride ) { for( j=i; j<(i+ntslice*tstride); j+=tstride ) { dat[j] += 5; } } msgOutf( "", "** %f seconds to complete test", status, smf_timerupdate(&tv1,&tv2,status) ); msgOut("","Array index test #3: one multiply in outer loop",status); smf_timerinit( &tv1, &tv2, status ); offset = 0; for( i=0; i<nbolo; i++ ) { offset = i*bstride; for( j=0; j<ntslice; j++ ) { dat[offset] += 5; offset += tstride; } } msgOutf( "", "** %f seconds to complete test", status, smf_timerupdate(&tv1,&tv2,status) ); /* Clean up */ if( res ) { for( i=0; i<nchunks; i++ ) { if( res[i] ) { smf_close_related( &res[i], status ); } } res = astFree( res ); } job_data = astFree( job_data ); /* Ensure that FFTW doesn't have any used memory kicking around */ fftw_cleanup(); }