dim_t smf_get_padding( AstKeyMap *keymap, int report, const smfHead *hdr, double steptime, int *status ) { /* Local Variables: */ AstObject *obj; const char *key; dim_t pad; dim_t result; double f_low; double filt_edgehigh; double filt_edgelarge; double filt_edgelow; double filt_edgesmall; double filt_notchlow[ SMF__MXNOTCH ]; int f_nnotch; int iel; int nel; int temp; double scalelen; double downsampscale; double downsampfreq; /* Initialise */ result = 0; /* Main routine */ if (*status != SAI__OK) return result; /* If the keymap contains a PAD value, return it. */ if( astMapGet0I( keymap, "PAD", &temp ) ) { if( temp < 0 && *status == SAI__OK ) { *status = SAI__ERROR; errRep( "", "PAD cannot be < 0.", status ); } else { result = (dim_t) temp; } /* If the keymap contains no PAD value, calculate a default on the basis of the FILT_ values. */ } else { /* If the keymap does not contain a PAD value, not even an "<undef>" value, an error will have been reported by astMapget0I above. In this case, annull the error and continue since the KeyMap may contain FILT_ values that can be used to calculate a default PAD value. */ if( *status == AST__MPKER ) errAnnul( status ); /* Search for filtering parameters in the keymap. None of these parameters represent a number of time clies, so we can set the smfData (the 2nd argument) to NULL. */ f_nnotch = 0; smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, &filt_edgelow, &filt_edgehigh, &filt_edgesmall, &filt_edgelarge, filt_notchlow, NULL, &f_nnotch, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, &downsampscale, &downsampfreq, NULL, status ); if( downsampscale && downsampfreq ) { *status = SAI__ERROR; errRep( "", FUNC_NAME ": both downsampscale and downsampfreq are set", status ); } /* If any were not found, annul the error and continue with those that were found (plus defaults for the others). */ if( *status == AST__MPKER ) errAnnul( status ); if( *status == SAI__OK ) { /* Modify edge filters if spatial scales were requested */ smf_scale2freq( filt_edgesmall, filt_edgelarge, hdr, &filt_edgelow, &filt_edgehigh, status ); if( *status == SMF__TELSTAT ) { errAnnul( status ); } /* Find the lowest of these frequencies. The lowest frequency will give the greatest padding. */ f_low = ( filt_edgehigh > 0.0 ) ? filt_edgehigh : VAL__MAXD; if( filt_edgelow > 0.0 && filt_edgelow < f_low ) f_low = filt_edgelow; for( iel = 0; iel < f_nnotch; iel++ ) { if( filt_notchlow[ iel ] > 0.0 && filt_notchlow[ iel ] < f_low ) f_low = filt_notchlow[ iel ]; } /* If any down-sampling is occurring, set the step time to value implied by the requested downsampling factor. This should match the calculation in smf_grp_related */ if( downsampscale || downsampfreq ) { if( downsampscale ) { scalelen = hdr->steptime * hdr->scanvel / downsampscale; } else { if( hdr->steptime ) { scalelen = downsampfreq / (1./hdr->steptime); } else { *status = SAI__ERROR; scalelen = VAL__BADD; errRep( "", FUNC_NAME ": can't resample because smfData has " "unknown sample rate", status ); } } if( (*status==SAI__OK) && (scalelen<=SMF__DOWNSAMPLIMIT) ) steptime = hdr->steptime/scalelen; /* If no downsampling is happening, use the steptime in the header unless another steptime was supplied. */ } else if( steptime == VAL__BADD ) { steptime = hdr->steptime; } /* Find the corresponding padding. */ if( f_low != VAL__MAXD ) { result = 1.0/( steptime * f_low ); } else { result = 0; } /* Now check the supplied keymap for any nested keymaps. Assumes that each entry in the supplied KeyMap contain either a primitive value or a scalar KeyMap pointer. */ nel = astMapSize( keymap ); for( iel = 0; iel < nel; iel++ ) { key = astMapKey( keymap, iel ); /* If this entry is a KeyMap (assuming no other class of AST object is stored in the KeyMap)... */ if( astMapType( keymap, key ) == AST__OBJECTTYPE ) { /* Get a pointer to the KeyMap. */ (void) astMapGet0A( keymap, key, &obj ); /* Call this function to get the padding implied by the sub-KeyMap. */ pad = smf_get_padding( (AstKeyMap *) obj, report, hdr, steptime, status ); /* Use the larger of the two paddings. */ if( pad > result ) result = pad; } } /* If required, report the default value. */ if( report ) { msgSeti( "P", (int) result ); msgOutif( MSG__VERB, "", "Padding each time stream with ^P zero " "values at start and end.", status ); } } } /* Return the result. */ return result; }
void smf_grp_related( const Grp *igrp, const size_t grpsize, const int grouping, const int checksubinst, double maxlen_s, double *srate_maxlen, AstKeyMap *keymap, dim_t *maxconcatlen, dim_t *maxfilelen, smfGroup **group, Grp **basegrp, dim_t *pad, int *status ) { /* Local variables */ size_t *chunk=NULL; /* Array of flags for continuous chunks */ dim_t * chunklen = NULL; /* Length of continuous chunk */ size_t currentindex = 0; /* Counter */ char cwave[10]; /* String containing wavelength */ smfData *data = NULL; /* Current smfData */ double downsampscale=0; /* Angular scale downsampling size */ double downsampfreq=0; /* Target downsampling frequency */ AstKeyMap * grouped = NULL; /* Primary AstKeyMap for grouping */ size_t i; /* Loop counter for index into Grp */ int isFFT=0; /* Set if data are 4d FFT */ size_t j; /* Loop counter */ int *keepchunk=NULL; /* Flag for chunks that will be kept */ dim_t maxconcat=0; /* Longest continuous chunk length */ dim_t maxflen=0; /* Max file length in time steps */ dim_t maxlen=0; /* Maximum concat length in samples */ int maxlen_scaled=0; /* Set once maxlen has been scaled, if needed */ dim_t maxpad=0; /* Maximum padding neeed for any input file */ size_t maxrelated = 0; /* Keep track of max number of related items */ size_t *new_chunk=NULL; /* keeper chunks associated with subgroups */ dim_t *new_tlen=NULL; /* tlens for new_subgroup */ size_t ngroups = 0; /* Counter for subgroups to be stored */ size_t nkeep = 0; /* Number of chunks to keep */ dim_t * piecelen = NULL; /* Length of single file */ smf_subinst_t refsubinst; /* Subinst of first file */ size_t **subgroups = NULL; /* Array containing index arrays to parent Grp */ smf_subinst_t subinst; /* Subinst of current file */ if ( *status != SAI__OK ) return; if( maxlen_s < 0 ) { *status = SAI__ERROR; errRep( "", FUNC_NAME ": maxlen_s cannot be < 0!", status ); return; } /* Get downsampling parameters */ if( keymap ) { smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, &downsampscale, &downsampfreq, NULL, NULL, NULL, NULL, status ); if( downsampscale && downsampfreq ) { *status = SAI__ERROR; errRep( "", FUNC_NAME ": both downsampscale and downsampfreq are set", status ); return; } } /* Initialize refcwave */ refsubinst = SMF__SUBINST_NONE; /* Loop over files in input Grp: remember Grps are indexed from 1 */ grouped = astKeyMap( "SortBy=KeyUp" ); for (i=1; i<=grpsize; i++) { char newkey[128]; char dateobs[81]; char subarray[10]; size_t nrelated = 0; AstKeyMap * filemap = NULL; AstKeyMap * indexmap = NULL; /* First step: open file and harvest metadata */ smf_open_file( NULL, igrp, i, "READ", SMF__NOCREATE_DATA, &data, status ); if (*status != SAI__OK) break; if( i==1 ) { isFFT = smf_isfft( data, NULL, NULL, NULL, NULL, NULL, status ); } else if( smf_isfft(data, NULL, NULL, NULL, NULL, NULL, status) != isFFT ){ *status = SAI__ERROR; errRep( "", FUNC_NAME ": mixture of time-series and FFT data encountered!", status ); break; } /* If maxlen has not been set, do it here */ if( !maxlen && maxlen_s && data->hdr->steptime) { maxlen = (dim_t) (maxlen_s / data->hdr->steptime ); } /* Return srate_maxlen if requested: may want to know this number even if maxlen_s is not set. Only calculate once, although it gets overwritten once later if down-sampling. */ if( (i==1) && srate_maxlen && data->hdr->steptime ) { *srate_maxlen = 1. / (double) data->hdr->steptime; } /* If requested check to see if we are mixing wavelengths */ if( checksubinst ) { if( refsubinst == SMF__SUBINST_NONE ) { refsubinst = smf_calc_subinst( data->hdr, status ); } subinst = smf_calc_subinst( data->hdr, status ); if( subinst != refsubinst ) { const char *refsubstr = smf_subinst_str( refsubinst, status ); const char *substr = smf_subinst_str( subinst, status ); *status = SAI__ERROR; smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" ); msgSetc( "REFSUB", refsubstr ); msgSetc( "SUB", substr ); errRep( "", FUNC_NAME ": ^FILE uses sub-instrument ^SUB which doesn't match " "reference ^REFSUB", status ); } } /* Want to form a key that will be unique for a particular subscan We know that DATE-OBS will be set for SCUBA-2 files and be the same for a single set. Prefix by wavelength if we are grouping by wavelength. */ newkey[0] = '\0'; smf_find_subarray( data->hdr, subarray, sizeof(subarray), NULL, status ); if( grouping == 1 ) { /* Group different wavelengths separately */ smf_fits_getS( data->hdr, "WAVELEN", cwave, sizeof(cwave), status); one_strlcat( newkey, cwave, sizeof(newkey), status ); one_strlcat( newkey, "_", sizeof(newkey), status ); } if( grouping == 2 ) { /* Group different subarrays separately */ one_strlcat( newkey, subarray, sizeof(newkey), status ); } smf_fits_getS( data->hdr, "DATE-OBS", dateobs, sizeof(dateobs), status ); one_strlcat( newkey, dateobs, sizeof(newkey), status ); /* Include the dimentionality of the time series in the primary key so that we do not end up doing something confusing like relating a truncated file with a full length file */ if (*status == SAI__OK) { dim_t dims[3]; char formatted[32]; smf_get_dims( data, &dims[0], &dims[1], NULL, &dims[2], NULL, NULL, NULL, status ); sprintf(formatted, "_%" DIM_T_FMT "_%" DIM_T_FMT "_%" DIM_T_FMT, dims[0], dims[1], dims[2]); one_strlcat( newkey, formatted, sizeof(newkey), status ); } /* May want to read the dimensionality of the file outside of loop so that we can compare values when storing in the keymap */ /* Now we want to create a keymap based on this key */ if (!astMapGet0A( grouped, newkey, &filemap ) ) { int itemp = 0; double steptime = data->hdr->steptime; dim_t ntslice = 0; dim_t thispad; /* Padding neeed for current input file */ filemap = astKeyMap( " " ); astMapPut0A( grouped, newkey, filemap, NULL ); /* Fill up filemap with general information on this file */ smf_find_seqcount( data->hdr, &itemp, status ); astMapPut0I( filemap, "SEQCOUNT", itemp, NULL ); smf_fits_getI( data->hdr, "NSUBSCAN", &itemp, status ); astMapPut0I( filemap, "NSUBSCAN", itemp, NULL ); /* Number of time slices */ smf_get_dims( data, NULL, NULL, NULL, &ntslice, NULL, NULL, NULL, status ); /* Find length of down-sampled data, new steptime and maxlen */ if( (downsampscale || downsampfreq) && data->hdr && (*status==SAI__OK) ) { double scalelen; if( downsampscale ) { if( data->hdr->scanvel != VAL__BADD ) { double oldscale = steptime * data->hdr->scanvel; scalelen = oldscale / downsampscale; } else if( *status == SAI__OK ) { *status = SAI__ERROR; scalelen = VAL__BADD; smf_smfFile_msg( data->file, "FILE", 1, "" ); errRep( "", FUNC_NAME ": can't resample ^FILE because it has " "unknown scan velocity", status ); } } else { if( steptime ) { double oldsampfreq = 1./steptime; scalelen = downsampfreq / oldsampfreq; } else { *status = SAI__ERROR; scalelen = VAL__BADD; smf_smfFile_msg( data->file, "FILE", 1, "" ); errRep( "", FUNC_NAME ": can't resample ^FILE because it has " "unknown sample rate", status ); } } /* only down-sample if it will be a reasonable factor */ if( (*status==SAI__OK) && (scalelen <= SMF__DOWNSAMPLIMIT) ) { smf_smfFile_msg(data->file, "FILE", 1, "" ); msgOutiff( MSG__VERB, "", FUNC_NAME ": will down-sample file ^FILE from %5.1lf Hz to " "%5.1lf Hz", status, (1./steptime), (scalelen/steptime) ); ntslice = round(ntslice * scalelen); /* If maxlen has been requested, and we have not already worked out a scaled version (just uses the sample rates for the first file... should be close enough -- the alternative is a 2-pass system). */ if( !maxlen_scaled ) { maxlen = round(maxlen*scalelen); maxlen_scaled = 1; msgOutiff( MSG__VERB, "", FUNC_NAME ": requested maxlen %g seconds = %" DIM_T_FMT " down-sampled " "time-slices", status, maxlen_s, maxlen ); /* Return updated srate_maxlen for down-sampling if requested */ if( srate_maxlen ) { *srate_maxlen = scalelen/steptime; } } } } /* Check that an individual file is too long (we assume related files are all the same) */ if( maxlen && (ntslice > maxlen) && *status == SAI__OK) { *status = SAI__ERROR; msgSeti("NTSLICE",ntslice); msgSeti("MAXLEN",maxlen); smf_smfFile_msg( data->file, "FILE", 1, "" ); errRep(FUNC_NAME, "Number of time steps in file ^FILE time exceeds maximum " "(^NTSLICE>^MAXLEN)", status); } /* Scaled values of ntslice and maximum length */ astMapPut0I( filemap, "NTSLICE", ntslice, NULL ); /* Work out the padding needed for this file including downsampling. */ if( keymap ) { thispad = smf_get_padding( keymap, 0, data->hdr, VAL__BADD, status ); if( thispad > maxpad ) maxpad = thispad; } else { thispad = 0; } astMapPut0I( filemap, "PADDING", thispad, NULL ); /* Update maxflen */ if( ntslice > maxflen ) { maxflen = ntslice; } /* Store OBSID or OBSIDSS depending on whether we are grouping by wavelength */ if (grouping) { astMapPut0C( filemap, "OBSID", data->hdr->obsidss, NULL ); } else { char obsid[81]; smf_getobsidss( data->hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status ); astMapPut0C( filemap, "OBSID", obsid, NULL ); } } /* Store the file index in another keymap indexed by subarray */ if ( !astMapGet0A( filemap, "GRPINDICES", &indexmap ) ) { indexmap = astKeyMap( "SortBy=KeyUp" ); astMapPut0A( filemap, "GRPINDICES", indexmap, NULL ); } astMapPut0I( indexmap, subarray, i, NULL ); /* Need to track the largest number of related subarrays in a single slot */ nrelated = astMapSize( indexmap ); if (nrelated > maxrelated) maxrelated = nrelated; /* Free resources */ filemap = astAnnul( filemap ); indexmap = astAnnul( indexmap ); smf_close_file( NULL, &data, status ); } /* We now know how many groups there are */ ngroups = astMapSize( grouped ); /* Sort out chunking. The items are sorted by date and then by wavelength. We define a continuous chunk if it has the same OBSID, the same SEQCOUNT and NSUBSCAN increments by one from the previous entry. Also count number of related items in each slot. */ if (*status == SAI__OK) { typedef struct { /* somewhere to store the values easily */ char obsid[81]; char related[81]; /* for concatenated subarrays */ int nsubscan; int seqcount; } smfCompareSeq; smfCompareSeq current; smfCompareSeq previous; dim_t totlen = 0; size_t thischunk; /* Get the chunk flags and also store the size of the chunk */ chunk = astCalloc( ngroups, sizeof(*chunk) ); chunklen = astCalloc( ngroups, sizeof(*chunklen) ); piecelen = astCalloc( ngroups, sizeof(*piecelen) ); thischunk = 0; /* The current chunk */ for (i=0; i<ngroups; i++) { AstKeyMap * thismap = NULL; AstKeyMap * grpindices = NULL; const char * tempstr = NULL; int thistlen = 0; size_t nsubarrays = 0; /* Get the keymap entry for this slot */ astMapGet0A( grouped, astMapKey(grouped, i), &thismap ); /* Get info for length limits */ astMapGet0I( thismap, "NTSLICE", &thistlen ); piecelen[i] = thistlen; if (isFFT) { /* Never concatenate FFT data */ thismap = astAnnul(thismap); chunk[i] = i; chunklen[i] = thistlen; continue; } /* Get indices information and retrieve the sub-instrument names in sort order to concatenate for comparison. We only store in a continuous chunk if we have the same subarrays for the whole chunk. */ astMapGet0A( thismap, "GRPINDICES", &grpindices ); nsubarrays = astMapSize( grpindices ); (current.related)[0] = '\0'; for (j = 0; j < nsubarrays; j++ ) { one_strlcat( current.related, astMapKey(grpindices, j), sizeof(current.related), status ); } grpindices = astAnnul( grpindices ); /* Fill in the current struct */ astMapGet0I( thismap, "SEQCOUNT", &(current.seqcount) ); astMapGet0I( thismap, "NSUBSCAN", &(current.nsubscan) ); astMapGet0C( thismap, "OBSID", &tempstr ); one_strlcpy( current.obsid, tempstr, sizeof(current.obsid), status ); /* First chunk is special, else compare */ if (i == 0) { totlen = thistlen; } else { if ( ( current.seqcount == previous.seqcount ) && ( current.nsubscan - previous.nsubscan == 1 ) && ( strcmp( current.obsid, previous.obsid ) == 0 ) && ( strcmp( current.related, previous.related ) == 0 ) ) { /* continuous - check length */ totlen += thistlen; if ( maxlen && totlen > maxlen ) { thischunk++; totlen = thistlen; /* reset length */ } else { /* Continuous */ } } else { /* discontinuity */ thischunk++; totlen = thistlen; /* Update length of current chunk */ } } chunklen[thischunk] = totlen; chunk[i] = thischunk; memcpy( &previous, ¤t, sizeof(current) ); thismap = astAnnul( thismap ); } } /* Decide if we are keeping a chunk by looking at the length. */ maxconcat = 0; nkeep = 0; keepchunk = astMalloc( ngroups*sizeof(*keepchunk) ); for (i=0; i<ngroups; i++) { size_t thischunk; thischunk = chunk[i]; if ( chunklen[thischunk] < SMF__MINCHUNKSAMP ) { /* Warning message */ msgSeti("LEN",chunklen[thischunk]); msgSeti("MIN",SMF__MINCHUNKSAMP); msgOut( " ", "SMF_GRP_RELATED: ignoring short chunk (^LEN<^MIN)", status); keepchunk[i] = 0; } else { keepchunk[i] = 1; if (maxconcat < chunklen[thischunk]) maxconcat = chunklen[thischunk]; nkeep++; } } /* If no useful chunks generate an error */ if( (*status==SAI__OK) && (!nkeep) ) { *status = SAI__ERROR; errRep( "", FUNC_NAME ": No useful chunks.", status ); goto CLEANUP; } /* Allocate a subgroup array of the right size and fill it. They keymap is sorted by date (and wavelength) so we can always index into it by using indices from the subgroup. */ subgroups = astCalloc( nkeep, sizeof(*subgroups) ); new_chunk = astCalloc( nkeep, sizeof(*new_chunk) ); new_tlen = astCalloc( nkeep, sizeof(*new_tlen) ); currentindex = 0; for (i=0;i<ngroups;i++) { AstKeyMap * thismap = NULL; AstKeyMap * grpindices = NULL; size_t nsubarrays = 0; size_t *indices = astCalloc( maxrelated, sizeof(*indices) ); /* skip if we are dropping this chunk */ if (!keepchunk[i]) continue; /* Get the keymap entry for this slot */ astMapGet0A( grouped, astMapKey(grouped, i), &thismap ); /* Get the indices keymap */ astMapGet0A( thismap, "GRPINDICES", &grpindices ); nsubarrays = astMapSize( grpindices ); for (j=0; j<nsubarrays; j++) { int myindex; astMapGet0I( grpindices, astMapKey(grpindices, j), &myindex ); indices[j] = myindex; } grpindices = astAnnul( grpindices ); thismap = astAnnul( thismap ); subgroups[currentindex] = indices; new_chunk[currentindex] = chunk[i]; new_tlen[currentindex] = piecelen[i]; currentindex++; } /* Create the smfGroup */ *group = smf_construct_smfGroup( igrp, subgroups, new_chunk, new_tlen, nkeep, maxrelated, 0, status ); /* Return maxfilelen if requested */ if( maxfilelen ) { *maxfilelen = maxflen; } /* Return maxconcatlen if requested */ if( maxconcatlen ) { *maxconcatlen = maxconcat; } /* Create a base group for output files if required */ /* Create a base group of filenames */ if (*status == SAI__OK && basegrp ) { *basegrp = smf_grp_new( (*group)->grp, "Base Group", status ); /* Loop over time chunks */ for( i=0; (*status==SAI__OK)&&(i<(*group)->ngroups); i++ ) { size_t idx; /* Check for new continuous chunk */ if( i==0 || ( (*group)->chunk[i] != (*group)->chunk[i-1]) ) { /* Loop over subarray */ for( idx=0; idx<(*group)->nrelated; idx++ ) { size_t grpindex = (*group)->subgroups[i][idx]; if ( grpindex > 0 ) { ndgCpsup( (*group)->grp, grpindex, *basegrp, status ); } } } } } CLEANUP: keepchunk = astFree( keepchunk ); chunk = astFree( chunk ); chunklen = astFree( chunklen ); piecelen = astFree( piecelen ); grouped = astAnnul( grouped ); if( *status != SAI__OK ) { /* free the group */ if (basegrp && *basegrp) grpDelet( basegrp, status ); if (group && *group) { smf_close_smfGroup( group, status ); } else { /* have to clean up manually */ new_chunk = astFree( new_chunk ); new_tlen = astFree( new_tlen ); if( subgroups ) { size_t isub; for( isub=0; isub<nkeep; isub++ ) { subgroups[isub] = astFree( subgroups[isub] ); } subgroups = astFree( subgroups ); } } } /* Return the maximum padding if required. */ if( pad ) *pad = maxpad; }