示例#1
0
AstKeyMap *smf_subinst_keymap( smf_subinst_t subinst, const smfData * indata,
                               const Grp * igrp, size_t idx, int * status ) {

  const smfHead * hdr = NULL;        /* Header of file to be examined */
  size_t i;
  smfData * sub_data = NULL;         /* File to be examined */
  AstKeyMap * sub_instruments;       /* Keymap to be filled */

  if (*status != SAI__OK) return NULL;

  if (subinst == SMF__SUBINST_NONE && !indata && !igrp) {
    *status = SAI__ERROR;
    errRep( "", "Must supply either a subinst, a smfData or a Grp"
            " (possible programming error)", status );
    return NULL;
  }

  /* Create new keymap */
  sub_instruments = astKeyMap( " " );

  /* prefill with the list of known sub-instruments. */
  for (i = 0; i < SMF__SUBINST_NSUBINST; i++ ) {
    const char * substr = smf_subinst_str( i, status );
    if (substr) astMapPut0I( sub_instruments, substr, 0, NULL );
  }

  /* If the current sub-instrument has not been supplied, get it from the file.
     Use indata in preference to the group */
  if( subinst == SMF__SUBINST_NONE ) {
    if (indata) {
      hdr = indata->hdr;
    } else {
      smf_open_file( igrp, idx, "READ", SMF__NOCREATE_DATA, &sub_data, status );
      if (sub_data) {
        hdr = sub_data->hdr;
      }
    }
    if (hdr) subinst = smf_calc_subinst( hdr, status );
  }

  /* flag this as being the relevant sub-instrument */
  if (subinst != SMF__SUBINST_NSUBINST ) {
    const char * substr = smf_subinst_str( subinst, status );
    if (substr) {
      astMapPut0I( sub_instruments, substr, 1, NULL );
    }
  }

  if (sub_data) smf_close_file( &sub_data, status );

  /* Free the keymap if we have bad status */
  if (*status != SAI__OK && sub_instruments) {
    sub_instruments = astAnnul( sub_instruments );
  }

  return sub_instruments;
}
示例#2
0
void smf_grp_related( const Grp *igrp, const size_t grpsize,
                      const int grouping, const int checksubinst,
                      double maxlen_s, double *srate_maxlen,
                      AstKeyMap *keymap, dim_t *maxconcatlen,
                      dim_t *maxfilelen, smfGroup **group,
                      Grp **basegrp, dim_t *pad, int *status ) {

  /* Local variables */
  size_t *chunk=NULL;         /* Array of flags for continuous chunks */
  dim_t * chunklen = NULL;    /* Length of continuous chunk */
  size_t currentindex = 0;    /* Counter */
  char cwave[10];             /* String containing wavelength */
  smfData *data = NULL;       /* Current smfData */
  double downsampscale=0;     /* Angular scale downsampling size */
  double downsampfreq=0;      /* Target downsampling frequency */
  AstKeyMap * grouped = NULL; /* Primary AstKeyMap for grouping */
  size_t i;                   /* Loop counter for index into Grp */
  int isFFT=0;                /* Set if data are 4d FFT */
  size_t j;                   /* Loop counter */
  int *keepchunk=NULL;        /* Flag for chunks that will be kept */
  dim_t maxconcat=0;          /* Longest continuous chunk length */
  dim_t maxflen=0;            /* Max file length in time steps */
  dim_t maxlen=0;             /* Maximum concat length in samples */
  int maxlen_scaled=0;        /* Set once maxlen has been scaled, if needed */
  dim_t maxpad=0;             /* Maximum padding neeed for any input file */
  size_t maxrelated = 0;      /* Keep track of max number of related items */
  size_t *new_chunk=NULL;     /* keeper chunks associated with subgroups */
  dim_t *new_tlen=NULL;       /* tlens for new_subgroup */
  size_t ngroups = 0;         /* Counter for subgroups to be stored */
  size_t nkeep = 0;           /* Number of chunks to keep */
  dim_t * piecelen = NULL;    /* Length of single file */
  smf_subinst_t refsubinst;   /* Subinst of first file */
  size_t **subgroups = NULL;  /* Array containing index arrays to parent Grp */
  smf_subinst_t subinst;      /* Subinst of current file */

  if ( *status != SAI__OK ) return;

  if( maxlen_s < 0 ) {
    *status = SAI__ERROR;
    errRep( "", FUNC_NAME ": maxlen_s cannot be < 0!", status );
    return;
  }

  /* Get downsampling parameters */

  if( keymap ) {
    smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
                      NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
                      NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
                      NULL, NULL, NULL, NULL, &downsampscale, &downsampfreq,
                      NULL, NULL, NULL, NULL, status );

    if( downsampscale && downsampfreq ) {
      *status = SAI__ERROR;
      errRep( "", FUNC_NAME ": both downsampscale and downsampfreq are set",
              status );
      return;
    }
  }

  /* Initialize refcwave */
  refsubinst = SMF__SUBINST_NONE;

  /* Loop over files in input Grp: remember Grps are indexed from 1 */
  grouped = astKeyMap( "SortBy=KeyUp" );
  for (i=1; i<=grpsize; i++) {
    char newkey[128];
    char dateobs[81];
    char subarray[10];
    size_t nrelated = 0;
    AstKeyMap * filemap = NULL;
    AstKeyMap * indexmap = NULL;

    /* First step: open file and harvest metadata */
    smf_open_file( NULL, igrp, i, "READ", SMF__NOCREATE_DATA, &data, status );
    if (*status != SAI__OK) break;

    if( i==1 ) {
      isFFT = smf_isfft( data, NULL, NULL, NULL, NULL, NULL, status );
    } else if( smf_isfft(data, NULL, NULL, NULL, NULL, NULL, status) != isFFT ){
      *status = SAI__ERROR;
      errRep( "", FUNC_NAME
              ": mixture of time-series and FFT data encountered!",
              status );
      break;
    }

    /* If maxlen has not been set, do it here */
    if( !maxlen && maxlen_s && data->hdr->steptime) {
      maxlen = (dim_t) (maxlen_s / data->hdr->steptime );
    }

    /* Return srate_maxlen if requested: may want to know this number
       even if maxlen_s is not set. Only calculate once, although it
       gets overwritten once later if down-sampling. */

    if( (i==1) && srate_maxlen && data->hdr->steptime ) {
      *srate_maxlen = 1. / (double) data->hdr->steptime;
    }


    /* If requested check to see if we are mixing wavelengths */
    if( checksubinst ) {
      if( refsubinst == SMF__SUBINST_NONE ) {
        refsubinst = smf_calc_subinst( data->hdr, status );
      }

      subinst = smf_calc_subinst( data->hdr, status );

      if( subinst != refsubinst ) {
        const char *refsubstr = smf_subinst_str( refsubinst, status );
        const char *substr = smf_subinst_str( subinst, status );

        *status = SAI__ERROR;
        smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
        msgSetc( "REFSUB", refsubstr );
        msgSetc( "SUB", substr );
        errRep( "", FUNC_NAME
                ": ^FILE uses sub-instrument ^SUB which doesn't match "
                "reference ^REFSUB", status );
      }
    }

    /* Want to form a key that will be unique for a particular subscan
       We know that DATE-OBS will be set for SCUBA-2 files and be the same
       for a single set. Prefix by wavelength if we are grouping by wavelength.
     */
    newkey[0] = '\0';

    smf_find_subarray( data->hdr, subarray, sizeof(subarray), NULL, status );

    if( grouping == 1 ) {
      /* Group different wavelengths separately */
      smf_fits_getS( data->hdr, "WAVELEN", cwave, sizeof(cwave), status);
      one_strlcat( newkey, cwave, sizeof(newkey), status );
      one_strlcat( newkey, "_", sizeof(newkey), status );
    }

    if( grouping == 2 ) {
      /* Group different subarrays separately */
      one_strlcat( newkey, subarray, sizeof(newkey), status );
    }

    smf_fits_getS( data->hdr, "DATE-OBS", dateobs, sizeof(dateobs), status );
    one_strlcat( newkey, dateobs, sizeof(newkey), status );

    /* Include the dimentionality of the time series in the primary key
       so that we do not end up doing something confusing like relating
       a truncated file with a full length file */
    if (*status == SAI__OK) {
      dim_t dims[3];
      char formatted[32];
      smf_get_dims( data, &dims[0], &dims[1], NULL, &dims[2], NULL, NULL, NULL,
                    status );
      sprintf(formatted, "_%" DIM_T_FMT "_%" DIM_T_FMT "_%" DIM_T_FMT, dims[0], dims[1], dims[2]);
      one_strlcat( newkey, formatted, sizeof(newkey), status );
    }

    /* May want to read the dimensionality of the file outside of loop
       so that we can compare values when storing in the keymap */

    /* Now we want to create a keymap based on this key */
    if (!astMapGet0A( grouped, newkey, &filemap ) ) {
      int itemp = 0;
      double steptime = data->hdr->steptime;
      dim_t ntslice = 0;
      dim_t thispad;              /* Padding neeed for current input file */

      filemap = astKeyMap( " " );
      astMapPut0A( grouped, newkey, filemap, NULL );

      /* Fill up filemap with general information on this file */
      smf_find_seqcount( data->hdr, &itemp, status );
      astMapPut0I( filemap, "SEQCOUNT", itemp, NULL );

      smf_fits_getI( data->hdr, "NSUBSCAN", &itemp, status );
      astMapPut0I( filemap, "NSUBSCAN", itemp, NULL );

      /* Number of time slices */
      smf_get_dims( data, NULL, NULL, NULL, &ntslice, NULL, NULL, NULL,
                    status );

      /* Find length of down-sampled data, new steptime and maxlen */
      if( (downsampscale || downsampfreq) && data->hdr && (*status==SAI__OK) ) {
        double scalelen;

        if( downsampscale ) {
          if( data->hdr->scanvel != VAL__BADD ) {
             double oldscale = steptime * data->hdr->scanvel;
             scalelen = oldscale / downsampscale;
          } else if( *status == SAI__OK ) {
             *status = SAI__ERROR;
            scalelen = VAL__BADD;
            smf_smfFile_msg( data->file, "FILE", 1, "" );
            errRep( "", FUNC_NAME ": can't resample ^FILE because it has "
                    "unknown scan velocity", status );
          }
        } else {
          if( steptime ) {
            double oldsampfreq = 1./steptime;
            scalelen = downsampfreq / oldsampfreq;
          } else {
            *status = SAI__ERROR;
            scalelen = VAL__BADD;
            smf_smfFile_msg( data->file, "FILE", 1, "" );
            errRep( "", FUNC_NAME ": can't resample ^FILE because it has "
                    "unknown sample rate", status );
          }
        }

        /* only down-sample if it will be a reasonable factor */
        if( (*status==SAI__OK) && (scalelen <= SMF__DOWNSAMPLIMIT) ) {
          smf_smfFile_msg(data->file, "FILE", 1, "" );
          msgOutiff( MSG__VERB, "", FUNC_NAME
                     ": will down-sample file ^FILE from %5.1lf Hz to "
                     "%5.1lf Hz", status, (1./steptime), (scalelen/steptime) );

          ntslice = round(ntslice * scalelen);

          /* If maxlen has been requested, and we have not already worked
             out a scaled version (just uses the sample rates for the first
             file... should be close enough -- the alternative is a 2-pass
             system). */

          if( !maxlen_scaled ) {
            maxlen = round(maxlen*scalelen);
            maxlen_scaled = 1;
            msgOutiff( MSG__VERB, "", FUNC_NAME
                       ": requested maxlen %g seconds = %" DIM_T_FMT " down-sampled "
                       "time-slices", status, maxlen_s, maxlen );

            /* Return updated srate_maxlen for down-sampling if requested */
            if( srate_maxlen ) {
              *srate_maxlen = scalelen/steptime;
            }
          }
        }
      }

      /* Check that an individual file is too long (we assume related
         files are all the same) */
      if( maxlen && (ntslice > maxlen) && *status == SAI__OK) {
        *status = SAI__ERROR;
        msgSeti("NTSLICE",ntslice);
        msgSeti("MAXLEN",maxlen);
        smf_smfFile_msg( data->file, "FILE", 1, "" );
        errRep(FUNC_NAME,
               "Number of time steps in file ^FILE time exceeds maximum "
               "(^NTSLICE>^MAXLEN)", status);
      }

      /* Scaled values of ntslice and maximum length */
      astMapPut0I( filemap, "NTSLICE", ntslice, NULL );

      /* Work out the padding needed for this file including downsampling. */
      if( keymap ) {
        thispad = smf_get_padding( keymap, 0, data->hdr, VAL__BADD, status );
        if( thispad > maxpad ) maxpad = thispad;
      } else {
        thispad = 0;
      }
      astMapPut0I( filemap, "PADDING", thispad, NULL );

      /* Update maxflen */
      if( ntslice > maxflen ) {
        maxflen = ntslice;
      }

      /* Store OBSID or OBSIDSS depending on whether we are grouping by wavelength */
      if (grouping) {
        astMapPut0C( filemap, "OBSID", data->hdr->obsidss, NULL );
      } else {
        char obsid[81];
        smf_getobsidss( data->hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status );
        astMapPut0C( filemap, "OBSID", obsid, NULL );
      }
    }

    /* Store the file index in another keymap indexed by subarray */
    if ( !astMapGet0A( filemap, "GRPINDICES", &indexmap ) ) {
      indexmap = astKeyMap( "SortBy=KeyUp" );
      astMapPut0A( filemap, "GRPINDICES", indexmap, NULL );
    }

    astMapPut0I( indexmap, subarray, i, NULL );

    /* Need to track the largest number of related subarrays in a single slot */
    nrelated = astMapSize( indexmap );
    if (nrelated > maxrelated) maxrelated = nrelated;

    /* Free resources */
    filemap = astAnnul( filemap );
    indexmap = astAnnul( indexmap );
    smf_close_file( NULL, &data, status );
  }

  /* We now know how many groups there are */
  ngroups = astMapSize( grouped );

  /* Sort out chunking. The items are sorted by date and then by wavelength.
     We define a continuous chunk if it has the same OBSID, the same SEQCOUNT
     and NSUBSCAN increments by one from the previous entry.

     Also count number of related items in each slot.
   */
  if (*status == SAI__OK) {
    typedef struct { /* somewhere to store the values easily */
      char obsid[81];
      char related[81];  /* for concatenated subarrays */
      int nsubscan;
      int seqcount;
    } smfCompareSeq;
    smfCompareSeq current;
    smfCompareSeq previous;
    dim_t totlen = 0;
    size_t thischunk;

    /* Get the chunk flags and also store the size of the chunk */
    chunk = astCalloc( ngroups, sizeof(*chunk) );
    chunklen = astCalloc( ngroups, sizeof(*chunklen) );
    piecelen = astCalloc( ngroups, sizeof(*piecelen) );

    thischunk = 0;  /* The current chunk */
    for (i=0; i<ngroups; i++) {
      AstKeyMap * thismap = NULL;
      AstKeyMap * grpindices = NULL;
      const char * tempstr = NULL;
      int thistlen = 0;
      size_t nsubarrays = 0;

      /* Get the keymap entry for this slot */
      astMapGet0A( grouped, astMapKey(grouped, i), &thismap );

      /* Get info for length limits */
      astMapGet0I( thismap, "NTSLICE", &thistlen );
      piecelen[i] = thistlen;

      if (isFFT) {
        /* Never concatenate FFT data */
        thismap = astAnnul(thismap);
        chunk[i] = i;
        chunklen[i] = thistlen;
        continue;
      }

      /* Get indices information and retrieve the sub-instrument names
         in sort order to concatenate for comparison. We only store in
         a continuous chunk if we have the same subarrays for the whole
         chunk. */
      astMapGet0A( thismap, "GRPINDICES", &grpindices );
      nsubarrays = astMapSize( grpindices );
      (current.related)[0] = '\0';
      for (j = 0; j < nsubarrays; j++ ) {
        one_strlcat( current.related, astMapKey(grpindices, j), sizeof(current.related), status );
      }
      grpindices = astAnnul( grpindices );

      /* Fill in the current struct */
      astMapGet0I( thismap, "SEQCOUNT", &(current.seqcount) );
      astMapGet0I( thismap, "NSUBSCAN", &(current.nsubscan) );
      astMapGet0C( thismap, "OBSID", &tempstr );
      one_strlcpy( current.obsid, tempstr, sizeof(current.obsid), status );

      /* First chunk is special, else compare */
      if (i == 0) {
        totlen = thistlen;
      } else {
        if (  ( current.seqcount == previous.seqcount  ) &&
              ( current.nsubscan - previous.nsubscan == 1 ) &&
              ( strcmp( current.obsid, previous.obsid ) == 0 ) &&
              ( strcmp( current.related, previous.related ) == 0 ) ) {
          /* continuous - check length */
          totlen += thistlen;
          if ( maxlen && totlen > maxlen ) {
            thischunk++;
            totlen = thistlen; /* reset length */
          } else {
            /* Continuous */
          }
        } else {
          /* discontinuity */
          thischunk++;
          totlen = thistlen;  /* Update length of current chunk */
        }
      }

      chunklen[thischunk] = totlen;
      chunk[i] = thischunk;
      memcpy( &previous, &current, sizeof(current) );

      thismap = astAnnul( thismap );
    }
  }

  /* Decide if we are keeping a chunk by looking at the length. */
  maxconcat = 0;
  nkeep = 0;
  keepchunk = astMalloc( ngroups*sizeof(*keepchunk) );
  for (i=0; i<ngroups; i++) {
    size_t thischunk;

    thischunk = chunk[i];
    if ( chunklen[thischunk] < SMF__MINCHUNKSAMP ) {
      /* Warning message */
      msgSeti("LEN",chunklen[thischunk]);
      msgSeti("MIN",SMF__MINCHUNKSAMP);
      msgOut( " ", "SMF_GRP_RELATED: ignoring short chunk (^LEN<^MIN)",
              status);
      keepchunk[i] = 0;
    } else {
      keepchunk[i] = 1;
      if (maxconcat < chunklen[thischunk]) maxconcat = chunklen[thischunk];
      nkeep++;
    }

  }

  /* If no useful chunks generate an error */
  if( (*status==SAI__OK) && (!nkeep) ) {
    *status = SAI__ERROR;
    errRep( "", FUNC_NAME ": No useful chunks.", status );
    goto CLEANUP;
  }

  /* Allocate a subgroup array of the right size and fill it. They keymap
     is sorted by date (and wavelength) so we can always index into it by using
     indices from the subgroup. */
  subgroups = astCalloc( nkeep, sizeof(*subgroups) );
  new_chunk = astCalloc( nkeep, sizeof(*new_chunk) );
  new_tlen  = astCalloc( nkeep, sizeof(*new_tlen) );

  currentindex = 0;
  for (i=0;i<ngroups;i++) {
    AstKeyMap * thismap = NULL;
    AstKeyMap * grpindices = NULL;
    size_t nsubarrays = 0;
    size_t *indices = astCalloc( maxrelated, sizeof(*indices) );

    /* skip if we are dropping this chunk */
    if (!keepchunk[i]) continue;

    /* Get the keymap entry for this slot */
    astMapGet0A( grouped, astMapKey(grouped, i), &thismap );

    /* Get the indices keymap */
    astMapGet0A( thismap, "GRPINDICES", &grpindices );
    nsubarrays = astMapSize( grpindices );
    for (j=0; j<nsubarrays; j++) {
      int myindex;
      astMapGet0I( grpindices, astMapKey(grpindices, j), &myindex );
      indices[j] = myindex;
    }
    grpindices = astAnnul( grpindices );
    thismap = astAnnul( thismap );

    subgroups[currentindex] = indices;
    new_chunk[currentindex] = chunk[i];
    new_tlen[currentindex]  = piecelen[i];
    currentindex++;

  }

  /* Create the smfGroup */
  *group = smf_construct_smfGroup( igrp, subgroups, new_chunk, new_tlen,
                                   nkeep, maxrelated, 0, status );

  /* Return maxfilelen if requested */
  if( maxfilelen ) {
    *maxfilelen = maxflen;
  }

  /* Return maxconcatlen if requested */
  if( maxconcatlen ) {
    *maxconcatlen = maxconcat;
  }

  /* Create a base group for output files if required */
  /* Create a base group of filenames */
  if (*status == SAI__OK && basegrp ) {
    *basegrp = smf_grp_new( (*group)->grp, "Base Group", status );

    /* Loop over time chunks */
    for( i=0; (*status==SAI__OK)&&(i<(*group)->ngroups); i++ ) {
      size_t idx;
      /* Check for new continuous chunk */
      if( i==0 || ( (*group)->chunk[i] != (*group)->chunk[i-1]) ) {
        /* Loop over subarray */
        for( idx=0; idx<(*group)->nrelated; idx++ ) {
          size_t grpindex = (*group)->subgroups[i][idx];
          if ( grpindex > 0 ) {
            ndgCpsup( (*group)->grp, grpindex, *basegrp, status );
          }
        }
      }
    }
  }

 CLEANUP:
  keepchunk = astFree( keepchunk );
  chunk = astFree( chunk );
  chunklen = astFree( chunklen );
  piecelen = astFree( piecelen );
  grouped = astAnnul( grouped );

  if( *status != SAI__OK ) {
    /* free the group */
    if (basegrp && *basegrp) grpDelet( basegrp, status );
    if (group && *group) {
      smf_close_smfGroup( group, status );
    } else {
      /* have to clean up manually */
      new_chunk = astFree( new_chunk );
      new_tlen = astFree( new_tlen );
      if( subgroups ) {
        size_t isub;
        for( isub=0; isub<nkeep; isub++ ) {
          subgroups[isub] = astFree( subgroups[isub] );
        }
        subgroups = astFree( subgroups );
      }
    }
  }

  /* Return the maximum padding if required. */
  if( pad ) *pad = maxpad;
}