Ejemplo n.º 1
0
/***************************************************************************
 * msr_pack:
 *
 * Pack data into SEED data records.  Using the record header values
 * in the MSRecord as a template the common header fields are packed
 * into the record header, blockettes in the blockettes chain are
 * packed and data samples are packed in the encoding format indicated
 * by the MSRecord->encoding field.  A Blockette 1000 will be added if
 * one is not present.
 *
 * The MSRecord->datasamples array and MSRecord->numsamples value will
 * not be changed by this routine.  It is the responsibility of the
 * calling routine to adjust the data buffer if desired.
 *
 * As each record is filled and finished they are passed to
 * record_handler which expects 1) a char * to the record, 2) the
 * length of the record and 3) a pointer supplied by the original
 * caller containing optional private data (handlerdata).  It is the
 * responsibility of record_handler to process the record, the memory
 * will be re-used or freed when record_handler returns.
 *
 * If the flush flag != 0 all of the data will be packed into data
 * records even though the last one will probably not be filled.
 *
 * Default values are: data record & quality indicator = 'D', record
 * length = 4096, encoding = 11 (Steim2) and byteorder = 1 (MSBF).
 * The defaults are triggered when the the msr->dataquality is 0 or
 * msr->reclen, msr->encoding and msr->byteorder are -1 respectively.
 *
 * Returns the number of records created on success and -1 on error.
 ***************************************************************************/
int
msr_pack ( MSRecord * msr, void (*record_handler) (char *, int, void *),
	   void *handlerdata, int64_t *packedsamples, flag flush, flag verbose )
{
  uint16_t *HPnumsamples;
  uint16_t *HPdataoffset;
  struct blkt_1001_s *HPblkt1001 = NULL;
  
  char *rawrec;
  char *envvariable;
  char srcname[50];
  
  flag headerswapflag = 0;
  flag dataswapflag = 0;
  flag packret;
  
  int samplesize;
  int headerlen;
  int dataoffset;
  int maxdatabytes;
  int maxsamples;
  int recordcnt = 0;
  int packsamples, packoffset;
  int64_t totalpackedsamples;
  hptime_t segstarttime;
  
  if ( ! msr )
    return -1;
  
  if ( ! record_handler )
    {
      ms_log (2, "msr_pack(): record_handler() function pointer not set!\n");
      return -1;
    }

  /* Allocate stream processing state space if needed */
  if ( ! msr->ststate )
    {
      msr->ststate = (StreamState *) malloc (sizeof(StreamState));
      if ( ! msr->ststate )
        {
          ms_log (2, "msr_pack(): Could not allocate memory for StreamState\n");
          return -1;
        }
      memset (msr->ststate, 0, sizeof(StreamState));
    }

  /* Generate source name for MSRecord */
  if ( msr_srcname (msr, srcname, 1) == NULL )
    {
      ms_log (2, "msr_unpack_data(): Cannot generate srcname\n");
      return MS_GENERROR;
    }
  
  /* Set shared srcname pointer to source name */
  PACK_SRCNAME = &srcname[0];
  
  /* Track original segment start time for new start time calculation */
  segstarttime = msr->starttime;

  /* Read possible environmental variables that force byteorder */
  if ( packheaderbyteorder == -2 )
    {
      if ( (envvariable = getenv("PACK_HEADER_BYTEORDER")) )
	{
	  if ( *envvariable != '0' && *envvariable != '1' )
	    {
	      ms_log (2, "Environment variable PACK_HEADER_BYTEORDER must be set to '0' or '1'\n");
	      return -1;
	    }
	  else if ( *envvariable == '0' )
	    {
	      packheaderbyteorder = 0;
	      if ( verbose > 2 )
		ms_log (1, "PACK_HEADER_BYTEORDER=0, packing little-endian header\n");
	    }
	  else
	    {
	      packheaderbyteorder = 1;
	      if ( verbose > 2 )
		ms_log (1, "PACK_HEADER_BYTEORDER=1, packing big-endian header\n");
	    }
	}
      else
	{
	  packheaderbyteorder = -1;
	}
    }
  if ( packdatabyteorder == -2 )
    {
      if ( (envvariable = getenv("PACK_DATA_BYTEORDER")) )
	{
	  if ( *envvariable != '0' && *envvariable != '1' )
	    {
	      ms_log (2, "Environment variable PACK_DATA_BYTEORDER must be set to '0' or '1'\n");
	      return -1;
	    }
	  else if ( *envvariable == '0' )
	    {
	      packdatabyteorder = 0;
	      if ( verbose > 2 )
		ms_log (1, "PACK_DATA_BYTEORDER=0, packing little-endian data samples\n");
	    }
	  else
	    {
	      packdatabyteorder = 1;
	      if ( verbose > 2 )
		ms_log (1, "PACK_DATA_BYTEORDER=1, packing big-endian data samples\n");
	    }
	}
      else
	{
	  packdatabyteorder = -1;
	}
    }

  /* Set default indicator, record length, byte order and encoding if needed */
  if ( msr->dataquality == 0 ) msr->dataquality = 'D';
  if ( msr->reclen == -1 ) msr->reclen = 4096;
  if ( msr->byteorder == -1 )  msr->byteorder = 1;
  if ( msr->encoding == -1 ) msr->encoding = DE_STEIM2;
  
  /* Cleanup/reset sequence number */
  if ( msr->sequence_number <= 0 || msr->sequence_number > 999999)
    msr->sequence_number = 1;
  
  if ( msr->reclen < MINRECLEN || msr->reclen > MAXRECLEN )
    {
      ms_log (2, "msr_pack(%s): Record length is out of range: %d\n",
	      PACK_SRCNAME, msr->reclen);
      return -1;
    }
  
  if ( msr->numsamples <= 0 )
    {
      ms_log (2, "msr_pack(%s): No samples to pack\n", PACK_SRCNAME);
      return -1;
    }
  
  samplesize = ms_samplesize (msr->sampletype);
  
  if ( ! samplesize )
    {
      ms_log (2, "msr_pack(%s): Unknown sample type '%c'\n",
	      PACK_SRCNAME, msr->sampletype);
      return -1;
    }
  
  /* Sanity check for msr/quality indicator */
  if ( ! MS_ISDATAINDICATOR(msr->dataquality) )
    {
      ms_log (2, "msr_pack(%s): Record header & quality indicator unrecognized: '%c'\n",
	      PACK_SRCNAME, msr->dataquality);
      ms_log (2, "msr_pack(%s): Packing failed.\n", PACK_SRCNAME);
      return -1;
    }
  
  /* Allocate space for data record */
  rawrec = (char *) malloc (msr->reclen);
  
  if ( rawrec == NULL )
    {
      ms_log (2, "msr_pack(%s): Cannot allocate memory\n", PACK_SRCNAME);
      return -1;
    }
  
  /* Set header pointers to known offsets into FSDH */
  HPnumsamples = (uint16_t *) (rawrec + 30);
  HPdataoffset = (uint16_t *) (rawrec + 44);
  
  /* Check to see if byte swapping is needed */
  if ( msr->byteorder != ms_bigendianhost() )
    headerswapflag = dataswapflag = 1;
  
  /* Check if byte order is forced */
  if ( packheaderbyteorder >= 0 )
    {
      headerswapflag = ( msr->byteorder != packheaderbyteorder ) ? 1 : 0;
    }
  
  if ( packdatabyteorder >= 0 )
    {
      dataswapflag = ( msr->byteorder != packdatabyteorder ) ? 1 : 0;
    }
  
  if ( verbose > 2 )
    {
      if ( headerswapflag && dataswapflag )
	ms_log (1, "%s: Byte swapping needed for packing of header and data samples\n", PACK_SRCNAME);
      else if ( headerswapflag )
	ms_log (1, "%s: Byte swapping needed for packing of header\n", PACK_SRCNAME);
      else if ( dataswapflag )
	ms_log (1, "%s: Byte swapping needed for packing of data samples\n", PACK_SRCNAME);
      else
	ms_log (1, "%s: Byte swapping NOT needed for packing\n", PACK_SRCNAME);
    }
  
  /* Add a blank 1000 Blockette if one is not present, the blockette values
     will be populated in msr_pack_header_raw()/msr_normalize_header() */
  if ( ! msr->Blkt1000 )
    {
      struct blkt_1000_s blkt1000;
      memset (&blkt1000, 0, sizeof (struct blkt_1000_s));
      
      if ( verbose > 2 )
	ms_log (1, "%s: Adding 1000 Blockette\n", PACK_SRCNAME);
      
      if ( ! msr_addblockette (msr, (char *) &blkt1000, sizeof(struct blkt_1000_s), 1000, 0) )
	{
	  ms_log (2, "msr_pack(%s): Error adding 1000 Blockette\n", PACK_SRCNAME);
	  return -1;
	}
    }
  
  headerlen = msr_pack_header_raw (msr, rawrec, msr->reclen, headerswapflag, 1, &HPblkt1001, verbose);
  
  if ( headerlen == -1 )
    {
      ms_log (2, "msr_pack(%s): Error packing header\n", PACK_SRCNAME);
      return -1;
    }
  
  /* Determine offset to encoded data */
  if ( msr->encoding == DE_STEIM1 || msr->encoding == DE_STEIM2 )
    {
      dataoffset = 64;
      while ( dataoffset < headerlen )
	dataoffset += 64;
      
      /* Zero memory between blockettes and data if any */
      memset (rawrec + headerlen, 0, dataoffset - headerlen);
    }
  else
    {
      dataoffset = headerlen;
    }
  
  *HPdataoffset = (uint16_t) dataoffset;
  if ( headerswapflag ) ms_gswap2 (HPdataoffset);
  
  /* Determine the max data bytes and sample count */
  maxdatabytes = msr->reclen - dataoffset;
  
  if ( msr->encoding == DE_STEIM1 )
    {
      maxsamples = (int) (maxdatabytes/64) * STEIM1_FRAME_MAX_SAMPLES;
    }
  else if ( msr->encoding == DE_STEIM2 )
    {
      maxsamples = (int) (maxdatabytes/64) * STEIM2_FRAME_MAX_SAMPLES;
    }
  else
    {
      maxsamples = maxdatabytes / samplesize;
    }
  
  /* Pack samples into records */
  *HPnumsamples = 0;
  totalpackedsamples = 0;
  if ( packedsamples ) *packedsamples = 0;
  packoffset = 0;
  
  while ( (msr->numsamples - totalpackedsamples) > maxsamples || flush )
    {
      packret = msr_pack_data (rawrec + dataoffset,
			       (char *) msr->datasamples + packoffset,
			       (int)(msr->numsamples - totalpackedsamples), maxdatabytes,
			       &packsamples, &msr->ststate->lastintsample, msr->ststate->comphistory,
			       msr->sampletype, msr->encoding, dataswapflag, verbose);
      
      if ( packret )
	{
	  ms_log (2, "msr_pack(%s): Error packing record\n", PACK_SRCNAME);
	  return -1;
	}
      
      packoffset += packsamples * samplesize;
      
      /* Update number of samples */
      *HPnumsamples = (uint16_t) packsamples;
      if ( headerswapflag ) ms_gswap2 (HPnumsamples);
      
      if ( verbose > 0 )
	ms_log (1, "%s: Packed %d samples\n", PACK_SRCNAME, packsamples);
      
      /* Send record to handler */
      record_handler (rawrec, msr->reclen, handlerdata);
      
      totalpackedsamples += packsamples;
      if ( packedsamples ) *packedsamples = totalpackedsamples;
      msr->ststate->packedsamples += packsamples;
      
      /* Update record header for next record */
      msr->sequence_number = ( msr->sequence_number >= 999999 ) ? 1 : msr->sequence_number + 1;
      if ( msr->samprate > 0 )
	msr->starttime = segstarttime + (hptime_t)(totalpackedsamples / msr->samprate * HPTMODULUS + 0.5);
      
      msr_update_header (msr, rawrec, headerswapflag, HPblkt1001, verbose);
      
      recordcnt++;
      msr->ststate->packedrecords++;
      
      /* Set compression history flag for subsequent records (Steim encodings) */
      if ( ! msr->ststate->comphistory )
        msr->ststate->comphistory = 1;
     
      if ( totalpackedsamples >= msr->numsamples )
	break;
    }
  
  if ( verbose > 2 )
    ms_log (1, "%s: Packed %d total samples\n", PACK_SRCNAME, totalpackedsamples);
  
  free (rawrec);
  
  return recordcnt;
} /* End of msr_pack() */
Ejemplo n.º 2
0
/***************************************************************************
 * ds_streamproc:
 *
 * Save MiniSEED records in a custom directory/file structure.  The
 * appropriate directories and files are created if nesecessary.  If
 * files already exist they are appended to.  If 'msr' is NULL then
 * ds_shutdown() will be called to close all open files and free all
 * associated memory.
 *
 * This version has been modified from others to add the add the suffix
 * integer supplied with ds_streamproc() to the defkey and file name.
 *
 * Returns 0 on success, -1 on error.
 ***************************************************************************/
extern int
ds_streamproc (DataStream *datastream, MSRecord *msr, long suffix, int verbose)
{
  DataStreamGroup *foundgroup = NULL;
  BTime stime;
  strlist *fnlist, *fnptr;
  char net[3], sta[6], loc[3], chan[4];
  char filename[400];
  char definition[400];
  char pathformat[600];
  char tstr[20];
  int fnlen = 0;
  
  /* Set Verbosity for ds_ functions */
  dsverbose = verbose;
  
  /* Special case for stream shutdown */
  if ( ! msr )
    {
      if ( dsverbose >= 1 )
        fprintf (stderr, "Closing archiving for: %s\n", datastream->path );
      
      ds_shutdown ( datastream );
      return 0;
    }
  
  if ( ! msr->fsdh )
    {
      fprintf (stderr, "ds_streamproc(): msr->fsdh must be available\n");
      return -1;
    }
  
  /* Build file path and name from datastream->path */
  filename[0] = '\0';
  definition[0] = '\0';
  snprintf (pathformat, sizeof(pathformat), "%s", datastream->path);
  strparse (pathformat, "/", &fnlist);

  fnptr = fnlist;

  /* Special case of an absolute path (first entry is empty) */
  if ( *fnptr->element == '\0' )
    {
      if ( fnptr->next != 0 )
	{
	  strncat (filename, "/", sizeof(filename));
	  fnptr = fnptr->next;
	}
      else
	{
	  fprintf (stderr, "ds_streamproc(): empty path format\n");
	  strparse (NULL, NULL, &fnlist);
	  return -1;
	}
    }
  
  /* Convert normalized starttime to BTime structure */
  if ( ms_hptime2btime (msr->starttime, &stime) )
    {
      fprintf (stderr, "ds_streamproc(): cannot convert start time to separate fields\n");
      strparse (NULL, NULL, &fnlist);
      return -1;
    }
  
  while ( fnptr != 0 )
    {
      int tdy;
      char *w, *p, def;

      p = fnptr->element;

      /* Special case of no file given */
      if ( *p == '\0' && fnptr->next == 0 )
	{
	  fprintf (stderr, "ds_streamproc(): no file name specified, only %s\n",
		   filename);
	  strparse (NULL, NULL, &fnlist);
	  return -1;
	}

      while ( (w = strpbrk (p, "%#")) != NULL )
	{
	  def = ( *w == '%' );
	  *w = '\0';
	  strncat (filename, p, (sizeof(filename) - fnlen));
	  fnlen = strlen (filename);

	  w += 1;

	  switch ( *w )
	    {
	    case 'n' :
	      ms_strncpclean (net, msr->fsdh->network, 2);
	      strncat (filename, net, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, net, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 's' :
	      ms_strncpclean (sta, msr->fsdh->station, 5);
	      strncat (filename, sta, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, sta, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'l' :
	      ms_strncpclean (loc, msr->fsdh->location, 2);
	      strncat (filename, loc, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, loc, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'c' :
	      ms_strncpclean (chan, msr->fsdh->channel, 3);
	      strncat (filename, chan, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, chan, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'Y' :
	      snprintf (tstr, sizeof(tstr), "%04d", (int) stime.year);
	      strncat (filename, tstr, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, tstr, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'y' :
	      tdy = (int) stime.year;
	      while ( tdy > 100 )
		{
		  tdy -= 100;
		}
	      snprintf (tstr, sizeof(tstr), "%02d", tdy);
	      strncat (filename, tstr, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, tstr, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'j' :
	      snprintf (tstr, sizeof(tstr), "%03d", (int) stime.day);
	      strncat (filename, tstr, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, tstr, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'H' :
	      snprintf (tstr, sizeof(tstr), "%02d", (int) stime.hour);
	      strncat (filename, tstr, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, tstr, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'M' :
	      snprintf (tstr, sizeof(tstr), "%02d", (int) stime.min);
	      strncat (filename, tstr, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, tstr, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'S' :
	      snprintf (tstr, sizeof(tstr), "%02d", (int) stime.sec);
	      strncat (filename, tstr, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, tstr, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'F' :
	      snprintf (tstr, sizeof(tstr), "%04d", (int) stime.fract);
	      strncat (filename, tstr, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, tstr, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'q' :
	      snprintf (tstr, sizeof(tstr), "%c", msr->dataquality);
	      strncat (filename, tstr, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, tstr, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'L' :
	      snprintf (tstr, sizeof(tstr), "%d", msr->reclen);
	      strncat (filename, tstr, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, tstr, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'r' :
	      snprintf (tstr, sizeof(tstr), "%ld", (long int) (msr->samprate+0.5));
	      strncat (filename, tstr, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, tstr, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case 'R' :
	      snprintf (tstr, sizeof(tstr), "%.6f", msr->samprate);
	      strncat (filename, tstr, (sizeof(filename) - fnlen));
	      if ( def ) strncat (definition, tstr, (sizeof(definition) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case '%' :
	      strncat (filename, "%", (sizeof(filename) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    case '#' :
	      strncat (filename, "#", (sizeof(filename) - fnlen));
	      fnlen = strlen (filename);
	      p = w + 1;
	      break;
	    default :
	      fprintf (stderr, "Unknown layout format code: '%c'\n", *w);
	      p = w;
	      break;
	    }
	}
      
      strncat (filename, p, (sizeof(filename) - fnlen));
      fnlen = strlen (filename);

      /* If not the last entry then it should be a directory */
      if ( fnptr->next != 0 )
	{
	  if ( access (filename, F_OK) )
	    {
	      if ( errno == ENOENT )
		{
		  if ( dsverbose >= 1 )
		    fprintf (stderr, "Creating directory: %s\n", filename);

		  if (mkdir
		      (filename, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH))
		    {
		      fprintf (stderr, "ds_streamproc: mkdir(%s) %s\n", filename, strerror (errno));
		      strparse (NULL, NULL, &fnlist);
		      return -1;
		    }
		}
	      else
		{
		  fprintf (stderr, "%s: access denied, %s\n", filename, strerror(errno));
		  strparse (NULL, NULL, &fnlist);
		  return -1;
		}
	    }

	  strncat (filename, "/", (sizeof(filename) - fnlen));
	  fnlen++;
	}

      fnptr = fnptr->next;
    }

  strparse (NULL, NULL, &fnlist);

  /* Add ".suffix" to filename and definition if suffix is not 0 */
  if ( suffix )
    {
      snprintf (tstr, sizeof(tstr), ".%ld", suffix);
      strncat (filename, tstr, (sizeof(filename) - fnlen));
      strncat (definition, tstr, (sizeof(definition) - fnlen));
      fnlen = strlen (filename);
    }

  /* Make sure the filename and definition are NULL terminated */
  *(filename + sizeof(filename) - 1) = '\0';
  *(definition + sizeof(definition) -1) = '\0';

  /* Check for previously used stream entry, otherwise create it */
  foundgroup = ds_getstream (datastream, msr, definition, filename);

  if (foundgroup != NULL)
    {
      /* Write binary data samples to approriate file */
      if ( msr->datasamples && msr->numsamples )
	{
	  if ( dsverbose >= 3 )
	    fprintf (stderr, "Writing binary data samples to data stream file %s\n", filename);
	  
	  if ( !write (foundgroup->filed, msr->datasamples, msr->numsamples * ms_samplesize(msr->sampletype)) )
	    {
	      fprintf (stderr, "ds_streamproc: failed to write binary data samples\n");
	      return -1;
	    }
	  else
	    {
	      foundgroup->modtime = time (NULL);	  
	    }
	}
      /* Write the data record to the appropriate file */ 
      else
	{
	  if ( dsverbose >= 3 )
	    fprintf (stderr, "Writing data record to data stream file %s\n", filename);
	  
	  if ( !write (foundgroup->filed, msr->record, msr->reclen) )
	    {
	      fprintf (stderr, "ds_streamproc: failed to write data record\n");
	      return -1;
	    }
	  else
	    {
	      foundgroup->modtime = time (NULL);	  
	    }
	}

      return 0;
    }
  
  return -1;
}  /* End of ds_streamproc() */
Ejemplo n.º 3
0
// Function that reads from a MiniSEED binary file from a char buffer and
// returns a LinkedIDList.
LinkedIDList *
readMSEEDBuffer (char *mseed, int buflen, Selections *selections, flag
                 unpack_data, int reclen, flag verbose, flag details,
                 int header_byteorder, long (*allocData) (int, char),
                 void (*diag_print) (char*), void (*log_print) (char*))
{
    int retcode = 0;
    int retval = 0;
    flag swapflag = 0;

    // current offset of mseed char pointer
    int offset = 0;

    // Unpack without reading the data first
    flag dataflag = 0;

    // the timing_qual of BLK 1001
    uint8_t timing_qual = 0xFF;

    // the calibration type, availability of BLK 300, 310, 320, 390, 395
    int8_t calibration_type = -1;

    // Init all the pointers to NULL. Most compilers should do this anyway.
    LinkedIDList * idListHead = NULL;
    LinkedIDList * idListCurrent = NULL;
    LinkedIDList * idListLast = NULL;
    MSRecord *msr = NULL;
    ContinuousSegment * segmentCurrent = NULL;
    hptime_t lastgap = 0;
    hptime_t hptimetol = 0;
    hptime_t nhptimetol = 0;
    long data_offset;
    LinkedRecordList *recordHead = NULL;
    LinkedRecordList *recordPrevious = NULL;
    LinkedRecordList *recordCurrent = NULL;
    int datasize;
    int record_count = 0;

    // A negative verbosity suppressed as much as possible.
    if (verbose < 0) {
        ms_loginit(&empty_print, NULL, &empty_print, NULL);
    }
    else {
        ms_loginit(log_print, "INFO: ", diag_print, "ERROR: ");
    }

    if (header_byteorder >= 0) {
        // Enforce little endian.
        if (header_byteorder == 0) {
            MS_UNPACKHEADERBYTEORDER(0);
        }
        // Enforce big endian.
        else {
            MS_UNPACKHEADERBYTEORDER(1);
        }
    }
    else {
        MS_UNPACKHEADERBYTEORDER(-1);
    }

    //
    // Read all records and save them in a linked list.
    //
    while (offset < buflen) {
        msr = msr_init(NULL);
        if ( msr == NULL ) {
            ms_log (2, "readMSEEDBuffer(): Error initializing msr\n");
            return NULL;
        }
        if (verbose > 1) {
            ms_log(0, "readMSEEDBuffer(): calling msr_parse with "
                      "mseed+offset=%d+%d, buflen=%d, reclen=%d, dataflag=%d, verbose=%d\n",
                      mseed, offset, buflen, reclen, dataflag, verbose);
        }

        // If the record length is given, make sure at least that amount of data is available.
        if (reclen != -1) {
            if (offset + reclen > buflen) {
                ms_log(1, "readMSEEDBuffer(): Last reclen exceeds buflen, skipping.\n");
                msr_free(&msr);
                break;
            }
        }
        // Otherwise assume the smallest possible record length and assure that enough
        // data is present.
        else {
            if (offset + 256 > buflen) {
                ms_log(1, "readMSEEDBuffer(): Last record only has %i byte(s) which "
                          "is not enough to constitute a full SEED record. Corrupt data? "
                          "Record will be skipped.\n", buflen - offset);
                msr_free(&msr);
                break;
            }
        }

        // Pass (buflen - offset) because msr_parse() expects only a single record. This
        // way libmseed can take care to not overstep bounds.
        retcode = msr_parse ( (mseed+offset), buflen - offset, &msr, reclen, dataflag, verbose);
        if (retcode != MS_NOERROR) {
            switch ( retcode ) {
                case MS_ENDOFFILE:
                    ms_log(1, "readMSEEDBuffer(): Unexpected end of file when "
                              "parsing record starting at offset %d. The rest "
                              "of the file will not be read.\n", offset);
                    break;
                case MS_GENERROR:
                    ms_log(1, "readMSEEDBuffer(): Generic error when parsing "
                              "record starting at offset %d. The rest of the "
                              "file will not be read.\n", offset);
                    break;
                case MS_NOTSEED:
                    ms_log(1, "readMSEEDBuffer(): Record starting at offset "
                              "%d is not valid SEED. The rest of the file "
                              "will not be read.\n", offset);
                    break;
                case MS_WRONGLENGTH:
                    ms_log(1, "readMSEEDBuffer(): Length of data read was not "
                              "correct when parsing record starting at "
                              "offset %d. The rest of the file will not be "
                              "read.\n", offset);
                    break;
                case MS_OUTOFRANGE:
                    ms_log(1, "readMSEEDBuffer(): SEED record length out of "
                              "range for record starting at offset %d. The "
                              "rest of the file will not be read.\n", offset);
                    break;
                case MS_UNKNOWNFORMAT:
                    ms_log(1, "readMSEEDBuffer(): Unknown data encoding "
                              "format for record starting at offset %d. The "
                              "rest of the file will not be read.\n", offset);
                    break;
                case MS_STBADCOMPFLAG:
                    ms_log(1, "readMSEEDBuffer(): Invalid STEIM compression "
                              "flag(s) in record starting at offset %d. The "
                              "rest of the file will not be read.\n", offset);
                    break;
                default:
                    ms_log(1, "readMSEEDBuffer(): Unknown error '%d' in "
                              "record starting at offset %d. The rest of the "
                              "file will not be read.\n", retcode, offset);
                    break;
            }
            msr_free(&msr);
            break;
        }
        if (offset + msr->reclen > buflen) {
            ms_log(1, "readMSEEDBuffer(): Last msr->reclen exceeds buflen, skipping.\n");
            msr_free(&msr);
            break;
        }

        // Test against selections if supplied
        if ( selections ) {
            char srcname[50];
            hptime_t endtime;
            msr_srcname (msr, srcname, 1);
            endtime = msr_endtime (msr);
            if ( ms_matchselect (selections, srcname, msr->starttime, endtime, NULL) == NULL ) {
                // Add the record length for the next iteration
                offset += msr->reclen;
                // Free record.
                msr_free(&msr);
                continue;
            }
        }
        record_count += 1;

        recordCurrent = lrl_init ();
        // Append to linked record list if one exists.
        if ( recordHead != NULL ) {
            recordPrevious->next = recordCurrent;
            recordCurrent->previous = recordPrevious;
            recordCurrent->next = NULL;
            recordPrevious = recordCurrent;
        }
        // Otherwise create a new one.
        else {
            recordHead = recordCurrent;
            recordCurrent->previous = NULL;
            recordPrevious = recordCurrent;
        }
        recordCurrent->record = msr;

        // Determine the byte order swapflag only for the very first record.
        // The byte order should not change within the file.
        // XXX: Maybe check for every record?
        if (swapflag <= 0) {
            // Returns 0 if the host is little endian, otherwise 1.
            flag bigendianhost = ms_bigendianhost();
            // Set the swapbyteflag if it is needed.
            if ( msr->Blkt1000 != 0) {
                /* If BE host and LE data need swapping */
                if ( bigendianhost && msr->byteorder == 0 ) {
                    swapflag = 1;
                }
                /* If LE host and BE data (or bad byte order value) need swapping */
                if ( !bigendianhost && msr->byteorder > 0 ) {
                    swapflag = 1;
                }
            }
        }

        // Actually unpack the data if the flag is not set.
        if (unpack_data != 0) {
            retval = msr_unpack_data (msr, swapflag, verbose);
        }

        if ( retval > 0 ) {
            msr->numsamples = retval;
        }

        // Add the record length for the next iteration
        offset += msr->reclen;
    }

    // Return empty id list if no records could be found.
    if (record_count == 0) {
        idListHead = lil_init();
        return idListHead;
    }


    // All records that match the selection are now stored in a LinkedRecordList
    // that starts at recordHead. The next step is to sort them by matching ids
    // and then by time.
    recordCurrent = recordHead;
    while (recordCurrent != NULL) {
        // Check if the ID of the record is already available and if not create a
        // new one.
        // Start with the last id as it is most likely to be the correct one.
        idListCurrent = idListLast;
        while (idListCurrent != NULL) {
            if (strcmp(idListCurrent->network, recordCurrent->record->network) == 0 &&
                strcmp(idListCurrent->station, recordCurrent->record->station) == 0 &&
                strcmp(idListCurrent->location, recordCurrent->record->location) == 0 &&
                strcmp(idListCurrent->channel, recordCurrent->record->channel) == 0 &&
                idListCurrent->dataquality == recordCurrent->record->dataquality) {
                break;
            }
            else {
                idListCurrent = idListCurrent->previous;
            }
        }

        // Create a new id list if one is needed.
        if (idListCurrent == NULL) {
            idListCurrent = lil_init();
            idListCurrent->previous = idListLast;
            if (idListLast != NULL) {
                idListLast->next = idListCurrent;
            }
            idListLast = idListCurrent;
            if (idListHead == NULL) {
                idListHead = idListCurrent;
            }

            // Set the IdList attributes.
            strcpy(idListCurrent->network, recordCurrent->record->network);
            strcpy(idListCurrent->station, recordCurrent->record->station);
            strcpy(idListCurrent->location, recordCurrent->record->location);
            strcpy(idListCurrent->channel, recordCurrent->record->channel);
            idListCurrent->dataquality = recordCurrent->record->dataquality;
        }

        // Now check if the current record fits exactly to the end of the last
        // segment of the current id. If not create a new segment. Therefore
        // if records with the same id are in wrong order a new segment will be
        // created. This is on purpose.
        segmentCurrent = idListCurrent->lastSegment;
        if (segmentCurrent != NULL) {
            hptimetol = (hptime_t) (0.5 * segmentCurrent->hpdelta);
            nhptimetol = ( hptimetol ) ? -hptimetol : 0;
            lastgap = recordCurrent->record->starttime - segmentCurrent->endtime - segmentCurrent->hpdelta;
        }
        if (details == 1) {
            /* extract information on calibration BLKs */
            calibration_type = -1;
            if (recordCurrent->record->blkts) {
                BlktLink *cur_blkt = recordCurrent->record->blkts;
                while (cur_blkt) {
                    switch (cur_blkt->blkt_type) {
                    case 300:
                        calibration_type = 1;
                        break;
                    case 310:
                        calibration_type = 2;
                        break;
                    case 320:
                        calibration_type = 3;
                        break;
                    case 390:
                        calibration_type = 4;
                        break;
                    case 395:
                        calibration_type = -2;
                        break;
                    default:
                        break;
                    }
                    cur_blkt = cur_blkt->next;
                }
            }
            /* extract information based on timing quality */
            timing_qual = 0xFF;
            if (recordCurrent->record->Blkt1001 != 0) {
                timing_qual = recordCurrent->record->Blkt1001->timing_qual;
            }
        }
        if ( segmentCurrent != NULL &&
             segmentCurrent->sampletype == recordCurrent->record->sampletype &&
             // Test the default sample rate tolerance: abs(1-sr1/sr2) < 0.0001
             MS_ISRATETOLERABLE (segmentCurrent->samprate, recordCurrent->record->samprate) &&
             // Check if the times are within the time tolerance
             lastgap <= hptimetol && lastgap >= nhptimetol &&
             segmentCurrent->timing_qual == timing_qual &&
             segmentCurrent->calibration_type == calibration_type) {
            recordCurrent->previous = segmentCurrent->lastRecord;
            segmentCurrent->lastRecord = segmentCurrent->lastRecord->next = recordCurrent;
            segmentCurrent->samplecnt += recordCurrent->record->samplecnt;
            segmentCurrent->endtime = msr_endtime(recordCurrent->record);
        }
        // Otherwise create a new segment and add the current record.
        else {
            segmentCurrent = seg_init();
            segmentCurrent->previous = idListCurrent->lastSegment;
            if (idListCurrent->lastSegment != NULL) {
                idListCurrent->lastSegment->next = segmentCurrent;
            }
            else {
                idListCurrent->firstSegment = segmentCurrent;
            }
            idListCurrent->lastSegment = segmentCurrent;

            segmentCurrent->starttime = recordCurrent->record->starttime;
            segmentCurrent->endtime = msr_endtime(recordCurrent->record);
            segmentCurrent->samprate = recordCurrent->record->samprate;
            segmentCurrent->sampletype = recordCurrent->record->sampletype;
            segmentCurrent->samplecnt = recordCurrent->record->samplecnt;
            // Calculate high-precision sample period
            segmentCurrent->hpdelta = (hptime_t) (( recordCurrent->record->samprate ) ?
                           (HPTMODULUS / recordCurrent->record->samprate) : 0.0);
            segmentCurrent->timing_qual = timing_qual;
            segmentCurrent->calibration_type = calibration_type;
            segmentCurrent->firstRecord = segmentCurrent->lastRecord = recordCurrent;
            recordCurrent->previous = NULL;
        }
        recordPrevious = recordCurrent->next;
        recordCurrent->next = NULL;
        recordCurrent = recordPrevious;
    }


    // Now loop over all segments, combine the records and free the msr
    // structures.
    idListCurrent = idListHead;
    while (idListCurrent != NULL)
    {
        segmentCurrent = idListCurrent->firstSegment;

        while (segmentCurrent != NULL) {
            if (segmentCurrent->datasamples) {
                free(segmentCurrent->datasamples);
            }
            // Allocate data via a callback function.
            if (unpack_data != 0) {
                segmentCurrent->datasamples = (void *) allocData(segmentCurrent->samplecnt, segmentCurrent->sampletype);
            }

            // Loop over all records, write the data to the buffer and free the msr structures.
            recordCurrent = segmentCurrent->firstRecord;
            data_offset = (long)(segmentCurrent->datasamples);
            while (recordCurrent != NULL) {
                datasize = recordCurrent->record->samplecnt * ms_samplesize(recordCurrent->record->sampletype);
                memcpy((void *)data_offset, recordCurrent->record->datasamples, datasize);
                // Free the record.
                msr_free(&(recordCurrent->record));
                // Increase the data_offset and the record.
                data_offset += (long)datasize;
                recordCurrent = recordCurrent->next;
            }

            segmentCurrent = segmentCurrent->next;
        }
        idListCurrent = idListCurrent->next;
    }
    return idListHead;
}
Ejemplo n.º 4
0
/***************************************************************************
 * writeascii:
 * 
 * Write data buffer to output file as ASCII.
 *
 * Returns the number of samples written or -1 on error.
 ***************************************************************************/
static int
writeascii (MSTrace *mst)
{
  char outfile[1024];
  char *outname;
  char timestr[50];
  char srcname[50];
  char *samptype;
  
  int line, col, cnt, samplesize;
  int lines;
  void *sptr;


  
  if ( ! mst )
    return -1;
  
  if ( mst->numsamples == 0 || mst->samprate == 0.0 )
    return 0;
  
  if ( verbose )
    fprintf (stderr, "Writing ASCII for %.8s.%.8s.%.8s.%.8s\n",
	     mst->network, mst->station, mst->location, mst->channel);
  
  /* Generate source name and ISO time string */
  mst_srcname (mst, srcname, 1);
  ms_hptime2isotimestr (mst->starttime, timestr, 1);
    
  /* Set sample type description */

  if ( mst->sampletype == 'i' )
    {
      samptype = "INTEGER";
    }

  
  /* Generate and open output file name if single file not being used */
  if ( ! ofp )
    {
      /* Create output file name: Net.Sta.Loc.Chan.Qual.Year-Month-DayTHour:Min:Sec.Subsec.txt */
      snprintf (outfile, sizeof(outfile), "%s.%s.%s.%s.%c.%s.txt",
		mst->network, mst->station, mst->location, mst->channel,
		mst->dataquality, timestr);
      
         
      
      /* Open output file */
      if ( (ofp = fopen (outfile, "wb")) == NULL )
	{
	  fprintf (stderr, "Cannot open output file: %s (%s)\n",
		   outfile, strerror(errno));
	  return -1;
	}
      
      outname = outfile;
    }
  
  /* Header format:
   * "TIMESERIES Net_Sta_Loc_Chan_Qual, ## samples, ## sps, isotime, SLIST|TSPAIR, INTEGER|FLOAT|ASCII, Units" */
  
       /* Print header line */
      fprintf (ofp, "TIMESERIES %s, %lld samples, %g sps, %s, SLIST, %s\n",
	       srcname, (long long int)mst->numsamples, mst->samprate, timestr, samptype);
      
//      lines = (mst->numsamples / slistcols) + ((slistcols == 1) ? 0 : 1);
      lines = mst->numsamples;
      printf("lines: %d numsamples: %d\n",lines,mst->numsamples);
      
      if ( (samplesize = ms_samplesize(mst->sampletype)) == 0 )
	{
	  fprintf (stderr, "Unrecognized sample type: %c\n", mst->sampletype);
	}
      

      else
	for ( cnt = 0, line = 0; line < lines; line++ )
	  {
		if ( cnt < mst->numsamples )
		  {
		    sptr = (char*)mst->datasamples + (cnt * samplesize);
		    
		    if ( mst->sampletype == 'i' )
		      {
			  fprintf (ofp, "%d", *(int32_t *)sptr);
//			printf("%d\n",*(int32_t *)sptr);
		      }

		      cnt++;
		  }
	    fprintf (ofp, "\n");
	  }

  
  if ( outname == outfile )
    {
      fclose (ofp);
      ofp = 0;
    }
  
  fprintf (stderr, "Wrote %lld samples from %s to %s\n",
	   (long long int)mst->numsamples, srcname, outname);
  
  return mst->numsamples;
}  /* End of writeascii() */
Ejemplo n.º 5
0
static PyObject*
mseed_store_traces (PyObject *dummy, PyObject *args)
{
    char          *filename;
    MSTrace       *mst = NULL;
    PyObject      *array = NULL;
    PyObject      *in_traces = NULL;
    PyObject      *in_trace = NULL;
    PyArrayObject *contiguous_array = NULL;
    int           i;
    char          *network, *station, *location, *channel;
    char          mstype;
    int           msdetype;
    int           psamples, precords;
    int           numpytype;
    int           length;
    FILE          *outfile;

    if (!PyArg_ParseTuple(args, "Os", &in_traces, &filename)) {
        PyErr_SetString(MSeedError, "usage store_traces(traces, filename)" );
        return NULL;
    }
    if (!PySequence_Check( in_traces )) {
        PyErr_SetString(MSeedError, "Traces is not of sequence type." );
        return NULL;
    }

    outfile = fopen(filename, "w" );
    if (outfile == NULL) {
        PyErr_SetString(MSeedError, "Error opening file.");
        return NULL;
    }

    for (i=0; i<PySequence_Length(in_traces); i++) {
        
        in_trace = PySequence_GetItem(in_traces, i);
        if (!PyTuple_Check(in_trace)) {
            PyErr_SetString(MSeedError, "Trace record must be a tuple of (network, station, location, channel, starttime, endtime, samprate, data)." );
            Py_DECREF(in_trace);
            return NULL;
        }
        mst = mst_init (NULL);
        
        if (!PyArg_ParseTuple(in_trace, "ssssLLdO",
                                    &network,
                                    &station,
                                    &location,
                                    &channel,
                                    &(mst->starttime),
                                    &(mst->endtime),
                                    &(mst->samprate),
                                    &array )) {
            PyErr_SetString(MSeedError, "Trace record must be a tuple of (network, station, location, channel, starttime, endtime, samprate, data)." );
            mst_free( &mst );  
            Py_DECREF(in_trace);
            return NULL;
        }

        strncpy( mst->network, network, 10);
        strncpy( mst->station, station, 10);
        strncpy( mst->location, location, 10);
        strncpy( mst->channel, channel, 10);
        mst->network[10] = '\0';
        mst->station[10] = '\0';
        mst->location[10] ='\0';
        mst->channel[10] = '\0';
        
        if (!PyArray_Check(array)) {
            PyErr_SetString(MSeedError, "Data must be given as NumPy array." );
            mst_free( &mst );  
            Py_DECREF(in_trace);
            return NULL;
        }
        numpytype = PyArray_TYPE(array);
        switch (numpytype) {
                case NPY_INT32:
                    assert( ms_samplesize('i') == 4 );
                    mstype = 'i';
                    msdetype = DE_STEIM1;
                    break;
                case NPY_INT8:
                    assert( ms_samplesize('a') == 1 );
                    mstype = 'a';
                    msdetype = DE_ASCII;
                    break;
                case NPY_FLOAT32:
                    assert( ms_samplesize('f') == 4 );
                    mstype = 'f';
                    msdetype = DE_FLOAT32;
                    break;
                case NPY_FLOAT64:
                    assert( ms_samplesize('d') == 8 );
                    mstype = 'd';
                    msdetype = DE_FLOAT64;
                    break;
                default:
                    PyErr_SetString(MSeedError, "Data must be of type float64, float32, int32 or int8.");
                    mst_free( &mst );  
                    Py_DECREF(in_trace);
                    return NULL;
            }
        mst->sampletype = mstype;

        contiguous_array = PyArray_GETCONTIGUOUS((PyArrayObject*)array);

        length = PyArray_SIZE(contiguous_array);
        mst->numsamples = length;
        mst->samplecnt = length;

        mst->datasamples = calloc(length,ms_samplesize(mstype));
        memcpy(mst->datasamples, PyArray_DATA(contiguous_array), length*ms_samplesize(mstype));
        Py_DECREF(contiguous_array);

        precords = mst_pack (mst, &record_handler, outfile, 4096, msdetype,
                                     1, &psamples, 1, 0, NULL);
        mst_free( &mst );
        Py_DECREF(in_trace);
    }
    fclose( outfile );

    Py_INCREF(Py_None);
    return Py_None;
}
Ejemplo n.º 6
0
static PyObject*
mseed_get_traces (PyObject *dummy, PyObject *args)
{
    char          *filename;
    MSTraceGroup  *mstg = NULL;
    MSTrace       *mst = NULL;
    int           retcode;
    npy_intp      array_dims[1] = {0};
    PyObject      *array = NULL;
    PyObject      *out_traces = NULL;
    PyObject      *out_trace = NULL;
    int           numpytype;
    char          strbuf[BUFSIZE];
    PyObject      *unpackdata = NULL;

    if (!PyArg_ParseTuple(args, "sO", &filename, &unpackdata)) {
        PyErr_SetString(MSeedError, "usage get_traces(filename, dataflag)" );
        return NULL;
    }

    if (!PyBool_Check(unpackdata)) {
        PyErr_SetString(MSeedError, "Second argument must be a boolean" );
        return NULL;
    }
  
    /* get data from mseed file */
    retcode = ms_readtraces (&mstg, filename, 0, -1.0, -1.0, 0, 1, (unpackdata == Py_True), 0);
    if ( retcode < 0 ) {
        snprintf (strbuf, BUFSIZE, "Cannot read file '%s': %s", filename, ms_errorstr(retcode));
        PyErr_SetString(MSeedError, strbuf);
        return NULL;
    }

    if ( ! mstg ) {
        snprintf (strbuf, BUFSIZE, "Error reading file");
        PyErr_SetString(MSeedError, strbuf);
        return NULL;
    }

    /* check that there is data in the traces */
    if (unpackdata == Py_True) {
        mst = mstg->traces;
        while (mst) {
            if (mst->datasamples == NULL) {
                snprintf (strbuf, BUFSIZE, "Error reading file - datasamples is NULL");
                PyErr_SetString(MSeedError, strbuf);
                return NULL;
            }
            mst = mst->next;
        }
    }

    out_traces = Py_BuildValue("[]");

    mst = mstg->traces;

    /* convert data to python tuple */

    while (mst) {
        
        if (unpackdata == Py_True) {
            array_dims[0] = mst->numsamples;
            switch (mst->sampletype) {
                case 'i':
                    assert( ms_samplesize('i') == 4 );
                    numpytype = NPY_INT32;
                    break;
                case 'a':
                    assert( ms_samplesize('a') == 1 );
                    numpytype = NPY_INT8;
                    break;
                case 'f':
                    assert( ms_samplesize('f') == 4 );
                    numpytype = NPY_FLOAT32;
                    break;
                case 'd':
                    assert( ms_samplesize('d') == 8 );
                    numpytype = NPY_FLOAT64;
                    break;
                default:
                    snprintf (strbuf, BUFSIZE, "Unknown sampletype %c\n", mst->sampletype);
                    PyErr_SetString(MSeedError, strbuf);
                    Py_XDECREF(out_traces);
                    return NULL;
            }
            array = PyArray_SimpleNew(1, array_dims, numpytype);
            memcpy( PyArray_DATA(array), mst->datasamples, mst->numsamples*ms_samplesize(mst->sampletype) );
        } else {
            Py_INCREF(Py_None);
            array = Py_None;
        }

        out_trace = Py_BuildValue( "(c,s,s,s,s,L,L,d,N)",
                                    mst->dataquality,
                                    mst->network,
                                    mst->station,
                                    mst->location,
                                    mst->channel,
                                    mst->starttime,
                                    mst->endtime,
                                    mst->samprate,
                                    array );

        
        PyList_Append(out_traces, out_trace);
        Py_DECREF(out_trace);
        mst = mst->next;
    }

    mst_freegroup (&mstg);

    return out_traces;
}
Ejemplo n.º 7
0
Archivo: msi.c Proyecto: ovsm-dev/sdp
int
main (int argc, char **argv)
{
  struct filelink *flp;
  MSRecord *msr = 0;
  MSTraceList *mstl = 0;
  FILE *bfp = 0;
  FILE *ofp = 0;
  int retcode = MS_NOERROR;
  
  char envvariable[100];
  int dataflag = 0;
  long long int totalrecs  = 0;
  long long int totalsamps = 0;
  long long int totalfiles = 0;
  off_t filepos = 0;
  
  char srcname[50];
  char stime[30];
  
  /* Set default error message prefix */
  ms_loginit (NULL, NULL, NULL, "ERROR: ");
  
  /* Process given parameters (command line and parameter file) */
  if ( processparam (argc, argv) < 0 )
    return 1;
  
  /* Setup encoding environment variable if specified, ugly kludge */
  if ( encodingstr )
    {
      snprintf (envvariable, sizeof(envvariable), "UNPACK_DATA_FORMAT=%s", encodingstr);
      
      if ( putenv (envvariable) )
	{
	  ms_log (2, "Cannot set environment variable UNPACK_DATA_FORMAT\n");
	  return 1;
	}
    }
  
  /* Open the integer output file if specified */
  if ( binfile )
    {
      if ( strcmp (binfile, "-") == 0 )
	{
	  bfp = stdout;
	}
      else if ( (bfp = fopen (binfile, "wb")) == NULL )
	{
	  ms_log (2, "Cannot open binary data output file: %s (%s)\n",
		  binfile, strerror(errno));
	  return 1;
	}
    }

  /* Open the output file if specified */
  if ( outfile )
    {
      if ( strcmp (outfile, "-") == 0 )
	{
	  ofp = stdout;
	}
      else if ( (ofp = fopen (outfile, "wb")) == NULL )
	{
	  ms_log (2, "Cannot open output file: %s (%s)\n",
		  outfile, strerror(errno));
	  return 1;
	}
    }
  
  if ( printdata || binfile )
    dataflag = 1;
  
  if ( tracegapsum || tracegaponly )
    mstl = mstl_init (NULL);
  
  flp = filelist;
  
  while ( flp != 0 )
    {
      if ( verbose >= 2 )
	{
	  if ( flp->offset )
	    ms_log (1, "Processing: %s (starting at byte %lld)\n", flp->filename, flp->offset);
	  else
	    ms_log (1, "Processing: %s\n", flp->filename);
	}
      
      /* Set starting byte offset if supplied as negative file position */
      filepos = - flp->offset;
      
      /* Loop over the input file */
      while ( reccntdown != 0 )
	{
	  if ( (retcode = ms_readmsr (&msr, flp->filename, reclen, &filepos,
				      NULL, skipnotdata, 0, verbose)) != MS_NOERROR )
	    break;
	  
	  /* Check if record matches start/end time criteria */
	  if ( starttime != HPTERROR || endtime != HPTERROR )
	    {
	      hptime_t recendtime = msr_endtime (msr);
	      
	      if ( starttime != HPTERROR && (msr->starttime < starttime && ! (msr->starttime <= starttime && recendtime >= starttime)) )
		{
		  if ( verbose >= 3 )
		    {
		      msr_srcname (msr, srcname, 1);
		      ms_hptime2seedtimestr (msr->starttime, stime, 1);
		      ms_log (1, "Skipping (starttime) %s, %s\n", srcname, stime);
		    }
		  continue;
		}
	      
	      if ( endtime != HPTERROR && (recendtime > endtime && ! (msr->starttime <= endtime && recendtime >= endtime)) )
		{
		  if ( verbose >= 3 )
		    {
		      msr_srcname (msr, srcname, 1);
		      ms_hptime2seedtimestr (msr->starttime, stime, 1);
		      ms_log (1, "Skipping (starttime) %s, %s\n", srcname, stime);
		    }
		  continue;
		}
	    }
	  
	  if ( match || reject )
	    {
	      /* Generate the srcname with the quality code */
	      msr_srcname (msr, srcname, 1);
	      
	      /* Check if record is matched by the match regex */
	      if ( match )
		{
		  if ( regexec ( match, srcname, 0, 0, 0) != 0 )
		    {
		      if ( verbose >= 3 )
			{
			  ms_hptime2seedtimestr (msr->starttime, stime, 1);
			  ms_log (1, "Skipping (match) %s, %s\n", srcname, stime);
			}
		      continue;
		    }
		}
	      
	      /* Check if record is rejected by the reject regex */
	      if ( reject )
		{
		  if ( regexec ( reject, srcname, 0, 0, 0) == 0 )
		    {
		      if ( verbose >= 3 )
			{
			  ms_hptime2seedtimestr (msr->starttime, stime, 1);
			  ms_log (1, "Skipping (reject) %s, %s\n", srcname, stime);
			}
		      continue;
		    }
		}
	    }
	  
	  if ( reccntdown > 0 )
	    reccntdown--;
	  
	  totalrecs++;
	  totalsamps += msr->samplecnt;
	  
	  if ( ! tracegaponly )
	    {
	      if ( printoffset )
		ms_log (0, "%-10lld", (long long) filepos);
	      
	      if ( printlatency )
		ms_log (0, "%-10.6g secs ", msr_host_latency(msr));
	      
	      if ( printraw )
		ms_parse_raw (msr->record, msr->reclen, ppackets, -1);
	      else
		msr_print (msr, ppackets);
	    }
	  
	  if ( tracegapsum || tracegaponly )
	    mstl_addmsr (mstl, msr, dataquality, 1, timetol, sampratetol);
	  
	  if ( dataflag )
	    {
	      /* Parse the record (again) and unpack the data */
	      int rv = msr_unpack (msr->record, msr->reclen, &msr, 1, verbose);
	      
	      if ( rv == MS_NOERROR && printdata && ! tracegaponly )
		{
		  int line, col, cnt, samplesize;
		  int lines = (msr->numsamples / 6) + 1;
		  void *sptr;
		  
		  if ( (samplesize = ms_samplesize(msr->sampletype)) == 0 )
		    {
		      ms_log (2, "Unrecognized sample type: %c\n", msr->sampletype);
		    }
		  
		  if ( msr->sampletype == 'a' )
		    {
		      char *ascii = (char *)msr->datasamples;
		      int length = msr->numsamples;
		      
		      ms_log (0, "ASCII Data:\n");
		      
		      /* Print maximum log message segments */
		      while ( length > (MAX_LOG_MSG_LENGTH-1) )
			{
			  ms_log (0, "%.*s", (MAX_LOG_MSG_LENGTH-1), ascii);
			  ascii += MAX_LOG_MSG_LENGTH-1;
			  length -= MAX_LOG_MSG_LENGTH-1;
			}
		      
		      /* Print any remaining ASCII and add a newline */
		      if ( length > 0 )
			{
			  ms_log (0, "%.*s\n", length, ascii);
			}
		      else
			{
			  ms_log (0, "\n");
			}
		    }
		  else
		    for ( cnt = 0, line = 0; line < lines; line++ )
		      {
			for ( col = 0; col < 6 ; col ++ )
			  {
			    if ( cnt < msr->numsamples )
			      {
				sptr = (char*)msr->datasamples + (cnt * samplesize);
				
				if ( msr->sampletype == 'i' )
				  ms_log (0, "%10d  ", *(int32_t *)sptr);
				
				else if ( msr->sampletype == 'f' )
				  ms_log (0, "%10.8g  ", *(float *)sptr);
				
				else if ( msr->sampletype == 'd' )
				  ms_log (0, "%10.10g  ", *(double *)sptr);
				
				cnt++;
			      }
			  }
			ms_log (0, "\n");
			
			/* If only printing the first 6 samples break out here */
			if ( printdata == 1 )
			  break;
		      }
		}
	      
	      if ( binfile )
		{
		  uint8_t samplesize = ms_samplesize (msr->sampletype);
		  
		  if ( samplesize )
		    {
		      fwrite (msr->datasamples, samplesize, msr->numsamples, bfp);
		    }
		  else
		    {
		      ms_log (1, "Cannot write to binary file, unknown sample type: %c\n",
			      msr->sampletype);
		    }
		}
	    }
	  
	  if ( outfile )
	    {
	      fwrite (msr->record, 1, msr->reclen, ofp);
	    }
	}
      
      /* Print error if not EOF and not counting down records */
      if ( retcode != MS_ENDOFFILE && reccntdown != 0 )
	{
	  ms_log (2, "Cannot read %s: %s\n", flp->filename, ms_errorstr(retcode));
	  ms_readmsr (&msr, NULL, 0, NULL, NULL, 0, 0, 0);
	  exit (1);
	}
      
      /* Make sure everything is cleaned up */
      ms_readmsr (&msr, NULL, 0, NULL, NULL, 0, 0, 0);
      
      totalfiles++;
      flp = flp->next;
    } /* End of looping over file list */
  
  if ( binfile )
    fclose (bfp);
  
  if ( outfile )
    fclose (ofp);
  
  if ( basicsum )
    ms_log (0, "Files: %lld, Records: %lld, Samples: %lld\n", totalfiles, totalrecs, totalsamps);
  
  if ( tracegapsum || tracegaponly )
    {
      if ( tracegapsum == 1 || tracegaponly == 1 )
	{
	  mstl_printtracelist (mstl, timeformat, 1, tracegaps);
	}
      if ( tracegapsum == 2 || tracegaponly == 2 )
	{
	  mstl_printgaplist (mstl, timeformat, mingapptr, maxgapptr);
	}
      if ( tracegaponly == 3 )
	{
	  mstl_printsynclist (mstl, NULL, 1);
	}
    }
  
  if ( mstl )
    mstl_free (&mstl, 0);
  
  return 0;
}  /* End of main() */
Ejemplo n.º 8
0
// Function that reads from a MiniSEED binary file from a char buffer and
// returns a LinkedIDList.
LinkedIDList *
readMSEEDBuffer (char *mseed, int buflen, Selections *selections, flag
                 unpack_data, int reclen, flag verbose, flag details,
                 int header_byteorder, long long (*allocData) (int, char),
                 void (*diag_print) (char*), void (*log_print) (char*))
{
    int retcode = 0;
    int retval = 0;
    flag swapflag = 0;
    flag bigendianhost = ms_bigendianhost();

    // current offset of mseed char pointer
    int offset = 0;

    // Unpack without reading the data first
    flag dataflag = 0;

    // the timing_qual of BLK 1001
    uint8_t timing_qual = 0xFF;

    // the calibration type, availability of BLK 300, 310, 320, 390, 395
    int8_t calibration_type = -1;

    // Init all the pointers to NULL. Most compilers should do this anyway.
    LinkedIDList * idListHead = NULL;
    LinkedIDList * idListCurrent = NULL;
    LinkedIDList * idListLast = NULL;
    MSRecord *msr = NULL;
    ContinuousSegment * segmentCurrent = NULL;
    hptime_t lastgap = 0;
    hptime_t hptimetol = 0;
    hptime_t nhptimetol = 0;
    long long data_offset;
    LinkedRecordList *recordHead = NULL;
    LinkedRecordList *recordPrevious = NULL;
    LinkedRecordList *recordCurrent = NULL;
    int datasize;
    int record_count = 0;

    // A negative verbosity suppresses as much as possible.
    if (verbose < 0) {
        ms_loginit(&empty_print, NULL, &empty_print, NULL);
    }
    else {
        ms_loginit(log_print, "INFO: ", diag_print, "ERROR: ");
    }

    if (header_byteorder >= 0) {
        // Enforce little endian.
        if (header_byteorder == 0) {
            MS_UNPACKHEADERBYTEORDER(0);
        }
        // Enforce big endian.
        else {
            MS_UNPACKHEADERBYTEORDER(1);
        }
    }
    else {
        MS_UNPACKHEADERBYTEORDER(-1);
    }

    // Read all records and save them in a linked list.
    while (offset < buflen) {
        msr = msr_init(NULL);
        if ( msr == NULL ) {
            ms_log (2, "readMSEEDBuffer(): Error initializing msr\n");
            return NULL;
        }
        if (verbose > 1) {
            ms_log(0, "readMSEEDBuffer(): calling msr_parse with "
                      "mseed+offset=%d+%d, buflen=%d, reclen=%d, dataflag=%d, verbose=%d\n",
                      mseed, offset, buflen, reclen, dataflag, verbose);
        }

        // If the record length is given, make sure at least that amount of data is available.
        if (reclen != -1) {
            if (offset + reclen > buflen) {
                ms_log(1, "readMSEEDBuffer(): Last reclen exceeds buflen, skipping.\n");
                msr_free(&msr);
                break;
            }
        }
        // Otherwise assume the smallest possible record length and assure that enough
        // data is present.
        else {
            if (offset + MINRECLEN > buflen) {
                ms_log(1, "readMSEEDBuffer(): Last record only has %i byte(s) which "
                          "is not enough to constitute a full SEED record. Corrupt data? "
                          "Record will be skipped.\n", buflen - offset);
                msr_free(&msr);
                break;
            }
        }

        // Skip empty or noise records.
        if (OBSPY_ISVALIDBLANK(mseed + offset)) {
            offset += MINRECLEN;
            continue;
        }

        // Pass (buflen - offset) because msr_parse() expects only a single record. This
        // way libmseed can take care to not overstep bounds.
        // Return values:
        //   0 : Success, populates the supplied MSRecord.
        //  >0 : Data record detected but not enough data is present, the
        //       return value is a hint of how many more bytes are needed.
        //  <0 : libmseed error code (listed in libmseed.h) is returned.
        retcode = msr_parse ((mseed+offset), buflen - offset, &msr, reclen, dataflag, verbose);
        // Handle error.
        if (retcode < 0) {
            log_error(retcode, offset);
            msr_free(&msr);
            break;
        }
        // msr_parse() returns > 0 if a data record has been detected but the buffer either has not enough
        // data (this cannot happen with ObsPy's logic) or the last record has no Blockette 1000 and it cannot
        // determine the record length because there is no next record (this can happen in ObsPy) - handle that
        // case by just calling msr_parse() with an explicit record length set.
        else if ( retcode > 0 && retcode < (buflen - offset)) {

            // Check if the remaining bytes can exactly make up a record length.
            int r_bytes = buflen - offset;
            float exp = log10((float)r_bytes) / log10(2.0);
            if ((fmodf(exp, 1.0) < 0.0000001) && ((int)roundf_(exp) >= 7) && ((int)roundf_(exp) <= 256)) {

                retcode = msr_parse((mseed + offset), buflen - offset, &msr, r_bytes, dataflag, verbose);

                if ( retcode != 0 ) {
                    log_error(retcode, offset);
                    msr_free(&msr);
                    break;
                }

            }
            else {
                msr_free(&msr);
                break;
            }
        }

        if (offset + msr->reclen > buflen) {
            ms_log(1, "readMSEEDBuffer(): Last msr->reclen exceeds buflen, skipping.\n");
            msr_free(&msr);
            break;
        }

        // Test against selections if supplied
        if ( selections ) {
            char srcname[50];
            hptime_t endtime;
            msr_srcname (msr, srcname, 1);
            endtime = msr_endtime (msr);
            if ( ms_matchselect (selections, srcname, msr->starttime, endtime, NULL) == NULL ) {
                // Add the record length for the next iteration
                offset += msr->reclen;
                // Free record.
                msr_free(&msr);
                continue;
            }
        }
        record_count += 1;

        recordCurrent = lrl_init ();
        // Append to linked record list if one exists.
        if ( recordHead != NULL ) {
            recordPrevious->next = recordCurrent;
            recordCurrent->previous = recordPrevious;
            recordCurrent->next = NULL;
            recordPrevious = recordCurrent;
        }
        // Otherwise create a new one.
        else {
            recordHead = recordCurrent;
            recordCurrent->previous = NULL;
            recordPrevious = recordCurrent;
        }
        recordCurrent->record = msr;


        // Figure out if the byte-order of the data has to be swapped.
        swapflag = 0;
        // If blockette 1000 is present, use it.
        if ( msr->Blkt1000 != 0) {
            /* If BE host and LE data need swapping */
            if ( bigendianhost && msr->byteorder == 0 ) {
                swapflag = 1;
            }
            /* If LE host and BE data (or bad byte order value) need swapping */
            if ( !bigendianhost && msr->byteorder > 0 ) {
                swapflag = 1;
            }
        }
        // Otherwise assume the data has the same byte order as the header.
        // This needs to be done on the raw header bytes as libmseed only returns
        // header fields in the native byte order.
        else {
            unsigned char* _t = (unsigned char*)mseed + offset + 20;
            unsigned int year = _t[0] | _t[1] << 8;
            unsigned int day = _t[2] | _t[3] << 8;
            // Swap data if header needs to be swapped.
            if (!MS_ISVALIDYEARDAY(year, day)) {
                swapflag = 1;
            }
        }

        // Actually unpack the data if the flag is not set and if the data
        // offset is valid.
        if ((unpack_data != 0) && (msr->fsdh->data_offset >= 48) &&
            (msr->fsdh->data_offset < msr->reclen) &&
            (msr->samplecnt > 0)) {
            retval = msr_unpack_data (msr, swapflag, verbose);
        }

        if ( retval > 0 ) {
            msr->numsamples = retval;
        }

        if ( msr->fsdh->start_time.fract > 9999 ) {
            ms_log(1, "readMSEEDBuffer(): Record with offset=%d has a "
                      "fractional second (.0001 seconds) of %d. This is not "
                      "strictly valid but will be interpreted as one or more "
                      "additional seconds.",
                      offset, msr->fsdh->start_time.fract);
        }

        // Add the record length for the next iteration
        offset += msr->reclen;
    }

    // Return empty id list if no records could be found.
    if (record_count == 0) {
        idListHead = lil_init();
        return idListHead;
    }


    // All records that match the selection are now stored in a LinkedRecordList
    // that starts at recordHead. The next step is to sort them by matching ids
    // and then by time.
    recordCurrent = recordHead;
    while (recordCurrent != NULL) {
        // Check if the ID of the record is already available and if not create a
        // new one.
        // Start with the last id as it is most likely to be the correct one.
        idListCurrent = idListLast;
        while (idListCurrent != NULL) {
            if (strcmp(idListCurrent->network, recordCurrent->record->network) == 0 &&
                strcmp(idListCurrent->station, recordCurrent->record->station) == 0 &&
                strcmp(idListCurrent->location, recordCurrent->record->location) == 0 &&
                strcmp(idListCurrent->channel, recordCurrent->record->channel) == 0 &&
                idListCurrent->dataquality == recordCurrent->record->dataquality) {
                break;
            }
            else {
                idListCurrent = idListCurrent->previous;
            }
        }

        // Create a new id list if one is needed.
        if (idListCurrent == NULL) {
            idListCurrent = lil_init();
            idListCurrent->previous = idListLast;
            if (idListLast != NULL) {
                idListLast->next = idListCurrent;
            }
            idListLast = idListCurrent;
            if (idListHead == NULL) {
                idListHead = idListCurrent;
            }

            // Set the IdList attributes.
            strcpy(idListCurrent->network, recordCurrent->record->network);
            strcpy(idListCurrent->station, recordCurrent->record->station);
            strcpy(idListCurrent->location, recordCurrent->record->location);
            strcpy(idListCurrent->channel, recordCurrent->record->channel);
            idListCurrent->dataquality = recordCurrent->record->dataquality;
        }

        // Now check if the current record fits exactly to the end of the last
        // segment of the current id. If not create a new segment. Therefore
        // if records with the same id are in wrong order a new segment will be
        // created. This is on purpose.
        segmentCurrent = idListCurrent->lastSegment;
        if (segmentCurrent != NULL) {
            hptimetol = (hptime_t) (0.5 * segmentCurrent->hpdelta);
            nhptimetol = ( hptimetol ) ? -hptimetol : 0;
            lastgap = recordCurrent->record->starttime - segmentCurrent->endtime - segmentCurrent->hpdelta;
        }
        if (details == 1) {
            /* extract information on calibration BLKs */
            calibration_type = -1;
            if (recordCurrent->record->blkts) {
                BlktLink *cur_blkt = recordCurrent->record->blkts;
                while (cur_blkt) {
                    switch (cur_blkt->blkt_type) {
                    case 300:
                        calibration_type = 1;
                        break;
                    case 310:
                        calibration_type = 2;
                        break;
                    case 320:
                        calibration_type = 3;
                        break;
                    case 390:
                        calibration_type = 4;
                        break;
                    case 395:
                        calibration_type = -2;
                        break;
                    default:
                        break;
                    }
                    cur_blkt = cur_blkt->next;
                }
            }
            /* extract information based on timing quality */
            timing_qual = 0xFF;
            if (recordCurrent->record->Blkt1001 != 0) {
                timing_qual = recordCurrent->record->Blkt1001->timing_qual;
            }
        }
        if ( segmentCurrent != NULL &&

             // This is important for zero data record coupled with not unpacking
             // the data. It needs to be split in two places: Before the zero data
             // record and after it.
             recordCurrent->record->samplecnt > 0 && segmentCurrent->samplecnt > 0 &&

             segmentCurrent->sampletype == recordCurrent->record->sampletype &&
             // Test the default sample rate tolerance: abs(1-sr1/sr2) < 0.0001
             MS_ISRATETOLERABLE (segmentCurrent->samprate, recordCurrent->record->samprate) &&
             // Check if the times are within the time tolerance
             lastgap <= hptimetol && lastgap >= nhptimetol &&
             segmentCurrent->timing_qual == timing_qual &&
             segmentCurrent->calibration_type == calibration_type) {
            recordCurrent->previous = segmentCurrent->lastRecord;
            segmentCurrent->lastRecord = segmentCurrent->lastRecord->next = recordCurrent;
            segmentCurrent->samplecnt += recordCurrent->record->samplecnt;
            segmentCurrent->endtime = msr_endtime(recordCurrent->record);
        }
        // Otherwise create a new segment and add the current record.
        else {
            segmentCurrent = seg_init();
            segmentCurrent->previous = idListCurrent->lastSegment;
            if (idListCurrent->lastSegment != NULL) {
                idListCurrent->lastSegment->next = segmentCurrent;
            }
            else {
                idListCurrent->firstSegment = segmentCurrent;
            }
            idListCurrent->lastSegment = segmentCurrent;

            segmentCurrent->starttime = recordCurrent->record->starttime;
            segmentCurrent->endtime = msr_endtime(recordCurrent->record);
            segmentCurrent->samprate = recordCurrent->record->samprate;
            segmentCurrent->sampletype = recordCurrent->record->sampletype;
            segmentCurrent->samplecnt = recordCurrent->record->samplecnt;
            // Calculate high-precision sample period
            segmentCurrent->hpdelta = (hptime_t) (( recordCurrent->record->samprate ) ?
                           (HPTMODULUS / recordCurrent->record->samprate) : 0.0);
            segmentCurrent->timing_qual = timing_qual;
            segmentCurrent->calibration_type = calibration_type;
            segmentCurrent->firstRecord = segmentCurrent->lastRecord = recordCurrent;
            recordCurrent->previous = NULL;
        }
        recordPrevious = recordCurrent->next;
        recordCurrent->next = NULL;
        recordCurrent = recordPrevious;
    }


    // Now loop over all segments, combine the records and free the msr
    // structures.
    idListCurrent = idListHead;
    while (idListCurrent != NULL)
    {
        segmentCurrent = idListCurrent->firstSegment;

        while (segmentCurrent != NULL) {
            if (segmentCurrent->datasamples) {
                free(segmentCurrent->datasamples);
            }
            // Allocate data via a callback function.
            if (unpack_data != 0) {
                segmentCurrent->datasamples = (void *) allocData(segmentCurrent->samplecnt, segmentCurrent->sampletype);
            }

            // Loop over all records, write the data to the buffer and free the msr structures.
            recordCurrent = segmentCurrent->firstRecord;
            data_offset = (long long)(segmentCurrent->datasamples);
            while (recordCurrent != NULL) {
                datasize = recordCurrent->record->samplecnt * ms_samplesize(recordCurrent->record->sampletype);
                memcpy((void *)data_offset, recordCurrent->record->datasamples, datasize);
                // Free the record.
                msr_free(&(recordCurrent->record));
                // Increase the data_offset and the record.
                data_offset += (long long)datasize;
                recordCurrent = recordCurrent->next;
            }

            segmentCurrent = segmentCurrent->next;
        }
        idListCurrent = idListCurrent->next;
    }
    return idListHead;
}
Ejemplo n.º 9
0
// Function that reads from a MiniSEED binary file from a char buffer and
// returns a LinkedIDList.
LinkedIDList *
readMSEEDBuffer (char *mseed, int buflen, Selections *selections, flag
                 unpack_data, int reclen, flag verbose, flag details,
                 long (*allocData) (int, char))
{
    int retcode = 0;
    int retval = 0;
    flag swapflag = 0;

    // current offset of mseed char pointer
    int offset = 0;

    // Unpack without reading the data first
    flag dataflag = 0;

    // the timing_qual of BLK 1001
    uint8_t timing_qual = 0xFF;

    // the calibration type, availability of BLK 300, 310, 320, 390, 395
    int8_t calibration_type = -1;

    // Init all the pointers to NULL. Most compilers should do this anyway.
    LinkedIDList * idListHead = NULL;
    LinkedIDList * idListCurrent = NULL;
    LinkedIDList * idListLast = NULL;
    MSRecord *msr = NULL;
    ContinuousSegment * segmentCurrent = NULL;
    hptime_t lastgap;
    hptime_t hptimetol;
    hptime_t nhptimetol;
    long data_offset;
    LinkedRecordList *recordHead = NULL;
    LinkedRecordList *recordPrevious = NULL;
    LinkedRecordList *recordCurrent = NULL;
    int datasize;


    //
    // Read all records and save them in a linked list.
    //
    int record_count = 0;
    while (offset < buflen) {
        msr = msr_init(NULL);
        retcode = msr_parse ( (mseed+offset), buflen, &msr, reclen, dataflag, verbose);
        if ( ! (retcode == MS_NOERROR)) {
            msr_free(&msr);
            break;
        }

        // Test against selections if supplied
        if ( selections ) {
            char srcname[50];
            hptime_t endtime;
            msr_srcname (msr, srcname, 1);
            endtime = msr_endtime (msr);
            if ( ms_matchselect (selections, srcname, msr->starttime, endtime, NULL) == NULL ) {
                // Add the record length for the next iteration
                offset += msr->reclen;
                // Free record.
                msr_free(&msr);
                continue;
            }
        }
        record_count += 1;

        recordCurrent = lrl_init ();
        // Append to linked record list if one exists.
        if ( recordHead != NULL ) {
            recordPrevious->next = recordCurrent;
            recordCurrent->previous = recordPrevious;
            recordCurrent->next = NULL;
            recordPrevious = recordCurrent;
        }
        // Otherwise create a new one.
        else {
            recordHead = recordCurrent;
            recordCurrent->previous = NULL;
            recordPrevious = recordCurrent;
        }
        recordCurrent->record = msr;

        // Determine the byteorder swapflag only for the very first record. The byteorder
        // should not change within the file.
        // XXX: Maybe check for every record?
        if (swapflag <= 0) {
            // Returns 0 if the host is little endian, otherwise 1.
            flag bigendianhost = ms_bigendianhost();
            // Set the swapbyteflag if it is needed.
            if ( msr->Blkt1000 != 0) {
                /* If BE host and LE data need swapping */
                if ( bigendianhost && msr->byteorder == 0 ) {
                    swapflag = 1;
                }
                /* If LE host and BE data (or bad byte order value) need swapping */
                if ( !bigendianhost && msr->byteorder > 0 ) {
                    swapflag = 1;
                }
            }
        }

        // Actually unpack the data if the flag is not set.
        if (unpack_data != 0) {
            retval = msr_unpack_data (msr, swapflag, verbose);
        }

        if ( retval > 0 ) {
            msr->numsamples = retval;
        }

        // Add the record length for the next iteration
        offset += msr->reclen;
    }

    // Return empty id list if no records could be found.
    if (record_count == 0) {
        idListHead = lil_init();
        return idListHead;
    }


    // All records that match the selection are now stored in a LinkedRecordList
    // that starts at recordHead. The next step is to sort them by matching ids
    // and then by time.
    recordCurrent = recordHead;
    while (recordCurrent != NULL) {
        // Check if the ID of the record is already available and if not create a
        // new one.
        // Start with the last id as it is most likely to be the correct one.
        idListCurrent = idListLast;
        while (idListCurrent != NULL) {
            if (strcmp(idListCurrent->network, recordCurrent->record->network) == 0 &&
                strcmp(idListCurrent->station, recordCurrent->record->station) == 0 &&
                strcmp(idListCurrent->location, recordCurrent->record->location) == 0 &&
                strcmp(idListCurrent->channel, recordCurrent->record->channel) == 0 &&
                idListCurrent->dataquality == recordCurrent->record->dataquality) {
                break;
            }
            else {
                idListCurrent = idListCurrent->previous;
            }
        }

        // Create a new id list if one is needed.
        if (idListCurrent == NULL) {
            idListCurrent = lil_init();
            idListCurrent->previous = idListLast;
            if (idListLast != NULL) {
                idListLast->next = idListCurrent;
            }
            idListLast = idListCurrent;
            if (idListHead == NULL) {
                idListHead = idListCurrent;
            }

            // Set the IdList attributes.
            strcpy(idListCurrent->network, recordCurrent->record->network);
            strcpy(idListCurrent->station, recordCurrent->record->station);
            strcpy(idListCurrent->location, recordCurrent->record->location);
            strcpy(idListCurrent->channel, recordCurrent->record->channel);
            idListCurrent->dataquality = recordCurrent->record->dataquality;
        }

        // Now check if the current record fits exactly to the end of the last
        // segment of the current id. If not create a new segment. Therefore
        // if records with the same id are in wrong order a new segment will be
        // created. This is on purpose.
        segmentCurrent = idListCurrent->lastSegment;
        if (segmentCurrent != NULL) {
            hptimetol = (hptime_t) (0.5 * segmentCurrent->hpdelta);
            nhptimetol = ( hptimetol ) ? -hptimetol : 0;
            lastgap = recordCurrent->record->starttime - segmentCurrent->endtime - segmentCurrent->hpdelta;
        }
        if (details == 1) {
            /* extract information on calibration BLKs */
            calibration_type = -1;
            if (recordCurrent->record->blkts) {
                BlktLink *cur_blkt = recordCurrent->record->blkts;
                while (cur_blkt) {
                    switch (cur_blkt->blkt_type) {
                    case 300:
                        calibration_type = 1;
                        break;
                    case 310:
                        calibration_type = 2;
                        break;
                    case 320:
                        calibration_type = 3;
                        break;
                    case 390:
                        calibration_type = 4;
                        break;
                    case 395:
                        calibration_type = -2;
                        break;
                    default:
                        break;
                    }
                    cur_blkt = cur_blkt->next;
                }
            }
            /* extract information based on timing quality */
            timing_qual = 0xFF;
            if (recordCurrent->record->Blkt1001 != 0) {
                timing_qual = recordCurrent->record->Blkt1001->timing_qual;
            }
        }
        if ( segmentCurrent != NULL &&
             segmentCurrent->sampletype == recordCurrent->record->sampletype &&
             // Test the default sample rate tolerance: abs(1-sr1/sr2) < 0.0001
             MS_ISRATETOLERABLE (segmentCurrent->samprate, recordCurrent->record->samprate) &&
             // Check if the times are within the time tolerance
             lastgap <= hptimetol && lastgap >= nhptimetol &&
             segmentCurrent->timing_qual == timing_qual &&
             segmentCurrent->calibration_type == calibration_type) {
            recordCurrent->previous = segmentCurrent->lastRecord;
            segmentCurrent->lastRecord = segmentCurrent->lastRecord->next = recordCurrent;
            segmentCurrent->samplecnt += recordCurrent->record->samplecnt;
            segmentCurrent->endtime = msr_endtime(recordCurrent->record);
        }
        // Otherwise create a new segment and add the current record.
        else {
            segmentCurrent = seg_init();
            segmentCurrent->previous = idListCurrent->lastSegment;
            if (idListCurrent->lastSegment != NULL) {
                idListCurrent->lastSegment->next = segmentCurrent;
            }
            else {
                idListCurrent->firstSegment = segmentCurrent;
            }
            idListCurrent->lastSegment = segmentCurrent;

            segmentCurrent->starttime = recordCurrent->record->starttime;
            segmentCurrent->endtime = msr_endtime(recordCurrent->record);
            segmentCurrent->samprate = recordCurrent->record->samprate;
            segmentCurrent->sampletype = recordCurrent->record->sampletype;
            segmentCurrent->samplecnt = recordCurrent->record->samplecnt;
            // Calculate high-precision sample period
            segmentCurrent->hpdelta = (hptime_t) (( recordCurrent->record->samprate ) ?
                           (HPTMODULUS / recordCurrent->record->samprate) : 0.0);
            segmentCurrent->timing_qual = timing_qual;
            segmentCurrent->calibration_type = calibration_type;
            segmentCurrent->firstRecord = segmentCurrent->lastRecord = recordCurrent;
            recordCurrent->previous = NULL;
        }
        recordPrevious = recordCurrent->next;
        recordCurrent->next = NULL;
        recordCurrent = recordPrevious;
    }


    // Now loop over all segments, combine the records and free the msr
    // structures.
    idListCurrent = idListHead;
    while (idListCurrent != NULL)
    {
        segmentCurrent = idListCurrent->firstSegment;

        while (segmentCurrent != NULL) {
            if (segmentCurrent->datasamples) {
                free(segmentCurrent->datasamples);
            }
            // Allocate data via a callback function.
            if (unpack_data != 0) {
                segmentCurrent->datasamples = (void *) allocData(segmentCurrent->samplecnt, segmentCurrent->sampletype);
            }

            // Loop over all records, write the data to the buffer and free the msr structures.
            recordCurrent = segmentCurrent->firstRecord;
            data_offset = (long)(segmentCurrent->datasamples);
            while (recordCurrent != NULL) {
                datasize = recordCurrent->record->samplecnt * ms_samplesize(recordCurrent->record->sampletype);
                memcpy((void *)data_offset, recordCurrent->record->datasamples, datasize);
                // Free the record.
                msr_free(&(recordCurrent->record));
                // Increase the data_offset and the record.
                data_offset += (long)datasize;
                recordCurrent = recordCurrent->next;
            }

            segmentCurrent = segmentCurrent->next;
        }
        idListCurrent = idListCurrent->next;
    }
    return idListHead;
}
Ejemplo n.º 10
0
int
main (int argc, char **argv)
{
  MSRecord *msr = 0;

  int64_t totalrecs = 0;
  int64_t totalsamps = 0;
  int retcode;

#ifndef WIN32
  /* Signal handling, use POSIX calls with standardized semantics */
  struct sigaction sa;

  sa.sa_flags = SA_RESTART;
  sigemptyset (&sa.sa_mask);

  sa.sa_handler = term_handler;
  sigaction (SIGINT, &sa, NULL);
  sigaction (SIGQUIT, &sa, NULL);
  sigaction (SIGTERM, &sa, NULL);

  sa.sa_handler = SIG_IGN;
  sigaction (SIGHUP, &sa, NULL);
  sigaction (SIGPIPE, &sa, NULL);
#endif

  /* Process given parameters (command line and parameter file) */
  if (parameter_proc (argc, argv) < 0)
    return -1;

  /* Loop over the input file */
  while ((retcode = ms_readmsr (&msr, inputfile, reclen, NULL, NULL, 1,
                                printdata, verbose)) == MS_NOERROR)
  {
    totalrecs++;
    totalsamps += msr->samplecnt;

    msr_print (msr, ppackets);

    if (printdata && msr->numsamples > 0)
    {
      int line, col, cnt, samplesize;
      int lines = (msr->numsamples / 6) + 1;
      void *sptr;

      if ((samplesize = ms_samplesize (msr->sampletype)) == 0)
      {
        ms_log (2, "Unrecognized sample type: '%c'\n", msr->sampletype);
      }

      for (cnt = 0, line = 0; line < lines; line++)
      {
        for (col = 0; col < 6; col++)
        {
          if (cnt < msr->numsamples)
          {
            sptr = (char *)msr->datasamples + (cnt * samplesize);

            if (msr->sampletype == 'i')
              ms_log (0, "%10d  ", *(int32_t *)sptr);

            else if (msr->sampletype == 'f')
              ms_log (0, "%10.8g  ", *(float *)sptr);

            else if (msr->sampletype == 'd')
              ms_log (0, "%10.10g  ", *(double *)sptr);

            cnt++;
          }
        }
        ms_log (0, "\n");

        /* If only printing the first 6 samples break out here */
        if (printdata == 1)
          break;
      }
    }
  }

  if (retcode != MS_ENDOFFILE)
    ms_log (2, "Cannot read %s: %s\n", inputfile, ms_errorstr (retcode));

  /* Make sure everything is cleaned up */
  ms_readmsr (&msr, NULL, 0, NULL, NULL, 0, 0, 0);

  if (basicsum)
    ms_log (1, "Records: %" PRId64 ", Samples: %" PRId64 "\n",
            totalrecs, totalsamps);

  return 0;
} /* End of main() */
Ejemplo n.º 11
0
/***************************************************************************
 * msr_duplicate:
 *
 * Duplicate an MSRecord struct
 * including the fixed-section data
 * header and blockette chain.  If
 * the datadup flag is true and the
 * source MSRecord has associated
 * data samples copy them as well.
 *
 * Returns a pointer to a new MSRecord on success and NULL on error.
 ***************************************************************************/
MSRecord *
msr_duplicate (MSRecord *msr, flag datadup)
{
  MSRecord *dupmsr = 0;
  int samplesize = 0;
  
  if ( ! msr )
    return NULL;
  
  /* Allocate target MSRecord structure */
  if ( (dupmsr = msr_init (NULL)) == NULL )
    return NULL;
  
  /* Copy MSRecord structure */
  memcpy (dupmsr, msr, sizeof(MSRecord));
  
  /* Copy fixed-section data header structure */
  if ( msr->fsdh )
    {
      /* Allocate memory for new FSDH structure */
      if ( (dupmsr->fsdh = (struct fsdh_s *) malloc (sizeof(struct fsdh_s))) == NULL )
	{
	  ms_log (2, "msr_duplicate(): Error allocating memory\n");
	  free (dupmsr);
	  return NULL;
	}
      
      /* Copy the contents */
      memcpy (dupmsr->fsdh, msr->fsdh, sizeof(struct fsdh_s));
    }
  
  /* Copy the blockette chain */
  if ( msr->blkts )
    {
      BlktLink *blkt = msr->blkts;
      BlktLink *next = NULL;
      
      dupmsr->blkts = 0;
      while ( blkt )
	{
	  next = blkt->next;
	  
	  /* Add blockette to chain of new MSRecord */
	  if ( msr_addblockette (dupmsr, blkt->blktdata, blkt->blktdatalen,
				 blkt->blkt_type, 0) == NULL )
	    {
	      ms_log (2, "msr_duplicate(): Error adding blockettes\n");
	      msr_free (&dupmsr);
	      return NULL;
	    }
	  
	  blkt = next;
	}
    }
  
  /* Copy data samples if requested and available */
  if ( datadup && msr->datasamples )
    {
      /* Determine size of samples in bytes */
      samplesize = ms_samplesize (msr->sampletype);
      
      if ( samplesize == 0 )
	{
	  ms_log (2, "msr_duplicate(): unrecognized sample type: '%c'\n",
		  msr->sampletype);
	  free (dupmsr);
	  return NULL;
	}
      
      /* Allocate memory for new data array */
      if ( (dupmsr->datasamples = (void *) malloc ((size_t)(msr->numsamples * samplesize))) == NULL )
	{
	  ms_log (2, "msr_duplicate(): Error allocating memory\n");
	  free (dupmsr);
	  return NULL;
	}
      
      /* Copy the data array */
      memcpy (dupmsr->datasamples, msr->datasamples, ((size_t)(msr->numsamples * samplesize)));
    }
  /* Otherwise make sure the sample array and count are zero */
  else
    {
      dupmsr->datasamples = 0;
      dupmsr->numsamples = 0;
    }
  
  return dupmsr;
} /* End of msr_duplicate() */