Ejemplo n.º 1
0
/**********************************************************************
 * ms_readmsr_main:
 *
 * This routine will open and read, with subsequent calls, all
 * Mini-SEED records in specified file.
 *
 * All static file reading parameters are stored in a MSFileParam
 * struct and returned (via a pointer to a pointer) for the calling
 * routine to use in subsequent calls.  A MSFileParam struct will be
 * allocated if necessary.  This routine is thread safe and can be
 * used to read multiple files in parallel as long as the file reading
 * parameters are managed appropriately.
 *
 * If reclen is 0 or negative the length of every record is
 * automatically detected.  For auto detection of record length the
 * record must include a 1000 blockette or be followed by a valid
 * record header or end of file.
 *
 * If *fpos is not NULL it will be updated to reflect the file
 * position (offset from the beginning in bytes) from where the
 * returned record was read.  As a special case, if *fpos is not NULL
 * and the value it points to is less than 0 this will be interpreted
 * as a (positive) starting offset from which to begin reading data;
 * this feature does not work with packed files.
 *
 * If *last is not NULL it will be set to 1 when the last record in
 * the file is being returned, otherwise it will be 0.
 *
 * If the skipnotdata flag is true any data chunks read that do not
 * have valid data record indicators (D, R, Q, M, etc.) will be skipped.
 *
 * dataflag will be passed directly to msr_unpack().
 *
 * If a Selections list is supplied it will be used to determine when
 * a section of data in a packed file may be skipped, packed files are
 * internal to the IRIS DMC.
 *
 * After reading all the records in a file the controlling program
 * should call it one last time with msfile set to NULL.  This will
 * close the file and free allocated memory.
 *
 * Returns MS_NOERROR and populates an MSRecord struct at *ppmsr on
 * successful read, returns MS_ENDOFFILE on EOF, otherwise returns a
 * libmseed error code (listed in libmseed.h) and *ppmsr is set to
 * NULL.
 *********************************************************************/
int
ms_readmsr_main (MSFileParam **ppmsfp, MSRecord **ppmsr, const char *msfile,
                 int reclen, off_t *fpos, int *last, flag skipnotdata,
                 flag dataflag, Selections *selections, flag verbose)
{
  MSFileParam *msfp;
  off_t packdatasize = 0;
  int packskipsize;
  int parseval  = 0;
  int readsize  = 0;
  int readcount = 0;
  int retcode   = MS_NOERROR;

  if (!ppmsr)
    return MS_GENERROR;

  if (!ppmsfp)
    return MS_GENERROR;

  msfp = *ppmsfp;

  /* Initialize the file read parameters if needed */
  if (!msfp)
  {
    msfp = (MSFileParam *)malloc (sizeof (MSFileParam));

    if (msfp == NULL)
    {
      ms_log (2, "ms_readmsr_main(): Cannot allocate memory for MSFP\n");
      return MS_GENERROR;
    }

    /* Redirect the supplied pointer to the allocated params */
    *ppmsfp = msfp;

    msfp->fp            = NULL;
    msfp->filename[0]   = '\0';
    msfp->rawrec        = NULL;
    msfp->readlen       = 0;
    msfp->readoffset    = 0;
    msfp->packtype      = 0;
    msfp->packhdroffset = 0;
    msfp->filepos       = 0;
    msfp->filesize      = 0;
    msfp->recordcount   = 0;
  }

  /* When cleanup is requested */
  if (msfile == NULL)
  {
    msr_free (ppmsr);

    if (msfp->fp != NULL)
      fclose (msfp->fp);

    if (msfp->rawrec != NULL)
      free (msfp->rawrec);

    /* If the file parameters are the global parameters reset them */
    if (*ppmsfp == &gMSFileParam)
    {
      gMSFileParam.fp            = NULL;
      gMSFileParam.filename[0]   = '\0';
      gMSFileParam.rawrec        = NULL;
      gMSFileParam.readlen       = 0;
      gMSFileParam.readoffset    = 0;
      gMSFileParam.packtype      = 0;
      gMSFileParam.packhdroffset = 0;
      gMSFileParam.filepos       = 0;
      gMSFileParam.filesize      = 0;
      gMSFileParam.recordcount   = 0;
    }
    /* Otherwise free the MSFileParam */
    else
    {
      free (*ppmsfp);
      *ppmsfp = NULL;
    }

    return MS_NOERROR;
  }

  /* Allocate reading buffer */
  if (msfp->rawrec == NULL)
  {
    if (!(msfp->rawrec = (char *)malloc (MAXRECLEN)))
    {
      ms_log (2, "ms_readmsr_main(): Cannot allocate memory for read buffer\n");
      return MS_GENERROR;
    }
  }

  /* Sanity check: track if we are reading the same file */
  if (msfp->fp && strncmp (msfile, msfp->filename, sizeof (msfp->filename)))
  {
    ms_log (2, "ms_readmsr_main() called with a different file name without being reset\n");

    /* Close previous file and reset needed variables */
    if (msfp->fp != NULL)
      fclose (msfp->fp);

    msfp->fp            = NULL;
    msfp->readlen       = 0;
    msfp->readoffset    = 0;
    msfp->packtype      = 0;
    msfp->packhdroffset = 0;
    msfp->filepos       = 0;
    msfp->filesize      = 0;
    msfp->recordcount   = 0;
  }

  /* Open the file if needed, redirect to stdin if file is "-" */
  if (msfp->fp == NULL)
  {
    /* Store the filename for tracking */
    strncpy (msfp->filename, msfile, sizeof (msfp->filename) - 1);
    msfp->filename[sizeof (msfp->filename) - 1] = '\0';

    if (strcmp (msfile, "-") == 0)
    {
      msfp->fp = stdin;
    }
    else
    {
      if ((msfp->fp = fopen (msfile, "rb")) == NULL)
      {
        ms_log (2, "Cannot open file: %s (%s)\n", msfile, strerror (errno));
        msr_free (ppmsr);

        return MS_GENERROR;
      }
      else
      {
        /* Determine file size */
        struct stat sbuf;

        if (fstat (fileno (msfp->fp), &sbuf))
        {
          ms_log (2, "Cannot open file: %s (%s)\n", msfile, strerror (errno));
          msr_free (ppmsr);

          return MS_GENERROR;
        }

        msfp->filesize = sbuf.st_size;
      }
    }
  }

  /* Seek to a specified offset if requested */
  if (fpos != NULL && *fpos < 0)
  {
    /* Only try to seek in real files, not stdin */
    if (msfp->fp != stdin)
    {
      if (lmp_fseeko (msfp->fp, *fpos * -1, SEEK_SET))
      {
        ms_log (2, "Cannot seek in file: %s (%s)\n", msfile, strerror (errno));

        return MS_GENERROR;
      }

      msfp->filepos    = *fpos * -1;
      msfp->readlen    = 0;
      msfp->readoffset = 0;
    }
  }

  /* Zero the last record indicator */
  if (last)
    *last = 0;

  /* Read data and search for records */
  for (;;)
  {
    /* Read more data into buffer if not at EOF and buffer has less than MINRECLEN
       * or more data is needed for the current record detected in buffer. */
    if (!feof (msfp->fp) && (MSFPBUFLEN (msfp) < MINRECLEN || parseval > 0))
    {
      /* Reset offsets if no unprocessed data in buffer */
      if (MSFPBUFLEN (msfp) <= 0)
      {
        msfp->readlen    = 0;
        msfp->readoffset = 0;
      }
      /* Otherwise shift existing data to beginning of buffer */
      else if (msfp->readoffset > 0)
      {
        ms_shift_msfp (msfp, msfp->readoffset);
      }

      /* Determine read size */
      readsize = (MAXRECLEN - msfp->readlen);

      /* Read data into record buffer */
      readcount = ms_fread (msfp->rawrec + msfp->readlen, 1, readsize, msfp->fp);

      if (readcount != readsize)
      {
        if (!feof (msfp->fp))
        {
          ms_log (2, "Short read of %d bytes starting from %" PRId64 "\n",
                  readsize, msfp->filepos);
          retcode = MS_GENERROR;
          break;
        }
      }

      /* Update read buffer length */
      msfp->readlen += readcount;

      /* File position corresponding to start of buffer; not strictly necessary */
      if (msfp->fp != stdin)
        msfp->filepos = lmp_ftello (msfp->fp) - msfp->readlen;
    }

    /* Test for packed file signature at the beginning of the file */
    if (msfp->filepos == 0 && *(MSFPREADPTR (msfp)) == 'P' && MSFPBUFLEN (msfp) >= 48)
    {
      msfp->packtype = 0;

      /* Determine pack type, the negative pack type indicates initial header */
      if (!memcmp ("PED", MSFPREADPTR (msfp), 3))
        msfp->packtype = -1;
      else if (!memcmp ("PSD", MSFPREADPTR (msfp), 3))
        msfp->packtype = -2;
      else if (!memcmp ("PLC", MSFPREADPTR (msfp), 3))
        msfp->packtype = -6;
      else if (!memcmp ("PQI", MSFPREADPTR (msfp), 3))
        msfp->packtype = -7;
      else if (!memcmp ("PLS", MSFPREADPTR (msfp), 3))
        msfp->packtype = -8;

      if (verbose > 0)
        ms_log (1, "Detected packed file (%3.3s: type %d)\n", MSFPREADPTR (msfp), -msfp->packtype);
    }

    /* Read pack headers, initial and subsequent headers including (ignored) chksum values */
    if (msfp->packtype && (msfp->packtype < 0 || msfp->filepos == msfp->packhdroffset) && MSFPBUFLEN (msfp) >= 48)
    {
      char hdrstr[30];
      int64_t datasize;

      /* Determine bytes to skip before header: either initial ID block or type-specific chksum block */
      packskipsize = (msfp->packtype < 0) ? 10 : packtypes[msfp->packtype][2];

      if (msfp->packtype < 0)
        msfp->packtype = -msfp->packtype;

      /* Read pack length from pack header accounting for bytes that should be skipped */
      memset (hdrstr, 0, sizeof (hdrstr));
      memcpy (hdrstr, MSFPREADPTR (msfp) + (packtypes[msfp->packtype][0] + packskipsize - packtypes[msfp->packtype][1]),
              packtypes[msfp->packtype][1]);
      sscanf (hdrstr, " %" SCNd64, &datasize);
      packdatasize = (off_t)datasize;

      /* Next pack header = File position + skipsize + header size + data size
	   * This offset is actually to the data block chksum which is skipped by the logic above,
	   * the next pack header should directly follow the chksum. */
      msfp->packhdroffset = msfp->filepos + packskipsize + packtypes[msfp->packtype][0] + packdatasize;

      if (verbose > 1)
        ms_log (1, "Read packed file header at offset %" PRId64 " (%d bytes follow), chksum offset: %" PRId64 "\n",
                (msfp->filepos + packskipsize), packdatasize,
                msfp->packhdroffset);

      /* Shift buffer to new reading offset (aligns records in buffer) */
      ms_shift_msfp (msfp, msfp->readoffset + (packskipsize + packtypes[msfp->packtype][0]));
    } /* End of packed header processing */

    /* Check for match if selections are supplied and pack header was read, */
    /* only when enough data is in buffer and not reading from stdin pipe */
    if (selections && msfp->packtype && packdatasize && MSFPBUFLEN (msfp) >= 48 && msfp->fp != stdin)
    {
      char srcname[100];

      ms_recsrcname (MSFPREADPTR (msfp), srcname, 1);

      if (!ms_matchselect (selections, srcname, HPTERROR, HPTERROR, NULL))
      {
        /* Update read position if next section is in buffer */
        if (MSFPBUFLEN (msfp) >= (msfp->packhdroffset - msfp->filepos))
        {
          if (verbose > 1)
          {
            ms_log (1, "Skipping (jump) packed section for %s (%d bytes) starting at offset %" PRId64 "\n",
                    srcname, (msfp->packhdroffset - msfp->filepos), msfp->filepos);
          }

          msfp->readoffset += (msfp->packhdroffset - msfp->filepos);
          msfp->filepos = msfp->packhdroffset;
          packdatasize  = 0;
        }

        /* Otherwise seek to next pack header and reset reading position */
        else
        {
          if (verbose > 1)
          {
            ms_log (1, "Skipping (seek) packed section for %s (%d bytes) starting at offset %" PRId64 "\n",
                    srcname, (msfp->packhdroffset - msfp->filepos), msfp->filepos);
          }

          if (lmp_fseeko (msfp->fp, msfp->packhdroffset, SEEK_SET))
          {
            ms_log (2, "Cannot seek in file: %s (%s)\n", msfile, strerror (errno));

            return MS_GENERROR;
            break;
          }

          msfp->filepos    = msfp->packhdroffset;
          msfp->readlen    = 0;
          msfp->readoffset = 0;
          packdatasize     = 0;
        }

        /* Return to top of loop for proper pack header handling */
        continue;
      }
    } /* End of selection processing */

    /* Attempt to parse record from buffer */
    if (MSFPBUFLEN (msfp) >= MINRECLEN)
    {
      int parselen = MSFPBUFLEN (msfp);

      /* Limit the parse length to offset of pack header if present in the buffer */
      if (msfp->packhdroffset && msfp->packhdroffset < (msfp->filepos + MSFPBUFLEN (msfp)))
        parselen = msfp->packhdroffset - msfp->filepos;

      parseval = msr_parse (MSFPREADPTR (msfp), parselen, ppmsr, reclen, dataflag, verbose);

      /* Record detected and parsed */
      if (parseval == 0)
      {
        if (verbose > 1)
          ms_log (1, "Read record length of %d bytes\n", (*ppmsr)->reclen);

        /* Test if this is the last record if file size is known (not pipe) */
        if (last && msfp->filesize)
          if ((msfp->filesize - (msfp->filepos + (*ppmsr)->reclen)) < MINRECLEN)
            *last = 1;

        /* Return file position for this record */
        if (fpos)
          *fpos = msfp->filepos;

        /* Update reading offset, file position and record count */
        msfp->readoffset += (*ppmsr)->reclen;
        msfp->filepos += (*ppmsr)->reclen;
        msfp->recordcount++;

        retcode = MS_NOERROR;
        break;
      }
      else if (parseval < 0)
      {
        /* Skip non-data if requested */
        if (skipnotdata)
        {
          if (verbose > 1)
          {
            if (MS_ISVALIDBLANK ((char *)MSFPREADPTR (msfp)))
              ms_log (1, "Skipped %d bytes of blank/noise record at byte offset %" PRId64 "\n",
                      MINRECLEN, msfp->filepos);
            else
              ms_log (1, "Skipped %d bytes of non-data record at byte offset %" PRId64 "\n",
                      MINRECLEN, msfp->filepos);
          }

          /* Skip MINRECLEN bytes, update reading offset and file position */
          msfp->readoffset += MINRECLEN;
          msfp->filepos += MINRECLEN;
        }
        /* Parsing errors */
        else
        {
          ms_log (2, "Cannot detect record at byte offset %" PRId64 ": %s\n",
                  msfp->filepos, msfile);

          /* Print common errors and raw details if verbose */
          ms_parse_raw (MSFPREADPTR (msfp), MSFPBUFLEN (msfp), verbose, -1);

          retcode = parseval;
          break;
        }
      }
      else /* parseval > 0 (found record but need more data) */
      {
        /* Determine implied record length if needed */
        int32_t impreclen = reclen;

        /* Check for parse hints that are larger than MAXRECLEN */
        if ((MSFPBUFLEN (msfp) + parseval) > MAXRECLEN)
        {
          if (skipnotdata)
          {
            /* Skip MINRECLEN bytes, update reading offset and file position */
            msfp->readoffset += MINRECLEN;
            msfp->filepos += MINRECLEN;
          }
          else
          {
            retcode = MS_OUTOFRANGE;
            break;
          }
        }

        /* Pack header check, if pack header offset is within buffer */
        else if (impreclen <= 0 && msfp->packhdroffset &&
                 msfp->packhdroffset < (msfp->filepos + MSFPBUFLEN (msfp)))
        {
          impreclen = msfp->packhdroffset - msfp->filepos;

          /* Check that record length is within range and a power of 2.
		   * Power of two if (X & (X - 1)) == 0 */
          if (impreclen >= MINRECLEN && impreclen <= MAXRECLEN &&
              (impreclen & (impreclen - 1)) == 0)
          {
            /* Set the record length implied by the next pack header */
            reclen = impreclen;
          }
          else
          {
            ms_log (1, "Implied record length (%d) is invalid\n", impreclen);

            retcode = MS_NOTSEED;
            break;
          }
        }

        /* End of file check */
        else if (impreclen <= 0 && feof (msfp->fp))
        {
          impreclen = msfp->filesize - msfp->filepos;

          /* Check that record length is within range and a power of 2.
		   * Power of two if (X & (X - 1)) == 0 */
          if (impreclen >= MINRECLEN && impreclen <= MAXRECLEN &&
              (impreclen & (impreclen - 1)) == 0)
          {
            /* Set the record length implied by the end of the file */
            reclen = impreclen;
          }
          /* Otherwise a trucated record */
          else
          {
            if (verbose)
            {
              if (msfp->filesize)
                ms_log (1, "Truncated record at byte offset %" PRId64 ", filesize %d: %s\n",
                        msfp->filepos, msfp->filesize, msfile);
              else
                ms_log (1, "Truncated record at byte offset %" PRId64 "\n",
                        msfp->filepos);
            }

            retcode = MS_ENDOFFILE;
            break;
          }
        }
      }
    } /* End of record detection */

    /* Finished when within MINRECLEN from EOF and buffer less than MINRECLEN */
    if ((msfp->filesize - msfp->filepos) < MINRECLEN && MSFPBUFLEN (msfp) < MINRECLEN)
    {
      if (msfp->recordcount == 0 && msfp->packtype == 0)
      {
        if (verbose > 0)
          ms_log (2, "%s: No data records read, not SEED?\n", msfile);
        retcode = MS_NOTSEED;
      }
      else
      {
        retcode = MS_ENDOFFILE;
      }

      break;
    }
  } /* End of reading, record detection and parsing loop */

  /* Cleanup target MSRecord if returning an error */
  if (retcode != MS_NOERROR)
  {
    msr_free (ppmsr);
  }

  return retcode;
} /* End of ms_readmsr_main() */
Ejemplo n.º 2
0
/********************************************************************
 * ms_detect:
 *
 * Determine SEED data record length with the following steps:
 *
 * 1) determine that the buffer contains a SEED data record by
 * verifying known signatures (fields with known limited values)
 *
 * 2) search the record up to recbuflen bytes for a 1000 blockette.
 *
 * 3) If no blockette 1000 is found search at 256-byte offsets for the
 * fixed section of the next header or blank/noise record, thereby
 * implying the record length.
 *
 * Returns:
 * -1 : data record not detected or error
 *  0 : data record detected but could not determine length
 * >0 : size of the record in bytes
 *********************************************************************/
int
ms_detect ( const char *record, int recbuflen )
{
  uint16_t blkt_offset;    /* Byte offset for next blockette */
  uint8_t swapflag  = 0;   /* Byte swapping flag */
  uint8_t foundlen = 0;    /* Found record length */
  int32_t reclen = -1;     /* Size of record in bytes */
  
  uint16_t blkt_type;
  uint16_t next_blkt;
  
  struct fsdh_s *fsdh;
  struct blkt_1000_s *blkt_1000;
  const char *nextfsdh;
  
  /* Buffer must be at least 48 bytes (the fixed section) */
  if ( recbuflen < 48 )
    return -1;

  /* Check for valid fixed section of header */
  if ( ! MS_ISVALIDHEADER(record) )
    return -1;
  
  fsdh = (struct fsdh_s *) record;
  
  /* Check to see if byte swapping is needed by checking for sane year */
  if ( (fsdh->start_time.year < 1900) ||
       (fsdh->start_time.year > 2050) )
    swapflag = 1;
  
  blkt_offset = fsdh->blockette_offset;
  
  /* Swap order of blkt_offset if needed */
  if ( swapflag ) ms_gswap2 (&blkt_offset);
  
  /* Loop through blockettes as long as number is non-zero and viable */
  while ( blkt_offset != 0 &&
	  blkt_offset <= recbuflen )
    {
      memcpy (&blkt_type, record + blkt_offset, 2);
      memcpy (&next_blkt, record + blkt_offset + 2, 2);
      
      if ( swapflag )
	{
	  ms_gswap2 (&blkt_type);
	  ms_gswap2 (&next_blkt);
	}
      
      /* Found a 1000 blockette, not truncated */
      if ( blkt_type == 1000  &&
	   (int)(blkt_offset + 4 + sizeof(struct blkt_1000_s)) <= recbuflen )
	{
          blkt_1000 = (struct blkt_1000_s *) (record + blkt_offset + 4);
	  
          foundlen = 1;
	  
          /* Calculate record size in bytes as 2^(blkt_1000->reclen) */
	  reclen = (unsigned int) 1 << blkt_1000->reclen;
	  
	  break;
        }
      
      /* Saftey check for invalid offset */
      if ( next_blkt != 0 && next_blkt < blkt_offset )
	{
	  ms_log (2, "Invalid blockette offset (%d) less than current offset (%d)\n",
		  next_blkt, blkt_offset);
	  return -1;
	}
      
      blkt_offset = next_blkt;
    }
  
  /* If record length was not determined by a 1000 blockette scan the buffer
   * and search for the next record */
  if ( reclen == -1 )
    {
      nextfsdh = record + 256;
      
      /* Check for record header or blank/noise record at 256 byte offsets */
      while ( ((nextfsdh - record) + 48) < recbuflen )
	{
	  if ( MS_ISVALIDHEADER(nextfsdh) || MS_ISVALIDBLANK(nextfsdh) )
            {
	      foundlen = 1;
	      reclen = nextfsdh - record;
	      break;
	    }
	  
	  nextfsdh += 256;
	}
    }
  
  if ( ! foundlen )
    return 0;
  else
    return reclen;
}  /* End of ms_detect() */