/**********************************************************************
* ms_readmsr_main:
*
* This routine will open and read, with subsequent calls, all
* Mini-SEED records in specified file.
*
* All static file reading parameters are stored in a MSFileParam
* struct and returned (via a pointer to a pointer) for the calling
* routine to use in subsequent calls. A MSFileParam struct will be
* allocated if necessary. This routine is thread safe and can be
* used to read multiple files in parallel as long as the file reading
* parameters are managed appropriately.
*
* If reclen is 0 or negative the length of every record is
* automatically detected. For auto detection of record length the
* record must include a 1000 blockette or be followed by a valid
* record header or end of file.
*
* If *fpos is not NULL it will be updated to reflect the file
* position (offset from the beginning in bytes) from where the
* returned record was read. As a special case, if *fpos is not NULL
* and the value it points to is less than 0 this will be interpreted
* as a (positive) starting offset from which to begin reading data;
* this feature does not work with packed files.
*
* If *last is not NULL it will be set to 1 when the last record in
* the file is being returned, otherwise it will be 0.
*
* If the skipnotdata flag is true any data chunks read that do not
* have valid data record indicators (D, R, Q, M, etc.) will be skipped.
*
* dataflag will be passed directly to msr_unpack().
*
* If a Selections list is supplied it will be used to determine when
* a section of data in a packed file may be skipped, packed files are
* internal to the IRIS DMC.
*
* After reading all the records in a file the controlling program
* should call it one last time with msfile set to NULL. This will
* close the file and free allocated memory.
*
* Returns MS_NOERROR and populates an MSRecord struct at *ppmsr on
* successful read, returns MS_ENDOFFILE on EOF, otherwise returns a
* libmseed error code (listed in libmseed.h) and *ppmsr is set to
* NULL.
*********************************************************************/
int
ms_readmsr_main (MSFileParam **ppmsfp, MSRecord **ppmsr, const char *msfile,
int reclen, off_t *fpos, int *last, flag skipnotdata,
flag dataflag, Selections *selections, flag verbose)
{
MSFileParam *msfp;
off_t packdatasize = 0;
int packskipsize;
int parseval = 0;
int readsize = 0;
int readcount = 0;
int retcode = MS_NOERROR;
if (!ppmsr)
return MS_GENERROR;
if (!ppmsfp)
return MS_GENERROR;
msfp = *ppmsfp;
/* Initialize the file read parameters if needed */
if (!msfp)
{
msfp = (MSFileParam *)malloc (sizeof (MSFileParam));
if (msfp == NULL)
{
ms_log (2, "ms_readmsr_main(): Cannot allocate memory for MSFP\n");
return MS_GENERROR;
}
/* Redirect the supplied pointer to the allocated params */
*ppmsfp = msfp;
msfp->fp = NULL;
msfp->filename[0] = '\0';
msfp->rawrec = NULL;
msfp->readlen = 0;
msfp->readoffset = 0;
msfp->packtype = 0;
msfp->packhdroffset = 0;
msfp->filepos = 0;
msfp->filesize = 0;
msfp->recordcount = 0;
}
/* When cleanup is requested */
if (msfile == NULL)
{
msr_free (ppmsr);
if (msfp->fp != NULL)
fclose (msfp->fp);
if (msfp->rawrec != NULL)
free (msfp->rawrec);
/* If the file parameters are the global parameters reset them */
if (*ppmsfp == &gMSFileParam)
{
gMSFileParam.fp = NULL;
gMSFileParam.filename[0] = '\0';
gMSFileParam.rawrec = NULL;
gMSFileParam.readlen = 0;
gMSFileParam.readoffset = 0;
gMSFileParam.packtype = 0;
gMSFileParam.packhdroffset = 0;
gMSFileParam.filepos = 0;
gMSFileParam.filesize = 0;
gMSFileParam.recordcount = 0;
}
/* Otherwise free the MSFileParam */
else
{
free (*ppmsfp);
*ppmsfp = NULL;
}
return MS_NOERROR;
}
/* Allocate reading buffer */
if (msfp->rawrec == NULL)
{
if (!(msfp->rawrec = (char *)malloc (MAXRECLEN)))
{
ms_log (2, "ms_readmsr_main(): Cannot allocate memory for read buffer\n");
return MS_GENERROR;
}
}
/* Sanity check: track if we are reading the same file */
if (msfp->fp && strncmp (msfile, msfp->filename, sizeof (msfp->filename)))
{
ms_log (2, "ms_readmsr_main() called with a different file name without being reset\n");
/* Close previous file and reset needed variables */
if (msfp->fp != NULL)
fclose (msfp->fp);
msfp->fp = NULL;
msfp->readlen = 0;
msfp->readoffset = 0;
msfp->packtype = 0;
msfp->packhdroffset = 0;
msfp->filepos = 0;
msfp->filesize = 0;
msfp->recordcount = 0;
}
/* Open the file if needed, redirect to stdin if file is "-" */
if (msfp->fp == NULL)
{
/* Store the filename for tracking */
strncpy (msfp->filename, msfile, sizeof (msfp->filename) - 1);
msfp->filename[sizeof (msfp->filename) - 1] = '\0';
if (strcmp (msfile, "-") == 0)
{
msfp->fp = stdin;
}
else
{
if ((msfp->fp = fopen (msfile, "rb")) == NULL)
{
ms_log (2, "Cannot open file: %s (%s)\n", msfile, strerror (errno));
msr_free (ppmsr);
return MS_GENERROR;
}
else
{
/* Determine file size */
struct stat sbuf;
if (fstat (fileno (msfp->fp), &sbuf))
{
ms_log (2, "Cannot open file: %s (%s)\n", msfile, strerror (errno));
msr_free (ppmsr);
return MS_GENERROR;
}
msfp->filesize = sbuf.st_size;
}
}
}
/* Seek to a specified offset if requested */
if (fpos != NULL && *fpos < 0)
{
/* Only try to seek in real files, not stdin */
if (msfp->fp != stdin)
{
if (lmp_fseeko (msfp->fp, *fpos * -1, SEEK_SET))
{
ms_log (2, "Cannot seek in file: %s (%s)\n", msfile, strerror (errno));
return MS_GENERROR;
}
msfp->filepos = *fpos * -1;
msfp->readlen = 0;
msfp->readoffset = 0;
}
}
/* Zero the last record indicator */
if (last)
*last = 0;
/* Read data and search for records */
for (;;)
{
/* Read more data into buffer if not at EOF and buffer has less than MINRECLEN
* or more data is needed for the current record detected in buffer. */
if (!feof (msfp->fp) && (MSFPBUFLEN (msfp) < MINRECLEN || parseval > 0))
{
/* Reset offsets if no unprocessed data in buffer */
if (MSFPBUFLEN (msfp) <= 0)
{
msfp->readlen = 0;
msfp->readoffset = 0;
}
/* Otherwise shift existing data to beginning of buffer */
else if (msfp->readoffset > 0)
{
ms_shift_msfp (msfp, msfp->readoffset);
}
/* Determine read size */
readsize = (MAXRECLEN - msfp->readlen);
/* Read data into record buffer */
readcount = ms_fread (msfp->rawrec + msfp->readlen, 1, readsize, msfp->fp);
if (readcount != readsize)
{
if (!feof (msfp->fp))
{
ms_log (2, "Short read of %d bytes starting from %" PRId64 "\n",
readsize, msfp->filepos);
retcode = MS_GENERROR;
break;
}
}
/* Update read buffer length */
msfp->readlen += readcount;
/* File position corresponding to start of buffer; not strictly necessary */
if (msfp->fp != stdin)
msfp->filepos = lmp_ftello (msfp->fp) - msfp->readlen;
}
/* Test for packed file signature at the beginning of the file */
if (msfp->filepos == 0 && *(MSFPREADPTR (msfp)) == 'P' && MSFPBUFLEN (msfp) >= 48)
{
msfp->packtype = 0;
/* Determine pack type, the negative pack type indicates initial header */
if (!memcmp ("PED", MSFPREADPTR (msfp), 3))
msfp->packtype = -1;
else if (!memcmp ("PSD", MSFPREADPTR (msfp), 3))
msfp->packtype = -2;
else if (!memcmp ("PLC", MSFPREADPTR (msfp), 3))
msfp->packtype = -6;
else if (!memcmp ("PQI", MSFPREADPTR (msfp), 3))
msfp->packtype = -7;
else if (!memcmp ("PLS", MSFPREADPTR (msfp), 3))
msfp->packtype = -8;
if (verbose > 0)
ms_log (1, "Detected packed file (%3.3s: type %d)\n", MSFPREADPTR (msfp), -msfp->packtype);
}
/* Read pack headers, initial and subsequent headers including (ignored) chksum values */
if (msfp->packtype && (msfp->packtype < 0 || msfp->filepos == msfp->packhdroffset) && MSFPBUFLEN (msfp) >= 48)
{
char hdrstr[30];
int64_t datasize;
/* Determine bytes to skip before header: either initial ID block or type-specific chksum block */
packskipsize = (msfp->packtype < 0) ? 10 : packtypes[msfp->packtype][2];
if (msfp->packtype < 0)
msfp->packtype = -msfp->packtype;
/* Read pack length from pack header accounting for bytes that should be skipped */
memset (hdrstr, 0, sizeof (hdrstr));
memcpy (hdrstr, MSFPREADPTR (msfp) + (packtypes[msfp->packtype][0] + packskipsize - packtypes[msfp->packtype][1]),
packtypes[msfp->packtype][1]);
sscanf (hdrstr, " %" SCNd64, &datasize);
packdatasize = (off_t)datasize;
/* Next pack header = File position + skipsize + header size + data size
* This offset is actually to the data block chksum which is skipped by the logic above,
* the next pack header should directly follow the chksum. */
msfp->packhdroffset = msfp->filepos + packskipsize + packtypes[msfp->packtype][0] + packdatasize;
if (verbose > 1)
ms_log (1, "Read packed file header at offset %" PRId64 " (%d bytes follow), chksum offset: %" PRId64 "\n",
(msfp->filepos + packskipsize), packdatasize,
msfp->packhdroffset);
/* Shift buffer to new reading offset (aligns records in buffer) */
ms_shift_msfp (msfp, msfp->readoffset + (packskipsize + packtypes[msfp->packtype][0]));
} /* End of packed header processing */
/* Check for match if selections are supplied and pack header was read, */
/* only when enough data is in buffer and not reading from stdin pipe */
if (selections && msfp->packtype && packdatasize && MSFPBUFLEN (msfp) >= 48 && msfp->fp != stdin)
{
char srcname[100];
ms_recsrcname (MSFPREADPTR (msfp), srcname, 1);
if (!ms_matchselect (selections, srcname, HPTERROR, HPTERROR, NULL))
{
/* Update read position if next section is in buffer */
if (MSFPBUFLEN (msfp) >= (msfp->packhdroffset - msfp->filepos))
{
if (verbose > 1)
{
ms_log (1, "Skipping (jump) packed section for %s (%d bytes) starting at offset %" PRId64 "\n",
srcname, (msfp->packhdroffset - msfp->filepos), msfp->filepos);
}
msfp->readoffset += (msfp->packhdroffset - msfp->filepos);
msfp->filepos = msfp->packhdroffset;
packdatasize = 0;
}
/* Otherwise seek to next pack header and reset reading position */
else
{
if (verbose > 1)
{
ms_log (1, "Skipping (seek) packed section for %s (%d bytes) starting at offset %" PRId64 "\n",
srcname, (msfp->packhdroffset - msfp->filepos), msfp->filepos);
}
if (lmp_fseeko (msfp->fp, msfp->packhdroffset, SEEK_SET))
{
ms_log (2, "Cannot seek in file: %s (%s)\n", msfile, strerror (errno));
return MS_GENERROR;
break;
}
msfp->filepos = msfp->packhdroffset;
msfp->readlen = 0;
msfp->readoffset = 0;
packdatasize = 0;
}
/* Return to top of loop for proper pack header handling */
continue;
}
} /* End of selection processing */
/* Attempt to parse record from buffer */
if (MSFPBUFLEN (msfp) >= MINRECLEN)
{
int parselen = MSFPBUFLEN (msfp);
/* Limit the parse length to offset of pack header if present in the buffer */
if (msfp->packhdroffset && msfp->packhdroffset < (msfp->filepos + MSFPBUFLEN (msfp)))
parselen = msfp->packhdroffset - msfp->filepos;
parseval = msr_parse (MSFPREADPTR (msfp), parselen, ppmsr, reclen, dataflag, verbose);
/* Record detected and parsed */
if (parseval == 0)
{
if (verbose > 1)
ms_log (1, "Read record length of %d bytes\n", (*ppmsr)->reclen);
/* Test if this is the last record if file size is known (not pipe) */
if (last && msfp->filesize)
if ((msfp->filesize - (msfp->filepos + (*ppmsr)->reclen)) < MINRECLEN)
*last = 1;
/* Return file position for this record */
if (fpos)
*fpos = msfp->filepos;
/* Update reading offset, file position and record count */
msfp->readoffset += (*ppmsr)->reclen;
msfp->filepos += (*ppmsr)->reclen;
msfp->recordcount++;
retcode = MS_NOERROR;
break;
}
else if (parseval < 0)
{
/* Skip non-data if requested */
if (skipnotdata)
{
if (verbose > 1)
{
if (MS_ISVALIDBLANK ((char *)MSFPREADPTR (msfp)))
ms_log (1, "Skipped %d bytes of blank/noise record at byte offset %" PRId64 "\n",
MINRECLEN, msfp->filepos);
else
ms_log (1, "Skipped %d bytes of non-data record at byte offset %" PRId64 "\n",
MINRECLEN, msfp->filepos);
}
/* Skip MINRECLEN bytes, update reading offset and file position */
msfp->readoffset += MINRECLEN;
msfp->filepos += MINRECLEN;
}
/* Parsing errors */
else
{
ms_log (2, "Cannot detect record at byte offset %" PRId64 ": %s\n",
msfp->filepos, msfile);
/* Print common errors and raw details if verbose */
ms_parse_raw (MSFPREADPTR (msfp), MSFPBUFLEN (msfp), verbose, -1);
retcode = parseval;
break;
}
}
else /* parseval > 0 (found record but need more data) */
{
/* Determine implied record length if needed */
int32_t impreclen = reclen;
/* Check for parse hints that are larger than MAXRECLEN */
if ((MSFPBUFLEN (msfp) + parseval) > MAXRECLEN)
{
if (skipnotdata)
{
/* Skip MINRECLEN bytes, update reading offset and file position */
msfp->readoffset += MINRECLEN;
msfp->filepos += MINRECLEN;
}
else
{
retcode = MS_OUTOFRANGE;
break;
}
}
/* Pack header check, if pack header offset is within buffer */
else if (impreclen <= 0 && msfp->packhdroffset &&
msfp->packhdroffset < (msfp->filepos + MSFPBUFLEN (msfp)))
{
impreclen = msfp->packhdroffset - msfp->filepos;
/* Check that record length is within range and a power of 2.
* Power of two if (X & (X - 1)) == 0 */
if (impreclen >= MINRECLEN && impreclen <= MAXRECLEN &&
(impreclen & (impreclen - 1)) == 0)
{
/* Set the record length implied by the next pack header */
reclen = impreclen;
}
else
{
ms_log (1, "Implied record length (%d) is invalid\n", impreclen);
retcode = MS_NOTSEED;
break;
}
}
/* End of file check */
else if (impreclen <= 0 && feof (msfp->fp))
{
impreclen = msfp->filesize - msfp->filepos;
/* Check that record length is within range and a power of 2.
* Power of two if (X & (X - 1)) == 0 */
if (impreclen >= MINRECLEN && impreclen <= MAXRECLEN &&
(impreclen & (impreclen - 1)) == 0)
{
/* Set the record length implied by the end of the file */
reclen = impreclen;
}
/* Otherwise a trucated record */
else
{
if (verbose)
{
if (msfp->filesize)
ms_log (1, "Truncated record at byte offset %" PRId64 ", filesize %d: %s\n",
msfp->filepos, msfp->filesize, msfile);
else
ms_log (1, "Truncated record at byte offset %" PRId64 "\n",
msfp->filepos);
}
retcode = MS_ENDOFFILE;
break;
}
}
}
} /* End of record detection */
/* Finished when within MINRECLEN from EOF and buffer less than MINRECLEN */
if ((msfp->filesize - msfp->filepos) < MINRECLEN && MSFPBUFLEN (msfp) < MINRECLEN)
{
if (msfp->recordcount == 0 && msfp->packtype == 0)
{
if (verbose > 0)
ms_log (2, "%s: No data records read, not SEED?\n", msfile);
retcode = MS_NOTSEED;
}
else
{
retcode = MS_ENDOFFILE;
}
break;
}
} /* End of reading, record detection and parsing loop */
/* Cleanup target MSRecord if returning an error */
if (retcode != MS_NOERROR)
{
msr_free (ppmsr);
}
return retcode;
} /* End of ms_readmsr_main() */
/********************************************************************
* ms_detect:
*
* Determine SEED data record length with the following steps:
*
* 1) determine that the buffer contains a SEED data record by
* verifying known signatures (fields with known limited values)
*
* 2) search the record up to recbuflen bytes for a 1000 blockette.
*
* 3) If no blockette 1000 is found search at 256-byte offsets for the
* fixed section of the next header or blank/noise record, thereby
* implying the record length.
*
* Returns:
* -1 : data record not detected or error
* 0 : data record detected but could not determine length
* >0 : size of the record in bytes
*********************************************************************/
int
ms_detect ( const char *record, int recbuflen )
{
uint16_t blkt_offset; /* Byte offset for next blockette */
uint8_t swapflag = 0; /* Byte swapping flag */
uint8_t foundlen = 0; /* Found record length */
int32_t reclen = -1; /* Size of record in bytes */
uint16_t blkt_type;
uint16_t next_blkt;
struct fsdh_s *fsdh;
struct blkt_1000_s *blkt_1000;
const char *nextfsdh;
/* Buffer must be at least 48 bytes (the fixed section) */
if ( recbuflen < 48 )
return -1;
/* Check for valid fixed section of header */
if ( ! MS_ISVALIDHEADER(record) )
return -1;
fsdh = (struct fsdh_s *) record;
/* Check to see if byte swapping is needed by checking for sane year */
if ( (fsdh->start_time.year < 1900) ||
(fsdh->start_time.year > 2050) )
swapflag = 1;
blkt_offset = fsdh->blockette_offset;
/* Swap order of blkt_offset if needed */
if ( swapflag ) ms_gswap2 (&blkt_offset);
/* Loop through blockettes as long as number is non-zero and viable */
while ( blkt_offset != 0 &&
blkt_offset <= recbuflen )
{
memcpy (&blkt_type, record + blkt_offset, 2);
memcpy (&next_blkt, record + blkt_offset + 2, 2);
if ( swapflag )
{
ms_gswap2 (&blkt_type);
ms_gswap2 (&next_blkt);
}
/* Found a 1000 blockette, not truncated */
if ( blkt_type == 1000 &&
(int)(blkt_offset + 4 + sizeof(struct blkt_1000_s)) <= recbuflen )
{
blkt_1000 = (struct blkt_1000_s *) (record + blkt_offset + 4);
foundlen = 1;
/* Calculate record size in bytes as 2^(blkt_1000->reclen) */
reclen = (unsigned int) 1 << blkt_1000->reclen;
break;
}
/* Saftey check for invalid offset */
if ( next_blkt != 0 && next_blkt < blkt_offset )
{
ms_log (2, "Invalid blockette offset (%d) less than current offset (%d)\n",
next_blkt, blkt_offset);
return -1;
}
blkt_offset = next_blkt;
}
/* If record length was not determined by a 1000 blockette scan the buffer
* and search for the next record */
if ( reclen == -1 )
{
nextfsdh = record + 256;
/* Check for record header or blank/noise record at 256 byte offsets */
while ( ((nextfsdh - record) + 48) < recbuflen )
{
if ( MS_ISVALIDHEADER(nextfsdh) || MS_ISVALIDBLANK(nextfsdh) )
{
foundlen = 1;
reclen = nextfsdh - record;
break;
}
nextfsdh += 256;
}
}
if ( ! foundlen )
return 0;
else
return reclen;
} /* End of ms_detect() */