int main (int argc, char **argv) { struct filelink *flp; MSRecord *msr = 0; MSTraceList *mstl = 0; FILE *bfp = 0; FILE *ofp = 0; int retcode = MS_NOERROR; char envvariable[100]; int dataflag = 0; long long int totalrecs = 0; long long int totalsamps = 0; long long int totalfiles = 0; off_t filepos = 0; char srcname[50]; char stime[30]; /* Set default error message prefix */ ms_loginit (NULL, NULL, NULL, "ERROR: "); /* Process given parameters (command line and parameter file) */ if ( processparam (argc, argv) < 0 ) return 1; /* Setup encoding environment variable if specified, ugly kludge */ if ( encodingstr ) { snprintf (envvariable, sizeof(envvariable), "UNPACK_DATA_FORMAT=%s", encodingstr); if ( putenv (envvariable) ) { ms_log (2, "Cannot set environment variable UNPACK_DATA_FORMAT\n"); return 1; } } /* Open the integer output file if specified */ if ( binfile ) { if ( strcmp (binfile, "-") == 0 ) { bfp = stdout; } else if ( (bfp = fopen (binfile, "wb")) == NULL ) { ms_log (2, "Cannot open binary data output file: %s (%s)\n", binfile, strerror(errno)); return 1; } } /* Open the output file if specified */ if ( outfile ) { if ( strcmp (outfile, "-") == 0 ) { ofp = stdout; } else if ( (ofp = fopen (outfile, "wb")) == NULL ) { ms_log (2, "Cannot open output file: %s (%s)\n", outfile, strerror(errno)); return 1; } } if ( printdata || binfile ) dataflag = 1; if ( tracegapsum || tracegaponly ) mstl = mstl_init (NULL); flp = filelist; while ( flp != 0 ) { if ( verbose >= 2 ) { if ( flp->offset ) ms_log (1, "Processing: %s (starting at byte %lld)\n", flp->filename, flp->offset); else ms_log (1, "Processing: %s\n", flp->filename); } /* Set starting byte offset if supplied as negative file position */ filepos = - flp->offset; /* Loop over the input file */ while ( reccntdown != 0 ) { if ( (retcode = ms_readmsr (&msr, flp->filename, reclen, &filepos, NULL, skipnotdata, 0, verbose)) != MS_NOERROR ) break; /* Check if record matches start/end time criteria */ if ( starttime != HPTERROR || endtime != HPTERROR ) { hptime_t recendtime = msr_endtime (msr); if ( starttime != HPTERROR && (msr->starttime < starttime && ! (msr->starttime <= starttime && recendtime >= starttime)) ) { if ( verbose >= 3 ) { msr_srcname (msr, srcname, 1); ms_hptime2seedtimestr (msr->starttime, stime, 1); ms_log (1, "Skipping (starttime) %s, %s\n", srcname, stime); } continue; } if ( endtime != HPTERROR && (recendtime > endtime && ! (msr->starttime <= endtime && recendtime >= endtime)) ) { if ( verbose >= 3 ) { msr_srcname (msr, srcname, 1); ms_hptime2seedtimestr (msr->starttime, stime, 1); ms_log (1, "Skipping (starttime) %s, %s\n", srcname, stime); } continue; } } if ( match || reject ) { /* Generate the srcname with the quality code */ msr_srcname (msr, srcname, 1); /* Check if record is matched by the match regex */ if ( match ) { if ( regexec ( match, srcname, 0, 0, 0) != 0 ) { if ( verbose >= 3 ) { ms_hptime2seedtimestr (msr->starttime, stime, 1); ms_log (1, "Skipping (match) %s, %s\n", srcname, stime); } continue; } } /* Check if record is rejected by the reject regex */ if ( reject ) { if ( regexec ( reject, srcname, 0, 0, 0) == 0 ) { if ( verbose >= 3 ) { ms_hptime2seedtimestr (msr->starttime, stime, 1); ms_log (1, "Skipping (reject) %s, %s\n", srcname, stime); } continue; } } } if ( reccntdown > 0 ) reccntdown--; totalrecs++; totalsamps += msr->samplecnt; if ( ! tracegaponly ) { if ( printoffset ) ms_log (0, "%-10lld", (long long) filepos); if ( printlatency ) ms_log (0, "%-10.6g secs ", msr_host_latency(msr)); if ( printraw ) ms_parse_raw (msr->record, msr->reclen, ppackets, -1); else msr_print (msr, ppackets); } if ( tracegapsum || tracegaponly ) mstl_addmsr (mstl, msr, dataquality, 1, timetol, sampratetol); if ( dataflag ) { /* Parse the record (again) and unpack the data */ int rv = msr_unpack (msr->record, msr->reclen, &msr, 1, verbose); if ( rv == MS_NOERROR && printdata && ! tracegaponly ) { int line, col, cnt, samplesize; int lines = (msr->numsamples / 6) + 1; void *sptr; if ( (samplesize = ms_samplesize(msr->sampletype)) == 0 ) { ms_log (2, "Unrecognized sample type: %c\n", msr->sampletype); } if ( msr->sampletype == 'a' ) { char *ascii = (char *)msr->datasamples; int length = msr->numsamples; ms_log (0, "ASCII Data:\n"); /* Print maximum log message segments */ while ( length > (MAX_LOG_MSG_LENGTH-1) ) { ms_log (0, "%.*s", (MAX_LOG_MSG_LENGTH-1), ascii); ascii += MAX_LOG_MSG_LENGTH-1; length -= MAX_LOG_MSG_LENGTH-1; } /* Print any remaining ASCII and add a newline */ if ( length > 0 ) { ms_log (0, "%.*s\n", length, ascii); } else { ms_log (0, "\n"); } } else for ( cnt = 0, line = 0; line < lines; line++ ) { for ( col = 0; col < 6 ; col ++ ) { if ( cnt < msr->numsamples ) { sptr = (char*)msr->datasamples + (cnt * samplesize); if ( msr->sampletype == 'i' ) ms_log (0, "%10d ", *(int32_t *)sptr); else if ( msr->sampletype == 'f' ) ms_log (0, "%10.8g ", *(float *)sptr); else if ( msr->sampletype == 'd' ) ms_log (0, "%10.10g ", *(double *)sptr); cnt++; } } ms_log (0, "\n"); /* If only printing the first 6 samples break out here */ if ( printdata == 1 ) break; } } if ( binfile ) { uint8_t samplesize = ms_samplesize (msr->sampletype); if ( samplesize ) { fwrite (msr->datasamples, samplesize, msr->numsamples, bfp); } else { ms_log (1, "Cannot write to binary file, unknown sample type: %c\n", msr->sampletype); } } } if ( outfile ) { fwrite (msr->record, 1, msr->reclen, ofp); } } /* Print error if not EOF and not counting down records */ if ( retcode != MS_ENDOFFILE && reccntdown != 0 ) { ms_log (2, "Cannot read %s: %s\n", flp->filename, ms_errorstr(retcode)); ms_readmsr (&msr, NULL, 0, NULL, NULL, 0, 0, 0); exit (1); } /* Make sure everything is cleaned up */ ms_readmsr (&msr, NULL, 0, NULL, NULL, 0, 0, 0); totalfiles++; flp = flp->next; } /* End of looping over file list */ if ( binfile ) fclose (bfp); if ( outfile ) fclose (ofp); if ( basicsum ) ms_log (0, "Files: %lld, Records: %lld, Samples: %lld\n", totalfiles, totalrecs, totalsamps); if ( tracegapsum || tracegaponly ) { if ( tracegapsum == 1 || tracegaponly == 1 ) { mstl_printtracelist (mstl, timeformat, 1, tracegaps); } if ( tracegapsum == 2 || tracegaponly == 2 ) { mstl_printgaplist (mstl, timeformat, mingapptr, maxgapptr); } if ( tracegaponly == 3 ) { mstl_printsynclist (mstl, NULL, 1); } } if ( mstl ) mstl_free (&mstl, 0); return 0; } /* End of main() */
/********************************************************************** * ms_readmsr_main: * * This routine will open and read, with subsequent calls, all * Mini-SEED records in specified file. * * All static file reading parameters are stored in a MSFileParam * struct and returned (via a pointer to a pointer) for the calling * routine to use in subsequent calls. A MSFileParam struct will be * allocated if necessary. This routine is thread safe and can be * used to read multiple files in parallel as long as the file reading * parameters are managed appropriately. * * If reclen is 0 or negative the length of every record is * automatically detected. For auto detection of record length the * record must include a 1000 blockette or be followed by a valid * record header or end of file. * * If *fpos is not NULL it will be updated to reflect the file * position (offset from the beginning in bytes) from where the * returned record was read. As a special case, if *fpos is not NULL * and the value it points to is less than 0 this will be interpreted * as a (positive) starting offset from which to begin reading data; * this feature does not work with packed files. * * If *last is not NULL it will be set to 1 when the last record in * the file is being returned, otherwise it will be 0. * * If the skipnotdata flag is true any data chunks read that do not * have valid data record indicators (D, R, Q, M, etc.) will be skipped. * * dataflag will be passed directly to msr_unpack(). * * If a Selections list is supplied it will be used to determine when * a section of data in a packed file may be skipped, packed files are * internal to the IRIS DMC. * * After reading all the records in a file the controlling program * should call it one last time with msfile set to NULL. This will * close the file and free allocated memory. * * Returns MS_NOERROR and populates an MSRecord struct at *ppmsr on * successful read, returns MS_ENDOFFILE on EOF, otherwise returns a * libmseed error code (listed in libmseed.h) and *ppmsr is set to * NULL. *********************************************************************/ int ms_readmsr_main (MSFileParam **ppmsfp, MSRecord **ppmsr, const char *msfile, int reclen, off_t *fpos, int *last, flag skipnotdata, flag dataflag, Selections *selections, flag verbose) { MSFileParam *msfp; off_t packdatasize = 0; int packskipsize; int parseval = 0; int readsize = 0; int readcount = 0; int retcode = MS_NOERROR; if (!ppmsr) return MS_GENERROR; if (!ppmsfp) return MS_GENERROR; msfp = *ppmsfp; /* Initialize the file read parameters if needed */ if (!msfp) { msfp = (MSFileParam *)malloc (sizeof (MSFileParam)); if (msfp == NULL) { ms_log (2, "ms_readmsr_main(): Cannot allocate memory for MSFP\n"); return MS_GENERROR; } /* Redirect the supplied pointer to the allocated params */ *ppmsfp = msfp; msfp->fp = NULL; msfp->filename[0] = '\0'; msfp->rawrec = NULL; msfp->readlen = 0; msfp->readoffset = 0; msfp->packtype = 0; msfp->packhdroffset = 0; msfp->filepos = 0; msfp->filesize = 0; msfp->recordcount = 0; } /* When cleanup is requested */ if (msfile == NULL) { msr_free (ppmsr); if (msfp->fp != NULL) fclose (msfp->fp); if (msfp->rawrec != NULL) free (msfp->rawrec); /* If the file parameters are the global parameters reset them */ if (*ppmsfp == &gMSFileParam) { gMSFileParam.fp = NULL; gMSFileParam.filename[0] = '\0'; gMSFileParam.rawrec = NULL; gMSFileParam.readlen = 0; gMSFileParam.readoffset = 0; gMSFileParam.packtype = 0; gMSFileParam.packhdroffset = 0; gMSFileParam.filepos = 0; gMSFileParam.filesize = 0; gMSFileParam.recordcount = 0; } /* Otherwise free the MSFileParam */ else { free (*ppmsfp); *ppmsfp = NULL; } return MS_NOERROR; } /* Allocate reading buffer */ if (msfp->rawrec == NULL) { if (!(msfp->rawrec = (char *)malloc (MAXRECLEN))) { ms_log (2, "ms_readmsr_main(): Cannot allocate memory for read buffer\n"); return MS_GENERROR; } } /* Sanity check: track if we are reading the same file */ if (msfp->fp && strncmp (msfile, msfp->filename, sizeof (msfp->filename))) { ms_log (2, "ms_readmsr_main() called with a different file name without being reset\n"); /* Close previous file and reset needed variables */ if (msfp->fp != NULL) fclose (msfp->fp); msfp->fp = NULL; msfp->readlen = 0; msfp->readoffset = 0; msfp->packtype = 0; msfp->packhdroffset = 0; msfp->filepos = 0; msfp->filesize = 0; msfp->recordcount = 0; } /* Open the file if needed, redirect to stdin if file is "-" */ if (msfp->fp == NULL) { /* Store the filename for tracking */ strncpy (msfp->filename, msfile, sizeof (msfp->filename) - 1); msfp->filename[sizeof (msfp->filename) - 1] = '\0'; if (strcmp (msfile, "-") == 0) { msfp->fp = stdin; } else { if ((msfp->fp = fopen (msfile, "rb")) == NULL) { ms_log (2, "Cannot open file: %s (%s)\n", msfile, strerror (errno)); msr_free (ppmsr); return MS_GENERROR; } else { /* Determine file size */ struct stat sbuf; if (fstat (fileno (msfp->fp), &sbuf)) { ms_log (2, "Cannot open file: %s (%s)\n", msfile, strerror (errno)); msr_free (ppmsr); return MS_GENERROR; } msfp->filesize = sbuf.st_size; } } } /* Seek to a specified offset if requested */ if (fpos != NULL && *fpos < 0) { /* Only try to seek in real files, not stdin */ if (msfp->fp != stdin) { if (lmp_fseeko (msfp->fp, *fpos * -1, SEEK_SET)) { ms_log (2, "Cannot seek in file: %s (%s)\n", msfile, strerror (errno)); return MS_GENERROR; } msfp->filepos = *fpos * -1; msfp->readlen = 0; msfp->readoffset = 0; } } /* Zero the last record indicator */ if (last) *last = 0; /* Read data and search for records */ for (;;) { /* Read more data into buffer if not at EOF and buffer has less than MINRECLEN * or more data is needed for the current record detected in buffer. */ if (!feof (msfp->fp) && (MSFPBUFLEN (msfp) < MINRECLEN || parseval > 0)) { /* Reset offsets if no unprocessed data in buffer */ if (MSFPBUFLEN (msfp) <= 0) { msfp->readlen = 0; msfp->readoffset = 0; } /* Otherwise shift existing data to beginning of buffer */ else if (msfp->readoffset > 0) { ms_shift_msfp (msfp, msfp->readoffset); } /* Determine read size */ readsize = (MAXRECLEN - msfp->readlen); /* Read data into record buffer */ readcount = ms_fread (msfp->rawrec + msfp->readlen, 1, readsize, msfp->fp); if (readcount != readsize) { if (!feof (msfp->fp)) { ms_log (2, "Short read of %d bytes starting from %" PRId64 "\n", readsize, msfp->filepos); retcode = MS_GENERROR; break; } } /* Update read buffer length */ msfp->readlen += readcount; /* File position corresponding to start of buffer; not strictly necessary */ if (msfp->fp != stdin) msfp->filepos = lmp_ftello (msfp->fp) - msfp->readlen; } /* Test for packed file signature at the beginning of the file */ if (msfp->filepos == 0 && *(MSFPREADPTR (msfp)) == 'P' && MSFPBUFLEN (msfp) >= 48) { msfp->packtype = 0; /* Determine pack type, the negative pack type indicates initial header */ if (!memcmp ("PED", MSFPREADPTR (msfp), 3)) msfp->packtype = -1; else if (!memcmp ("PSD", MSFPREADPTR (msfp), 3)) msfp->packtype = -2; else if (!memcmp ("PLC", MSFPREADPTR (msfp), 3)) msfp->packtype = -6; else if (!memcmp ("PQI", MSFPREADPTR (msfp), 3)) msfp->packtype = -7; else if (!memcmp ("PLS", MSFPREADPTR (msfp), 3)) msfp->packtype = -8; if (verbose > 0) ms_log (1, "Detected packed file (%3.3s: type %d)\n", MSFPREADPTR (msfp), -msfp->packtype); } /* Read pack headers, initial and subsequent headers including (ignored) chksum values */ if (msfp->packtype && (msfp->packtype < 0 || msfp->filepos == msfp->packhdroffset) && MSFPBUFLEN (msfp) >= 48) { char hdrstr[30]; int64_t datasize; /* Determine bytes to skip before header: either initial ID block or type-specific chksum block */ packskipsize = (msfp->packtype < 0) ? 10 : packtypes[msfp->packtype][2]; if (msfp->packtype < 0) msfp->packtype = -msfp->packtype; /* Read pack length from pack header accounting for bytes that should be skipped */ memset (hdrstr, 0, sizeof (hdrstr)); memcpy (hdrstr, MSFPREADPTR (msfp) + (packtypes[msfp->packtype][0] + packskipsize - packtypes[msfp->packtype][1]), packtypes[msfp->packtype][1]); sscanf (hdrstr, " %" SCNd64, &datasize); packdatasize = (off_t)datasize; /* Next pack header = File position + skipsize + header size + data size * This offset is actually to the data block chksum which is skipped by the logic above, * the next pack header should directly follow the chksum. */ msfp->packhdroffset = msfp->filepos + packskipsize + packtypes[msfp->packtype][0] + packdatasize; if (verbose > 1) ms_log (1, "Read packed file header at offset %" PRId64 " (%d bytes follow), chksum offset: %" PRId64 "\n", (msfp->filepos + packskipsize), packdatasize, msfp->packhdroffset); /* Shift buffer to new reading offset (aligns records in buffer) */ ms_shift_msfp (msfp, msfp->readoffset + (packskipsize + packtypes[msfp->packtype][0])); } /* End of packed header processing */ /* Check for match if selections are supplied and pack header was read, */ /* only when enough data is in buffer and not reading from stdin pipe */ if (selections && msfp->packtype && packdatasize && MSFPBUFLEN (msfp) >= 48 && msfp->fp != stdin) { char srcname[100]; ms_recsrcname (MSFPREADPTR (msfp), srcname, 1); if (!ms_matchselect (selections, srcname, HPTERROR, HPTERROR, NULL)) { /* Update read position if next section is in buffer */ if (MSFPBUFLEN (msfp) >= (msfp->packhdroffset - msfp->filepos)) { if (verbose > 1) { ms_log (1, "Skipping (jump) packed section for %s (%d bytes) starting at offset %" PRId64 "\n", srcname, (msfp->packhdroffset - msfp->filepos), msfp->filepos); } msfp->readoffset += (msfp->packhdroffset - msfp->filepos); msfp->filepos = msfp->packhdroffset; packdatasize = 0; } /* Otherwise seek to next pack header and reset reading position */ else { if (verbose > 1) { ms_log (1, "Skipping (seek) packed section for %s (%d bytes) starting at offset %" PRId64 "\n", srcname, (msfp->packhdroffset - msfp->filepos), msfp->filepos); } if (lmp_fseeko (msfp->fp, msfp->packhdroffset, SEEK_SET)) { ms_log (2, "Cannot seek in file: %s (%s)\n", msfile, strerror (errno)); return MS_GENERROR; break; } msfp->filepos = msfp->packhdroffset; msfp->readlen = 0; msfp->readoffset = 0; packdatasize = 0; } /* Return to top of loop for proper pack header handling */ continue; } } /* End of selection processing */ /* Attempt to parse record from buffer */ if (MSFPBUFLEN (msfp) >= MINRECLEN) { int parselen = MSFPBUFLEN (msfp); /* Limit the parse length to offset of pack header if present in the buffer */ if (msfp->packhdroffset && msfp->packhdroffset < (msfp->filepos + MSFPBUFLEN (msfp))) parselen = msfp->packhdroffset - msfp->filepos; parseval = msr_parse (MSFPREADPTR (msfp), parselen, ppmsr, reclen, dataflag, verbose); /* Record detected and parsed */ if (parseval == 0) { if (verbose > 1) ms_log (1, "Read record length of %d bytes\n", (*ppmsr)->reclen); /* Test if this is the last record if file size is known (not pipe) */ if (last && msfp->filesize) if ((msfp->filesize - (msfp->filepos + (*ppmsr)->reclen)) < MINRECLEN) *last = 1; /* Return file position for this record */ if (fpos) *fpos = msfp->filepos; /* Update reading offset, file position and record count */ msfp->readoffset += (*ppmsr)->reclen; msfp->filepos += (*ppmsr)->reclen; msfp->recordcount++; retcode = MS_NOERROR; break; } else if (parseval < 0) { /* Skip non-data if requested */ if (skipnotdata) { if (verbose > 1) { if (MS_ISVALIDBLANK ((char *)MSFPREADPTR (msfp))) ms_log (1, "Skipped %d bytes of blank/noise record at byte offset %" PRId64 "\n", MINRECLEN, msfp->filepos); else ms_log (1, "Skipped %d bytes of non-data record at byte offset %" PRId64 "\n", MINRECLEN, msfp->filepos); } /* Skip MINRECLEN bytes, update reading offset and file position */ msfp->readoffset += MINRECLEN; msfp->filepos += MINRECLEN; } /* Parsing errors */ else { ms_log (2, "Cannot detect record at byte offset %" PRId64 ": %s\n", msfp->filepos, msfile); /* Print common errors and raw details if verbose */ ms_parse_raw (MSFPREADPTR (msfp), MSFPBUFLEN (msfp), verbose, -1); retcode = parseval; break; } } else /* parseval > 0 (found record but need more data) */ { /* Determine implied record length if needed */ int32_t impreclen = reclen; /* Check for parse hints that are larger than MAXRECLEN */ if ((MSFPBUFLEN (msfp) + parseval) > MAXRECLEN) { if (skipnotdata) { /* Skip MINRECLEN bytes, update reading offset and file position */ msfp->readoffset += MINRECLEN; msfp->filepos += MINRECLEN; } else { retcode = MS_OUTOFRANGE; break; } } /* Pack header check, if pack header offset is within buffer */ else if (impreclen <= 0 && msfp->packhdroffset && msfp->packhdroffset < (msfp->filepos + MSFPBUFLEN (msfp))) { impreclen = msfp->packhdroffset - msfp->filepos; /* Check that record length is within range and a power of 2. * Power of two if (X & (X - 1)) == 0 */ if (impreclen >= MINRECLEN && impreclen <= MAXRECLEN && (impreclen & (impreclen - 1)) == 0) { /* Set the record length implied by the next pack header */ reclen = impreclen; } else { ms_log (1, "Implied record length (%d) is invalid\n", impreclen); retcode = MS_NOTSEED; break; } } /* End of file check */ else if (impreclen <= 0 && feof (msfp->fp)) { impreclen = msfp->filesize - msfp->filepos; /* Check that record length is within range and a power of 2. * Power of two if (X & (X - 1)) == 0 */ if (impreclen >= MINRECLEN && impreclen <= MAXRECLEN && (impreclen & (impreclen - 1)) == 0) { /* Set the record length implied by the end of the file */ reclen = impreclen; } /* Otherwise a trucated record */ else { if (verbose) { if (msfp->filesize) ms_log (1, "Truncated record at byte offset %" PRId64 ", filesize %d: %s\n", msfp->filepos, msfp->filesize, msfile); else ms_log (1, "Truncated record at byte offset %" PRId64 "\n", msfp->filepos); } retcode = MS_ENDOFFILE; break; } } } } /* End of record detection */ /* Finished when within MINRECLEN from EOF and buffer less than MINRECLEN */ if ((msfp->filesize - msfp->filepos) < MINRECLEN && MSFPBUFLEN (msfp) < MINRECLEN) { if (msfp->recordcount == 0 && msfp->packtype == 0) { if (verbose > 0) ms_log (2, "%s: No data records read, not SEED?\n", msfile); retcode = MS_NOTSEED; } else { retcode = MS_ENDOFFILE; } break; } } /* End of reading, record detection and parsing loop */ /* Cleanup target MSRecord if returning an error */ if (retcode != MS_NOERROR) { msr_free (ppmsr); } return retcode; } /* End of ms_readmsr_main() */