// Frees a LinkedRecordList. The given Record is assumed to be the head of the
// list.
void
lrl_free(LinkedRecordList * lrl)
{
LinkedRecordList * next;
while ( lrl != NULL) {
next = lrl->next;
msr_free(&lrl->record);
free(lrl);
if (next == NULL) {
break;
}
lrl = next;
}
lrl = NULL;
}
bool MSeedWriter::write(IntegerMSeedRecord::SharedPtr_t sampleRange)
{
// наполнения хедера mseed
MSRecord* msr = msr_init(NULL);
// общие для записей данные
strcpy(msr->network, sampleRange->network().toLatin1().constData());
strcpy(msr->station, sampleRange->station().toLatin1().constData());
strcpy(msr->location, sampleRange->location().toLatin1().constData());
strcpy(msr->channel, sampleRange->channelName().toLatin1().constData());
msr->samprate = sampleRange->samplingRateHz();
msr->reclen = _recordLength;
msr->record = NULL;
msr->encoding = _encoding; // compression
msr->byteorder = 1; // big endian byte order
BTime btime = dateTimeToBTime(sampleRange->startTime());
msr->starttime = ms_btime2hptime(&btime);
msr->sampletype = 'i'; // declare type to be 32-bit integers
msr->datasamples = sampleRange->data().data();
msr->numsamples = sampleRange->data().size();
flag verbose = _verbose;
_packedSamples = 0;
_packedRecords = msr_pack(msr, &binaryStreamRecorder, _binaryStream.get(), &_packedSamples, 1, verbose);
if (_packedRecords == -1)
{
return false;
}
msr->datasamples = NULL;
msr_free(&msr);
ms_log(0, "Packed %d samples into %d records\n", _packedSamples, _packedRecords);
return true;
}
int
main (int argc, char **argv)
{
MSRecord *msr = 0;
MSTraceGroup *mstg = 0;
MSTrace *mst;
int retcode;
int totalrecs = 0;
int totalsamps = 0;
int packedsamples;
int packedrecords;
int lastrecord;
int iseqnum = 1;
#ifndef WIN32
/* Signal handling, use POSIX calls with standardized semantics */
struct sigaction sa;
sa.sa_flags = SA_RESTART;
sigemptyset (&sa.sa_mask);
sa.sa_handler = term_handler;
sigaction (SIGINT, &sa, NULL);
sigaction (SIGQUIT, &sa, NULL);
sigaction (SIGTERM, &sa, NULL);
sa.sa_handler = SIG_IGN;
sigaction (SIGHUP, &sa, NULL);
sigaction (SIGPIPE, &sa, NULL);
#endif
/* Process given parameters (command line and parameter file) */
if (parameter_proc (argc, argv) < 0)
return -1;
/* Setup input encoding format if specified */
if ( encodingstr )
{
int inputencoding = strtoul (encodingstr, NULL, 10);
if ( inputencoding == 0 && errno == EINVAL )
{
ms_log (2, "Error parsing input encoding format: %s\n", encodingstr);
return -1;
}
MS_UNPACKENCODINGFORMAT (inputencoding);
}
/* Init MSTraceGroup */
mstg = mst_initgroup (mstg);
/* Loop over the input file */
while ( (retcode = ms_readmsr (&msr, inputfile, reclen, NULL, &lastrecord,
1, 1, verbose)) == MS_NOERROR )
{
totalrecs++;
totalsamps += msr->samplecnt;
msr_print (msr, ppackets);
if ( packreclen >= 0 )
msr->reclen = packreclen;
else
packreclen = msr->reclen;
if ( packencoding >= 0 )
msr->encoding = packencoding;
else
packencoding = msr->encoding;
if ( byteorder >= 0 )
msr->byteorder = byteorder;
else
byteorder = msr->byteorder;
/* After unpacking the record, the start time in msr->starttime
is a potentially corrected start time, if correction has been
applied make sure the correction bit flag is set as it will
be used as a packing template. */
if ( msr->fsdh->time_correct && ! (msr->fsdh->act_flags & 0x02) )
{
ms_log (1, "Setting time correction applied flag for %s_%s_%s_%s\n",
msr->network, msr->station, msr->location, msr->channel);
msr->fsdh->act_flags |= 0x02;
}
/* If no samples in the record just pack the header */
if ( outfile && msr->numsamples == 0 )
{
msr_pack_header (msr, 1, verbose);
record_handler (msr->record, msr->reclen, NULL);
}
/* Pack each record individually */
else if ( outfile && ! tracepack )
{
msr->sequence_number = iseqnum;
packedrecords = msr_pack (msr, &record_handler, NULL, &packedsamples, 1, verbose);
if ( packedrecords == -1 )
ms_log (2, "Cannot pack records\n");
else
ms_log (1, "Packed %d records\n", packedrecords);
iseqnum = msr->sequence_number;
}
/* Pack records from a MSTraceGroup */
else if ( outfile && tracepack )
{
mst = mst_addmsrtogroup (mstg, msr, 0, -1.0, -1.0);
if ( ! mst )
{
ms_log (2, "Error adding MSRecord to MStrace!\n");
break;
}
/* Reset sequence number and free previous template */
if ( mst->prvtptr )
{
MSRecord *tmsr = (MSRecord *) mst->prvtptr;
/* Retain sequence number from previous template */
msr->sequence_number = tmsr->sequence_number;
msr_free (&tmsr);
}
else
{
msr->sequence_number = 1;
}
/* Copy MSRecord and store as template */
mst->prvtptr = msr_duplicate (msr, 0);
if ( ! mst->prvtptr )
{
ms_log (2, "Error duplicating MSRecord for template!\n");
break;
}
/* Pack traces based on selected method */
packedrecords = 0;
if ( tracepack == 1 )
{
mst = mstg->traces;
while ( mst )
{
packedrecords += mst_pack (mst, &record_handler, NULL, packreclen,
packencoding, byteorder, &packedsamples,
lastrecord, verbose, (MSRecord *)mst->prvtptr);
mst = mst->next;
}
ms_log (1, "Packed %d records\n", packedrecords);
}
if ( tracepack == 2 && lastrecord )
{
mst = mstg->traces;
while ( mst )
{
packedrecords += mst_pack (mst, &record_handler, NULL, packreclen,
packencoding, byteorder, &packedsamples,
lastrecord, verbose, (MSRecord *)mst->prvtptr);
mst = mst->next;
}
ms_log (1, "Packed %d records\n", packedrecords);
}
}
}
if ( retcode != MS_ENDOFFILE )
ms_log (2, "Error reading %s: %s\n", inputfile, ms_errorstr(retcode));
/* Make sure everything is cleaned up */
ms_readmsr (&msr, NULL, 0, NULL, NULL, 0, 0, 0);
mst_freegroup (&mstg);
if ( outfile )
fclose (outfile);
getchar();
return 0;
} /* End of main() */
// Function that reads from a MiniSEED binary file from a char buffer and
// returns a LinkedIDList.
LinkedIDList *
readMSEEDBuffer (char *mseed, int buflen, Selections *selections, flag
unpack_data, int reclen, flag verbose, flag details,
int header_byteorder, long (*allocData) (int, char),
void (*diag_print) (char*), void (*log_print) (char*))
{
int retcode = 0;
int retval = 0;
flag swapflag = 0;
// current offset of mseed char pointer
int offset = 0;
// Unpack without reading the data first
flag dataflag = 0;
// the timing_qual of BLK 1001
uint8_t timing_qual = 0xFF;
// the calibration type, availability of BLK 300, 310, 320, 390, 395
int8_t calibration_type = -1;
// Init all the pointers to NULL. Most compilers should do this anyway.
LinkedIDList * idListHead = NULL;
LinkedIDList * idListCurrent = NULL;
LinkedIDList * idListLast = NULL;
MSRecord *msr = NULL;
ContinuousSegment * segmentCurrent = NULL;
hptime_t lastgap = 0;
hptime_t hptimetol = 0;
hptime_t nhptimetol = 0;
long data_offset;
LinkedRecordList *recordHead = NULL;
LinkedRecordList *recordPrevious = NULL;
LinkedRecordList *recordCurrent = NULL;
int datasize;
int record_count = 0;
// A negative verbosity suppressed as much as possible.
if (verbose < 0) {
ms_loginit(&empty_print, NULL, &empty_print, NULL);
}
else {
ms_loginit(log_print, "INFO: ", diag_print, "ERROR: ");
}
if (header_byteorder >= 0) {
// Enforce little endian.
if (header_byteorder == 0) {
MS_UNPACKHEADERBYTEORDER(0);
}
// Enforce big endian.
else {
MS_UNPACKHEADERBYTEORDER(1);
}
}
else {
MS_UNPACKHEADERBYTEORDER(-1);
}
//
// Read all records and save them in a linked list.
//
while (offset < buflen) {
msr = msr_init(NULL);
if ( msr == NULL ) {
ms_log (2, "readMSEEDBuffer(): Error initializing msr\n");
return NULL;
}
if (verbose > 1) {
ms_log(0, "readMSEEDBuffer(): calling msr_parse with "
"mseed+offset=%d+%d, buflen=%d, reclen=%d, dataflag=%d, verbose=%d\n",
mseed, offset, buflen, reclen, dataflag, verbose);
}
// If the record length is given, make sure at least that amount of data is available.
if (reclen != -1) {
if (offset + reclen > buflen) {
ms_log(1, "readMSEEDBuffer(): Last reclen exceeds buflen, skipping.\n");
msr_free(&msr);
break;
}
}
// Otherwise assume the smallest possible record length and assure that enough
// data is present.
else {
if (offset + 256 > buflen) {
ms_log(1, "readMSEEDBuffer(): Last record only has %i byte(s) which "
"is not enough to constitute a full SEED record. Corrupt data? "
"Record will be skipped.\n", buflen - offset);
msr_free(&msr);
break;
}
}
// Pass (buflen - offset) because msr_parse() expects only a single record. This
// way libmseed can take care to not overstep bounds.
retcode = msr_parse ( (mseed+offset), buflen - offset, &msr, reclen, dataflag, verbose);
if (retcode != MS_NOERROR) {
switch ( retcode ) {
case MS_ENDOFFILE:
ms_log(1, "readMSEEDBuffer(): Unexpected end of file when "
"parsing record starting at offset %d. The rest "
"of the file will not be read.\n", offset);
break;
case MS_GENERROR:
ms_log(1, "readMSEEDBuffer(): Generic error when parsing "
"record starting at offset %d. The rest of the "
"file will not be read.\n", offset);
break;
case MS_NOTSEED:
ms_log(1, "readMSEEDBuffer(): Record starting at offset "
"%d is not valid SEED. The rest of the file "
"will not be read.\n", offset);
break;
case MS_WRONGLENGTH:
ms_log(1, "readMSEEDBuffer(): Length of data read was not "
"correct when parsing record starting at "
"offset %d. The rest of the file will not be "
"read.\n", offset);
break;
case MS_OUTOFRANGE:
ms_log(1, "readMSEEDBuffer(): SEED record length out of "
"range for record starting at offset %d. The "
"rest of the file will not be read.\n", offset);
break;
case MS_UNKNOWNFORMAT:
ms_log(1, "readMSEEDBuffer(): Unknown data encoding "
"format for record starting at offset %d. The "
"rest of the file will not be read.\n", offset);
break;
case MS_STBADCOMPFLAG:
ms_log(1, "readMSEEDBuffer(): Invalid STEIM compression "
"flag(s) in record starting at offset %d. The "
"rest of the file will not be read.\n", offset);
break;
default:
ms_log(1, "readMSEEDBuffer(): Unknown error '%d' in "
"record starting at offset %d. The rest of the "
"file will not be read.\n", retcode, offset);
break;
}
msr_free(&msr);
break;
}
if (offset + msr->reclen > buflen) {
ms_log(1, "readMSEEDBuffer(): Last msr->reclen exceeds buflen, skipping.\n");
msr_free(&msr);
break;
}
// Test against selections if supplied
if ( selections ) {
char srcname[50];
hptime_t endtime;
msr_srcname (msr, srcname, 1);
endtime = msr_endtime (msr);
if ( ms_matchselect (selections, srcname, msr->starttime, endtime, NULL) == NULL ) {
// Add the record length for the next iteration
offset += msr->reclen;
// Free record.
msr_free(&msr);
continue;
}
}
record_count += 1;
recordCurrent = lrl_init ();
// Append to linked record list if one exists.
if ( recordHead != NULL ) {
recordPrevious->next = recordCurrent;
recordCurrent->previous = recordPrevious;
recordCurrent->next = NULL;
recordPrevious = recordCurrent;
}
// Otherwise create a new one.
else {
recordHead = recordCurrent;
recordCurrent->previous = NULL;
recordPrevious = recordCurrent;
}
recordCurrent->record = msr;
// Determine the byte order swapflag only for the very first record.
// The byte order should not change within the file.
// XXX: Maybe check for every record?
if (swapflag <= 0) {
// Returns 0 if the host is little endian, otherwise 1.
flag bigendianhost = ms_bigendianhost();
// Set the swapbyteflag if it is needed.
if ( msr->Blkt1000 != 0) {
/* If BE host and LE data need swapping */
if ( bigendianhost && msr->byteorder == 0 ) {
swapflag = 1;
}
/* If LE host and BE data (or bad byte order value) need swapping */
if ( !bigendianhost && msr->byteorder > 0 ) {
swapflag = 1;
}
}
}
// Actually unpack the data if the flag is not set.
if (unpack_data != 0) {
retval = msr_unpack_data (msr, swapflag, verbose);
}
if ( retval > 0 ) {
msr->numsamples = retval;
}
// Add the record length for the next iteration
offset += msr->reclen;
}
// Return empty id list if no records could be found.
if (record_count == 0) {
idListHead = lil_init();
return idListHead;
}
// All records that match the selection are now stored in a LinkedRecordList
// that starts at recordHead. The next step is to sort them by matching ids
// and then by time.
recordCurrent = recordHead;
while (recordCurrent != NULL) {
// Check if the ID of the record is already available and if not create a
// new one.
// Start with the last id as it is most likely to be the correct one.
idListCurrent = idListLast;
while (idListCurrent != NULL) {
if (strcmp(idListCurrent->network, recordCurrent->record->network) == 0 &&
strcmp(idListCurrent->station, recordCurrent->record->station) == 0 &&
strcmp(idListCurrent->location, recordCurrent->record->location) == 0 &&
strcmp(idListCurrent->channel, recordCurrent->record->channel) == 0 &&
idListCurrent->dataquality == recordCurrent->record->dataquality) {
break;
}
else {
idListCurrent = idListCurrent->previous;
}
}
// Create a new id list if one is needed.
if (idListCurrent == NULL) {
idListCurrent = lil_init();
idListCurrent->previous = idListLast;
if (idListLast != NULL) {
idListLast->next = idListCurrent;
}
idListLast = idListCurrent;
if (idListHead == NULL) {
idListHead = idListCurrent;
}
// Set the IdList attributes.
strcpy(idListCurrent->network, recordCurrent->record->network);
strcpy(idListCurrent->station, recordCurrent->record->station);
strcpy(idListCurrent->location, recordCurrent->record->location);
strcpy(idListCurrent->channel, recordCurrent->record->channel);
idListCurrent->dataquality = recordCurrent->record->dataquality;
}
// Now check if the current record fits exactly to the end of the last
// segment of the current id. If not create a new segment. Therefore
// if records with the same id are in wrong order a new segment will be
// created. This is on purpose.
segmentCurrent = idListCurrent->lastSegment;
if (segmentCurrent != NULL) {
hptimetol = (hptime_t) (0.5 * segmentCurrent->hpdelta);
nhptimetol = ( hptimetol ) ? -hptimetol : 0;
lastgap = recordCurrent->record->starttime - segmentCurrent->endtime - segmentCurrent->hpdelta;
}
if (details == 1) {
/* extract information on calibration BLKs */
calibration_type = -1;
if (recordCurrent->record->blkts) {
BlktLink *cur_blkt = recordCurrent->record->blkts;
while (cur_blkt) {
switch (cur_blkt->blkt_type) {
case 300:
calibration_type = 1;
break;
case 310:
calibration_type = 2;
break;
case 320:
calibration_type = 3;
break;
case 390:
calibration_type = 4;
break;
case 395:
calibration_type = -2;
break;
default:
break;
}
cur_blkt = cur_blkt->next;
}
}
/* extract information based on timing quality */
timing_qual = 0xFF;
if (recordCurrent->record->Blkt1001 != 0) {
timing_qual = recordCurrent->record->Blkt1001->timing_qual;
}
}
if ( segmentCurrent != NULL &&
segmentCurrent->sampletype == recordCurrent->record->sampletype &&
// Test the default sample rate tolerance: abs(1-sr1/sr2) < 0.0001
MS_ISRATETOLERABLE (segmentCurrent->samprate, recordCurrent->record->samprate) &&
// Check if the times are within the time tolerance
lastgap <= hptimetol && lastgap >= nhptimetol &&
segmentCurrent->timing_qual == timing_qual &&
segmentCurrent->calibration_type == calibration_type) {
recordCurrent->previous = segmentCurrent->lastRecord;
segmentCurrent->lastRecord = segmentCurrent->lastRecord->next = recordCurrent;
segmentCurrent->samplecnt += recordCurrent->record->samplecnt;
segmentCurrent->endtime = msr_endtime(recordCurrent->record);
}
// Otherwise create a new segment and add the current record.
else {
segmentCurrent = seg_init();
segmentCurrent->previous = idListCurrent->lastSegment;
if (idListCurrent->lastSegment != NULL) {
idListCurrent->lastSegment->next = segmentCurrent;
}
else {
idListCurrent->firstSegment = segmentCurrent;
}
idListCurrent->lastSegment = segmentCurrent;
segmentCurrent->starttime = recordCurrent->record->starttime;
segmentCurrent->endtime = msr_endtime(recordCurrent->record);
segmentCurrent->samprate = recordCurrent->record->samprate;
segmentCurrent->sampletype = recordCurrent->record->sampletype;
segmentCurrent->samplecnt = recordCurrent->record->samplecnt;
// Calculate high-precision sample period
segmentCurrent->hpdelta = (hptime_t) (( recordCurrent->record->samprate ) ?
(HPTMODULUS / recordCurrent->record->samprate) : 0.0);
segmentCurrent->timing_qual = timing_qual;
segmentCurrent->calibration_type = calibration_type;
segmentCurrent->firstRecord = segmentCurrent->lastRecord = recordCurrent;
recordCurrent->previous = NULL;
}
recordPrevious = recordCurrent->next;
recordCurrent->next = NULL;
recordCurrent = recordPrevious;
}
// Now loop over all segments, combine the records and free the msr
// structures.
idListCurrent = idListHead;
while (idListCurrent != NULL)
{
segmentCurrent = idListCurrent->firstSegment;
while (segmentCurrent != NULL) {
if (segmentCurrent->datasamples) {
free(segmentCurrent->datasamples);
}
// Allocate data via a callback function.
if (unpack_data != 0) {
segmentCurrent->datasamples = (void *) allocData(segmentCurrent->samplecnt, segmentCurrent->sampletype);
}
// Loop over all records, write the data to the buffer and free the msr structures.
recordCurrent = segmentCurrent->firstRecord;
data_offset = (long)(segmentCurrent->datasamples);
while (recordCurrent != NULL) {
datasize = recordCurrent->record->samplecnt * ms_samplesize(recordCurrent->record->sampletype);
memcpy((void *)data_offset, recordCurrent->record->datasamples, datasize);
// Free the record.
msr_free(&(recordCurrent->record));
// Increase the data_offset and the record.
data_offset += (long)datasize;
recordCurrent = recordCurrent->next;
}
segmentCurrent = segmentCurrent->next;
}
idListCurrent = idListCurrent->next;
}
return idListHead;
}
/***************************************************************************
* collect_and_write:
*
* Attempt to connect to a device, slows down the loop checking
* after 20 attempts with a larger delay to reduce pointless
* work being done.
*
* Returns 0 on success and -1 otherwise.
***************************************************************************/
int collect_and_write() {
int32_t idata[2000]; // enough space for data of 2 records
hptime_t hptime;
hptime_t start_hptime_est = 0;
hptime_t last_hptime;
DOUBLE dt, dt_est, sample_rate_est;
DOUBLE start_hptime_current, record_window_current, record_window_est;
DOUBLE prev_start_hptime_est = -1;
int n_start_hptime_est;
// debug
hptime_t start_hptime_nominal = 0;
hptime_t prev_start_next_hptime_est = 0;
double diff_end, diff_end_cumul = 0.0;
char seedtimestr[64];
// decay constant depends on required decay time and sample rate
//double decay_minutes = 60.0; // 1 hour
double decay_minutes = 1.0;
double decay_consant = 1.0 / (decay_minutes * 60.0 * (double) nominal_sample_rate);
// initialize last_hptime to current time
last_hptime = current_utc_hptime();
// initialize dt_est based on nominal sample rate
dt_est = (nominal_sample_rate == 80) ? 1.0 / SAMP_PER_SEC_80 : (nominal_sample_rate == 40) ? 1.0 / SAMP_PER_SEC_40 : 1.0 / SAMP_PER_SEC_20;
// ‘a’: 20.032 SPS
// ‘b’: 39.860 SPS
// ‘c’: 79.719 SPS
// initialize record_window_est based on nominal sample rate and record length
record_window_est = dt_est * num_samples_in_record;
if (DEBUG) {
logprintf(MSG_FLAG, "Initialize: last_hptime=%lld, dt_est=%lld, dt=%lf, dt_end=%lf, dt_end_cumul=%lf)\n",
last_hptime, dt_est, record_window_est);
}
int first = 1;
MSRecord *pmsrecord = msr_init(NULL);
strcpy(pmsrecord->network, station_network);
strcpy(pmsrecord->station, station_name);
strcpy(pmsrecord->location, "");
sprintf(pmsrecord->channel, "%s%s", channel_prefix, component);
pmsrecord->samprate = 1.0;
pmsrecord->reclen = SLRECSIZE;
pmsrecord->encoding = mswrite_data_encoding_type_code;
pmsrecord->byteorder = 1;
pmsrecord->datasamples = idata;
pmsrecord->numsamples = 0;
pmsrecord->sampletype = 'i';
while (1) {
// load data up to SLRECSIZE
long ivalue;
int nsamp = 0;
start_hptime_current = 0;
n_start_hptime_est = 0;
while (nsamp < num_samples_in_record) {
ivalue = read_next_value(&hptime, TIMEOUT_LARGE);
if (ivalue == READ_ERROR || ivalue < MIN_DATA || ivalue > MAX_DATA) {
logprintf(MSG_FLAG, "READ_ERROR: port=%s, nsamp=%d, ivalue=%ld\n", port_path, nsamp, ivalue);
pmsrecord->datasamples = NULL;
msr_free(&pmsrecord);
return (-1);
}
if (DEBUG && nsamp == 0) {
start_hptime_nominal = hptime;
}
idata[pmsrecord->numsamples + nsamp] = (int32_t) ivalue;
dt = (DOUBLE) (hptime - last_hptime) / (DOUBLE) HPTMODULUS;
last_hptime = hptime;
if (verbose > 3) {
logprintf(MSG_FLAG, "%d %ld %s (dt=%lf)\n", nsamp, ivalue, ms_hptime2seedtimestr(hptime, seedtimestr, 1), (double) dt);
}
// estimate start time and dt
// use only later samples in record since writing previous record may delay reading of first samples of this record
if (nsamp >= num_samples_in_record / 2) {
// 20131107 AJL - use all samples, may give better start time estimate, since buffering should compensate for any delay of first samples
//if (1) {
// start time estimate is timestamp of current data minus dt_est*nsamp
start_hptime_current += (hptime - (hptime_t) ((DOUBLE) 0.5 + dt_est * (DOUBLE) HPTMODULUS * (DOUBLE) nsamp));
n_start_hptime_est++;
// accumulate dt_est using low-pass filter
//dt_est = dt_est + (DOUBLE) decay_consant * (dt - dt_est);
}
nsamp++;
}
start_hptime_current /= n_start_hptime_est;
if (prev_start_hptime_est > 0) {
record_window_current = (DOUBLE) (start_hptime_current - prev_start_hptime_est) / (DOUBLE) HPTMODULUS;
} else {
record_window_current = record_window_est;
}
// accumulate record_window_est using low-pass filter
record_window_est = record_window_est + (DOUBLE) decay_consant * (record_window_current - record_window_est);
if (prev_start_hptime_est > 0) {
start_hptime_est = prev_start_hptime_est + (hptime_t) ((DOUBLE) 0.5 + record_window_est * (DOUBLE) HPTMODULUS);
} else {
start_hptime_est = start_hptime_current;
}
prev_start_hptime_est = start_hptime_est;
// test - truncate dt to 1/10000 s to match precision of miniseed btime
//logprintf(MSG_FLAG, "0 sample_rate_est=%lf (dt=%lfs)\n", (double) ((DOUBLE) 1.0 / dt_est), (double) dt_est);
dt_est = record_window_est / (DOUBLE) num_samples_in_record;
sample_rate_est = (DOUBLE) 1.0 / dt_est;
if (DEBUG) {
diff_end = (double) (start_hptime_est - prev_start_next_hptime_est) / (double) HPTMODULUS;
if (!first)
diff_end_cumul += diff_end;
logprintf(MSG_FLAG, "sample_rate_est=%lf (dt=%lfs)\n", (double) sample_rate_est, (double) dt_est);
logprintf(MSG_FLAG, "start_hptime_est=%lld, start_hptime_nominal=%lld, dt=%lf, dt_end=%lf, dt_end_cumul=%lf)\n", start_hptime_est, start_hptime_nominal,
(double) ((DOUBLE) (start_hptime_est - start_hptime_nominal) / (DOUBLE) HPTMODULUS), diff_end, diff_end_cumul);
prev_start_next_hptime_est = start_hptime_est + (hptime_t) ((DOUBLE) 0.5 + dt_est * (DOUBLE) HPTMODULUS * (DOUBLE) nsamp);
}
pmsrecord->starttime = start_hptime_est - (DOUBLE) HPTMODULUS * pmsrecord->numsamples / pmsrecord->samprate;
pmsrecord->samprate = mswrite_header_sample_rate > 0.0 ? mswrite_header_sample_rate : sample_rate_est;
pmsrecord->numsamples += nsamp;
int64_t npackedsamples = 0;
if (msr_pack(pmsrecord, record_handler, NULL, &npackedsamples, 0, verbose) < 0) {
logprintf(ERROR_FLAG, "Error encoding data!\n");
exit(1);
}
pmsrecord->numsamples -= npackedsamples;
memmove(&idata[0], &idata[npackedsamples], pmsrecord->numsamples * 4);
}
return (0);
}
/**********************************************************************
* ms_readmsr_main:
*
* This routine will open and read, with subsequent calls, all
* Mini-SEED records in specified file.
*
* All static file reading parameters are stored in a MSFileParam
* struct and returned (via a pointer to a pointer) for the calling
* routine to use in subsequent calls. A MSFileParam struct will be
* allocated if necessary. This routine is thread safe and can be
* used to read multiple files in parallel as long as the file reading
* parameters are managed appropriately.
*
* If reclen is 0 or negative the length of every record is
* automatically detected. For auto detection of record length the
* record must include a 1000 blockette or be followed by a valid
* record header or end of file.
*
* If *fpos is not NULL it will be updated to reflect the file
* position (offset from the beginning in bytes) from where the
* returned record was read. As a special case, if *fpos is not NULL
* and the value it points to is less than 0 this will be interpreted
* as a (positive) starting offset from which to begin reading data;
* this feature does not work with packed files.
*
* If *last is not NULL it will be set to 1 when the last record in
* the file is being returned, otherwise it will be 0.
*
* If the skipnotdata flag is true any data chunks read that do not
* have valid data record indicators (D, R, Q, M, etc.) will be skipped.
*
* dataflag will be passed directly to msr_unpack().
*
* If a Selections list is supplied it will be used to determine when
* a section of data in a packed file may be skipped, packed files are
* internal to the IRIS DMC.
*
* After reading all the records in a file the controlling program
* should call it one last time with msfile set to NULL. This will
* close the file and free allocated memory.
*
* Returns MS_NOERROR and populates an MSRecord struct at *ppmsr on
* successful read, returns MS_ENDOFFILE on EOF, otherwise returns a
* libmseed error code (listed in libmseed.h) and *ppmsr is set to
* NULL.
*********************************************************************/
int
ms_readmsr_main (MSFileParam **ppmsfp, MSRecord **ppmsr, const char *msfile,
int reclen, off_t *fpos, int *last, flag skipnotdata,
flag dataflag, Selections *selections, flag verbose)
{
MSFileParam *msfp;
off_t packdatasize = 0;
int packskipsize;
int parseval = 0;
int readsize = 0;
int readcount = 0;
int retcode = MS_NOERROR;
if (!ppmsr)
return MS_GENERROR;
if (!ppmsfp)
return MS_GENERROR;
msfp = *ppmsfp;
/* Initialize the file read parameters if needed */
if (!msfp)
{
msfp = (MSFileParam *)malloc (sizeof (MSFileParam));
if (msfp == NULL)
{
ms_log (2, "ms_readmsr_main(): Cannot allocate memory for MSFP\n");
return MS_GENERROR;
}
/* Redirect the supplied pointer to the allocated params */
*ppmsfp = msfp;
msfp->fp = NULL;
msfp->filename[0] = '\0';
msfp->rawrec = NULL;
msfp->readlen = 0;
msfp->readoffset = 0;
msfp->packtype = 0;
msfp->packhdroffset = 0;
msfp->filepos = 0;
msfp->filesize = 0;
msfp->recordcount = 0;
}
/* When cleanup is requested */
if (msfile == NULL)
{
msr_free (ppmsr);
if (msfp->fp != NULL)
fclose (msfp->fp);
if (msfp->rawrec != NULL)
free (msfp->rawrec);
/* If the file parameters are the global parameters reset them */
if (*ppmsfp == &gMSFileParam)
{
gMSFileParam.fp = NULL;
gMSFileParam.filename[0] = '\0';
gMSFileParam.rawrec = NULL;
gMSFileParam.readlen = 0;
gMSFileParam.readoffset = 0;
gMSFileParam.packtype = 0;
gMSFileParam.packhdroffset = 0;
gMSFileParam.filepos = 0;
gMSFileParam.filesize = 0;
gMSFileParam.recordcount = 0;
}
/* Otherwise free the MSFileParam */
else
{
free (*ppmsfp);
*ppmsfp = NULL;
}
return MS_NOERROR;
}
/* Allocate reading buffer */
if (msfp->rawrec == NULL)
{
if (!(msfp->rawrec = (char *)malloc (MAXRECLEN)))
{
ms_log (2, "ms_readmsr_main(): Cannot allocate memory for read buffer\n");
return MS_GENERROR;
}
}
/* Sanity check: track if we are reading the same file */
if (msfp->fp && strncmp (msfile, msfp->filename, sizeof (msfp->filename)))
{
ms_log (2, "ms_readmsr_main() called with a different file name without being reset\n");
/* Close previous file and reset needed variables */
if (msfp->fp != NULL)
fclose (msfp->fp);
msfp->fp = NULL;
msfp->readlen = 0;
msfp->readoffset = 0;
msfp->packtype = 0;
msfp->packhdroffset = 0;
msfp->filepos = 0;
msfp->filesize = 0;
msfp->recordcount = 0;
}
/* Open the file if needed, redirect to stdin if file is "-" */
if (msfp->fp == NULL)
{
/* Store the filename for tracking */
strncpy (msfp->filename, msfile, sizeof (msfp->filename) - 1);
msfp->filename[sizeof (msfp->filename) - 1] = '\0';
if (strcmp (msfile, "-") == 0)
{
msfp->fp = stdin;
}
else
{
if ((msfp->fp = fopen (msfile, "rb")) == NULL)
{
ms_log (2, "Cannot open file: %s (%s)\n", msfile, strerror (errno));
msr_free (ppmsr);
return MS_GENERROR;
}
else
{
/* Determine file size */
struct stat sbuf;
if (fstat (fileno (msfp->fp), &sbuf))
{
ms_log (2, "Cannot open file: %s (%s)\n", msfile, strerror (errno));
msr_free (ppmsr);
return MS_GENERROR;
}
msfp->filesize = sbuf.st_size;
}
}
}
/* Seek to a specified offset if requested */
if (fpos != NULL && *fpos < 0)
{
/* Only try to seek in real files, not stdin */
if (msfp->fp != stdin)
{
if (lmp_fseeko (msfp->fp, *fpos * -1, SEEK_SET))
{
ms_log (2, "Cannot seek in file: %s (%s)\n", msfile, strerror (errno));
return MS_GENERROR;
}
msfp->filepos = *fpos * -1;
msfp->readlen = 0;
msfp->readoffset = 0;
}
}
/* Zero the last record indicator */
if (last)
*last = 0;
/* Read data and search for records */
for (;;)
{
/* Read more data into buffer if not at EOF and buffer has less than MINRECLEN
* or more data is needed for the current record detected in buffer. */
if (!feof (msfp->fp) && (MSFPBUFLEN (msfp) < MINRECLEN || parseval > 0))
{
/* Reset offsets if no unprocessed data in buffer */
if (MSFPBUFLEN (msfp) <= 0)
{
msfp->readlen = 0;
msfp->readoffset = 0;
}
/* Otherwise shift existing data to beginning of buffer */
else if (msfp->readoffset > 0)
{
ms_shift_msfp (msfp, msfp->readoffset);
}
/* Determine read size */
readsize = (MAXRECLEN - msfp->readlen);
/* Read data into record buffer */
readcount = ms_fread (msfp->rawrec + msfp->readlen, 1, readsize, msfp->fp);
if (readcount != readsize)
{
if (!feof (msfp->fp))
{
ms_log (2, "Short read of %d bytes starting from %" PRId64 "\n",
readsize, msfp->filepos);
retcode = MS_GENERROR;
break;
}
}
/* Update read buffer length */
msfp->readlen += readcount;
/* File position corresponding to start of buffer; not strictly necessary */
if (msfp->fp != stdin)
msfp->filepos = lmp_ftello (msfp->fp) - msfp->readlen;
}
/* Test for packed file signature at the beginning of the file */
if (msfp->filepos == 0 && *(MSFPREADPTR (msfp)) == 'P' && MSFPBUFLEN (msfp) >= 48)
{
msfp->packtype = 0;
/* Determine pack type, the negative pack type indicates initial header */
if (!memcmp ("PED", MSFPREADPTR (msfp), 3))
msfp->packtype = -1;
else if (!memcmp ("PSD", MSFPREADPTR (msfp), 3))
msfp->packtype = -2;
else if (!memcmp ("PLC", MSFPREADPTR (msfp), 3))
msfp->packtype = -6;
else if (!memcmp ("PQI", MSFPREADPTR (msfp), 3))
msfp->packtype = -7;
else if (!memcmp ("PLS", MSFPREADPTR (msfp), 3))
msfp->packtype = -8;
if (verbose > 0)
ms_log (1, "Detected packed file (%3.3s: type %d)\n", MSFPREADPTR (msfp), -msfp->packtype);
}
/* Read pack headers, initial and subsequent headers including (ignored) chksum values */
if (msfp->packtype && (msfp->packtype < 0 || msfp->filepos == msfp->packhdroffset) && MSFPBUFLEN (msfp) >= 48)
{
char hdrstr[30];
int64_t datasize;
/* Determine bytes to skip before header: either initial ID block or type-specific chksum block */
packskipsize = (msfp->packtype < 0) ? 10 : packtypes[msfp->packtype][2];
if (msfp->packtype < 0)
msfp->packtype = -msfp->packtype;
/* Read pack length from pack header accounting for bytes that should be skipped */
memset (hdrstr, 0, sizeof (hdrstr));
memcpy (hdrstr, MSFPREADPTR (msfp) + (packtypes[msfp->packtype][0] + packskipsize - packtypes[msfp->packtype][1]),
packtypes[msfp->packtype][1]);
sscanf (hdrstr, " %" SCNd64, &datasize);
packdatasize = (off_t)datasize;
/* Next pack header = File position + skipsize + header size + data size
* This offset is actually to the data block chksum which is skipped by the logic above,
* the next pack header should directly follow the chksum. */
msfp->packhdroffset = msfp->filepos + packskipsize + packtypes[msfp->packtype][0] + packdatasize;
if (verbose > 1)
ms_log (1, "Read packed file header at offset %" PRId64 " (%d bytes follow), chksum offset: %" PRId64 "\n",
(msfp->filepos + packskipsize), packdatasize,
msfp->packhdroffset);
/* Shift buffer to new reading offset (aligns records in buffer) */
ms_shift_msfp (msfp, msfp->readoffset + (packskipsize + packtypes[msfp->packtype][0]));
} /* End of packed header processing */
/* Check for match if selections are supplied and pack header was read, */
/* only when enough data is in buffer and not reading from stdin pipe */
if (selections && msfp->packtype && packdatasize && MSFPBUFLEN (msfp) >= 48 && msfp->fp != stdin)
{
char srcname[100];
ms_recsrcname (MSFPREADPTR (msfp), srcname, 1);
if (!ms_matchselect (selections, srcname, HPTERROR, HPTERROR, NULL))
{
/* Update read position if next section is in buffer */
if (MSFPBUFLEN (msfp) >= (msfp->packhdroffset - msfp->filepos))
{
if (verbose > 1)
{
ms_log (1, "Skipping (jump) packed section for %s (%d bytes) starting at offset %" PRId64 "\n",
srcname, (msfp->packhdroffset - msfp->filepos), msfp->filepos);
}
msfp->readoffset += (msfp->packhdroffset - msfp->filepos);
msfp->filepos = msfp->packhdroffset;
packdatasize = 0;
}
/* Otherwise seek to next pack header and reset reading position */
else
{
if (verbose > 1)
{
ms_log (1, "Skipping (seek) packed section for %s (%d bytes) starting at offset %" PRId64 "\n",
srcname, (msfp->packhdroffset - msfp->filepos), msfp->filepos);
}
if (lmp_fseeko (msfp->fp, msfp->packhdroffset, SEEK_SET))
{
ms_log (2, "Cannot seek in file: %s (%s)\n", msfile, strerror (errno));
return MS_GENERROR;
break;
}
msfp->filepos = msfp->packhdroffset;
msfp->readlen = 0;
msfp->readoffset = 0;
packdatasize = 0;
}
/* Return to top of loop for proper pack header handling */
continue;
}
} /* End of selection processing */
/* Attempt to parse record from buffer */
if (MSFPBUFLEN (msfp) >= MINRECLEN)
{
int parselen = MSFPBUFLEN (msfp);
/* Limit the parse length to offset of pack header if present in the buffer */
if (msfp->packhdroffset && msfp->packhdroffset < (msfp->filepos + MSFPBUFLEN (msfp)))
parselen = msfp->packhdroffset - msfp->filepos;
parseval = msr_parse (MSFPREADPTR (msfp), parselen, ppmsr, reclen, dataflag, verbose);
/* Record detected and parsed */
if (parseval == 0)
{
if (verbose > 1)
ms_log (1, "Read record length of %d bytes\n", (*ppmsr)->reclen);
/* Test if this is the last record if file size is known (not pipe) */
if (last && msfp->filesize)
if ((msfp->filesize - (msfp->filepos + (*ppmsr)->reclen)) < MINRECLEN)
*last = 1;
/* Return file position for this record */
if (fpos)
*fpos = msfp->filepos;
/* Update reading offset, file position and record count */
msfp->readoffset += (*ppmsr)->reclen;
msfp->filepos += (*ppmsr)->reclen;
msfp->recordcount++;
retcode = MS_NOERROR;
break;
}
else if (parseval < 0)
{
/* Skip non-data if requested */
if (skipnotdata)
{
if (verbose > 1)
{
if (MS_ISVALIDBLANK ((char *)MSFPREADPTR (msfp)))
ms_log (1, "Skipped %d bytes of blank/noise record at byte offset %" PRId64 "\n",
MINRECLEN, msfp->filepos);
else
ms_log (1, "Skipped %d bytes of non-data record at byte offset %" PRId64 "\n",
MINRECLEN, msfp->filepos);
}
/* Skip MINRECLEN bytes, update reading offset and file position */
msfp->readoffset += MINRECLEN;
msfp->filepos += MINRECLEN;
}
/* Parsing errors */
else
{
ms_log (2, "Cannot detect record at byte offset %" PRId64 ": %s\n",
msfp->filepos, msfile);
/* Print common errors and raw details if verbose */
ms_parse_raw (MSFPREADPTR (msfp), MSFPBUFLEN (msfp), verbose, -1);
retcode = parseval;
break;
}
}
else /* parseval > 0 (found record but need more data) */
{
/* Determine implied record length if needed */
int32_t impreclen = reclen;
/* Check for parse hints that are larger than MAXRECLEN */
if ((MSFPBUFLEN (msfp) + parseval) > MAXRECLEN)
{
if (skipnotdata)
{
/* Skip MINRECLEN bytes, update reading offset and file position */
msfp->readoffset += MINRECLEN;
msfp->filepos += MINRECLEN;
}
else
{
retcode = MS_OUTOFRANGE;
break;
}
}
/* Pack header check, if pack header offset is within buffer */
else if (impreclen <= 0 && msfp->packhdroffset &&
msfp->packhdroffset < (msfp->filepos + MSFPBUFLEN (msfp)))
{
impreclen = msfp->packhdroffset - msfp->filepos;
/* Check that record length is within range and a power of 2.
* Power of two if (X & (X - 1)) == 0 */
if (impreclen >= MINRECLEN && impreclen <= MAXRECLEN &&
(impreclen & (impreclen - 1)) == 0)
{
/* Set the record length implied by the next pack header */
reclen = impreclen;
}
else
{
ms_log (1, "Implied record length (%d) is invalid\n", impreclen);
retcode = MS_NOTSEED;
break;
}
}
/* End of file check */
else if (impreclen <= 0 && feof (msfp->fp))
{
impreclen = msfp->filesize - msfp->filepos;
/* Check that record length is within range and a power of 2.
* Power of two if (X & (X - 1)) == 0 */
if (impreclen >= MINRECLEN && impreclen <= MAXRECLEN &&
(impreclen & (impreclen - 1)) == 0)
{
/* Set the record length implied by the end of the file */
reclen = impreclen;
}
/* Otherwise a trucated record */
else
{
if (verbose)
{
if (msfp->filesize)
ms_log (1, "Truncated record at byte offset %" PRId64 ", filesize %d: %s\n",
msfp->filepos, msfp->filesize, msfile);
else
ms_log (1, "Truncated record at byte offset %" PRId64 "\n",
msfp->filepos);
}
retcode = MS_ENDOFFILE;
break;
}
}
}
} /* End of record detection */
/* Finished when within MINRECLEN from EOF and buffer less than MINRECLEN */
if ((msfp->filesize - msfp->filepos) < MINRECLEN && MSFPBUFLEN (msfp) < MINRECLEN)
{
if (msfp->recordcount == 0 && msfp->packtype == 0)
{
if (verbose > 0)
ms_log (2, "%s: No data records read, not SEED?\n", msfile);
retcode = MS_NOTSEED;
}
else
{
retcode = MS_ENDOFFILE;
}
break;
}
} /* End of reading, record detection and parsing loop */
/* Cleanup target MSRecord if returning an error */
if (retcode != MS_NOERROR)
{
msr_free (ppmsr);
}
return retcode;
} /* End of ms_readmsr_main() */
// Function that reads from a MiniSEED binary file from a char buffer and
// returns a LinkedIDList.
LinkedIDList *
readMSEEDBuffer (char *mseed, int buflen, Selections *selections, flag
unpack_data, int reclen, flag verbose, flag details,
int header_byteorder, long long (*allocData) (int, char),
void (*diag_print) (char*), void (*log_print) (char*))
{
int retcode = 0;
int retval = 0;
flag swapflag = 0;
flag bigendianhost = ms_bigendianhost();
// current offset of mseed char pointer
int offset = 0;
// Unpack without reading the data first
flag dataflag = 0;
// the timing_qual of BLK 1001
uint8_t timing_qual = 0xFF;
// the calibration type, availability of BLK 300, 310, 320, 390, 395
int8_t calibration_type = -1;
// Init all the pointers to NULL. Most compilers should do this anyway.
LinkedIDList * idListHead = NULL;
LinkedIDList * idListCurrent = NULL;
LinkedIDList * idListLast = NULL;
MSRecord *msr = NULL;
ContinuousSegment * segmentCurrent = NULL;
hptime_t lastgap = 0;
hptime_t hptimetol = 0;
hptime_t nhptimetol = 0;
long long data_offset;
LinkedRecordList *recordHead = NULL;
LinkedRecordList *recordPrevious = NULL;
LinkedRecordList *recordCurrent = NULL;
int datasize;
int record_count = 0;
// A negative verbosity suppresses as much as possible.
if (verbose < 0) {
ms_loginit(&empty_print, NULL, &empty_print, NULL);
}
else {
ms_loginit(log_print, "INFO: ", diag_print, "ERROR: ");
}
if (header_byteorder >= 0) {
// Enforce little endian.
if (header_byteorder == 0) {
MS_UNPACKHEADERBYTEORDER(0);
}
// Enforce big endian.
else {
MS_UNPACKHEADERBYTEORDER(1);
}
}
else {
MS_UNPACKHEADERBYTEORDER(-1);
}
// Read all records and save them in a linked list.
while (offset < buflen) {
msr = msr_init(NULL);
if ( msr == NULL ) {
ms_log (2, "readMSEEDBuffer(): Error initializing msr\n");
return NULL;
}
if (verbose > 1) {
ms_log(0, "readMSEEDBuffer(): calling msr_parse with "
"mseed+offset=%d+%d, buflen=%d, reclen=%d, dataflag=%d, verbose=%d\n",
mseed, offset, buflen, reclen, dataflag, verbose);
}
// If the record length is given, make sure at least that amount of data is available.
if (reclen != -1) {
if (offset + reclen > buflen) {
ms_log(1, "readMSEEDBuffer(): Last reclen exceeds buflen, skipping.\n");
msr_free(&msr);
break;
}
}
// Otherwise assume the smallest possible record length and assure that enough
// data is present.
else {
if (offset + MINRECLEN > buflen) {
ms_log(1, "readMSEEDBuffer(): Last record only has %i byte(s) which "
"is not enough to constitute a full SEED record. Corrupt data? "
"Record will be skipped.\n", buflen - offset);
msr_free(&msr);
break;
}
}
// Skip empty or noise records.
if (OBSPY_ISVALIDBLANK(mseed + offset)) {
offset += MINRECLEN;
continue;
}
// Pass (buflen - offset) because msr_parse() expects only a single record. This
// way libmseed can take care to not overstep bounds.
// Return values:
// 0 : Success, populates the supplied MSRecord.
// >0 : Data record detected but not enough data is present, the
// return value is a hint of how many more bytes are needed.
// <0 : libmseed error code (listed in libmseed.h) is returned.
retcode = msr_parse ((mseed+offset), buflen - offset, &msr, reclen, dataflag, verbose);
// Handle error.
if (retcode < 0) {
log_error(retcode, offset);
msr_free(&msr);
break;
}
// msr_parse() returns > 0 if a data record has been detected but the buffer either has not enough
// data (this cannot happen with ObsPy's logic) or the last record has no Blockette 1000 and it cannot
// determine the record length because there is no next record (this can happen in ObsPy) - handle that
// case by just calling msr_parse() with an explicit record length set.
else if ( retcode > 0 && retcode < (buflen - offset)) {
// Check if the remaining bytes can exactly make up a record length.
int r_bytes = buflen - offset;
float exp = log10((float)r_bytes) / log10(2.0);
if ((fmodf(exp, 1.0) < 0.0000001) && ((int)roundf_(exp) >= 7) && ((int)roundf_(exp) <= 256)) {
retcode = msr_parse((mseed + offset), buflen - offset, &msr, r_bytes, dataflag, verbose);
if ( retcode != 0 ) {
log_error(retcode, offset);
msr_free(&msr);
break;
}
}
else {
msr_free(&msr);
break;
}
}
if (offset + msr->reclen > buflen) {
ms_log(1, "readMSEEDBuffer(): Last msr->reclen exceeds buflen, skipping.\n");
msr_free(&msr);
break;
}
// Test against selections if supplied
if ( selections ) {
char srcname[50];
hptime_t endtime;
msr_srcname (msr, srcname, 1);
endtime = msr_endtime (msr);
if ( ms_matchselect (selections, srcname, msr->starttime, endtime, NULL) == NULL ) {
// Add the record length for the next iteration
offset += msr->reclen;
// Free record.
msr_free(&msr);
continue;
}
}
record_count += 1;
recordCurrent = lrl_init ();
// Append to linked record list if one exists.
if ( recordHead != NULL ) {
recordPrevious->next = recordCurrent;
recordCurrent->previous = recordPrevious;
recordCurrent->next = NULL;
recordPrevious = recordCurrent;
}
// Otherwise create a new one.
else {
recordHead = recordCurrent;
recordCurrent->previous = NULL;
recordPrevious = recordCurrent;
}
recordCurrent->record = msr;
// Figure out if the byte-order of the data has to be swapped.
swapflag = 0;
// If blockette 1000 is present, use it.
if ( msr->Blkt1000 != 0) {
/* If BE host and LE data need swapping */
if ( bigendianhost && msr->byteorder == 0 ) {
swapflag = 1;
}
/* If LE host and BE data (or bad byte order value) need swapping */
if ( !bigendianhost && msr->byteorder > 0 ) {
swapflag = 1;
}
}
// Otherwise assume the data has the same byte order as the header.
// This needs to be done on the raw header bytes as libmseed only returns
// header fields in the native byte order.
else {
unsigned char* _t = (unsigned char*)mseed + offset + 20;
unsigned int year = _t[0] | _t[1] << 8;
unsigned int day = _t[2] | _t[3] << 8;
// Swap data if header needs to be swapped.
if (!MS_ISVALIDYEARDAY(year, day)) {
swapflag = 1;
}
}
// Actually unpack the data if the flag is not set and if the data
// offset is valid.
if ((unpack_data != 0) && (msr->fsdh->data_offset >= 48) &&
(msr->fsdh->data_offset < msr->reclen) &&
(msr->samplecnt > 0)) {
retval = msr_unpack_data (msr, swapflag, verbose);
}
if ( retval > 0 ) {
msr->numsamples = retval;
}
if ( msr->fsdh->start_time.fract > 9999 ) {
ms_log(1, "readMSEEDBuffer(): Record with offset=%d has a "
"fractional second (.0001 seconds) of %d. This is not "
"strictly valid but will be interpreted as one or more "
"additional seconds.",
offset, msr->fsdh->start_time.fract);
}
// Add the record length for the next iteration
offset += msr->reclen;
}
// Return empty id list if no records could be found.
if (record_count == 0) {
idListHead = lil_init();
return idListHead;
}
// All records that match the selection are now stored in a LinkedRecordList
// that starts at recordHead. The next step is to sort them by matching ids
// and then by time.
recordCurrent = recordHead;
while (recordCurrent != NULL) {
// Check if the ID of the record is already available and if not create a
// new one.
// Start with the last id as it is most likely to be the correct one.
idListCurrent = idListLast;
while (idListCurrent != NULL) {
if (strcmp(idListCurrent->network, recordCurrent->record->network) == 0 &&
strcmp(idListCurrent->station, recordCurrent->record->station) == 0 &&
strcmp(idListCurrent->location, recordCurrent->record->location) == 0 &&
strcmp(idListCurrent->channel, recordCurrent->record->channel) == 0 &&
idListCurrent->dataquality == recordCurrent->record->dataquality) {
break;
}
else {
idListCurrent = idListCurrent->previous;
}
}
// Create a new id list if one is needed.
if (idListCurrent == NULL) {
idListCurrent = lil_init();
idListCurrent->previous = idListLast;
if (idListLast != NULL) {
idListLast->next = idListCurrent;
}
idListLast = idListCurrent;
if (idListHead == NULL) {
idListHead = idListCurrent;
}
// Set the IdList attributes.
strcpy(idListCurrent->network, recordCurrent->record->network);
strcpy(idListCurrent->station, recordCurrent->record->station);
strcpy(idListCurrent->location, recordCurrent->record->location);
strcpy(idListCurrent->channel, recordCurrent->record->channel);
idListCurrent->dataquality = recordCurrent->record->dataquality;
}
// Now check if the current record fits exactly to the end of the last
// segment of the current id. If not create a new segment. Therefore
// if records with the same id are in wrong order a new segment will be
// created. This is on purpose.
segmentCurrent = idListCurrent->lastSegment;
if (segmentCurrent != NULL) {
hptimetol = (hptime_t) (0.5 * segmentCurrent->hpdelta);
nhptimetol = ( hptimetol ) ? -hptimetol : 0;
lastgap = recordCurrent->record->starttime - segmentCurrent->endtime - segmentCurrent->hpdelta;
}
if (details == 1) {
/* extract information on calibration BLKs */
calibration_type = -1;
if (recordCurrent->record->blkts) {
BlktLink *cur_blkt = recordCurrent->record->blkts;
while (cur_blkt) {
switch (cur_blkt->blkt_type) {
case 300:
calibration_type = 1;
break;
case 310:
calibration_type = 2;
break;
case 320:
calibration_type = 3;
break;
case 390:
calibration_type = 4;
break;
case 395:
calibration_type = -2;
break;
default:
break;
}
cur_blkt = cur_blkt->next;
}
}
/* extract information based on timing quality */
timing_qual = 0xFF;
if (recordCurrent->record->Blkt1001 != 0) {
timing_qual = recordCurrent->record->Blkt1001->timing_qual;
}
}
if ( segmentCurrent != NULL &&
// This is important for zero data record coupled with not unpacking
// the data. It needs to be split in two places: Before the zero data
// record and after it.
recordCurrent->record->samplecnt > 0 && segmentCurrent->samplecnt > 0 &&
segmentCurrent->sampletype == recordCurrent->record->sampletype &&
// Test the default sample rate tolerance: abs(1-sr1/sr2) < 0.0001
MS_ISRATETOLERABLE (segmentCurrent->samprate, recordCurrent->record->samprate) &&
// Check if the times are within the time tolerance
lastgap <= hptimetol && lastgap >= nhptimetol &&
segmentCurrent->timing_qual == timing_qual &&
segmentCurrent->calibration_type == calibration_type) {
recordCurrent->previous = segmentCurrent->lastRecord;
segmentCurrent->lastRecord = segmentCurrent->lastRecord->next = recordCurrent;
segmentCurrent->samplecnt += recordCurrent->record->samplecnt;
segmentCurrent->endtime = msr_endtime(recordCurrent->record);
}
// Otherwise create a new segment and add the current record.
else {
segmentCurrent = seg_init();
segmentCurrent->previous = idListCurrent->lastSegment;
if (idListCurrent->lastSegment != NULL) {
idListCurrent->lastSegment->next = segmentCurrent;
}
else {
idListCurrent->firstSegment = segmentCurrent;
}
idListCurrent->lastSegment = segmentCurrent;
segmentCurrent->starttime = recordCurrent->record->starttime;
segmentCurrent->endtime = msr_endtime(recordCurrent->record);
segmentCurrent->samprate = recordCurrent->record->samprate;
segmentCurrent->sampletype = recordCurrent->record->sampletype;
segmentCurrent->samplecnt = recordCurrent->record->samplecnt;
// Calculate high-precision sample period
segmentCurrent->hpdelta = (hptime_t) (( recordCurrent->record->samprate ) ?
(HPTMODULUS / recordCurrent->record->samprate) : 0.0);
segmentCurrent->timing_qual = timing_qual;
segmentCurrent->calibration_type = calibration_type;
segmentCurrent->firstRecord = segmentCurrent->lastRecord = recordCurrent;
recordCurrent->previous = NULL;
}
recordPrevious = recordCurrent->next;
recordCurrent->next = NULL;
recordCurrent = recordPrevious;
}
// Now loop over all segments, combine the records and free the msr
// structures.
idListCurrent = idListHead;
while (idListCurrent != NULL)
{
segmentCurrent = idListCurrent->firstSegment;
while (segmentCurrent != NULL) {
if (segmentCurrent->datasamples) {
free(segmentCurrent->datasamples);
}
// Allocate data via a callback function.
if (unpack_data != 0) {
segmentCurrent->datasamples = (void *) allocData(segmentCurrent->samplecnt, segmentCurrent->sampletype);
}
// Loop over all records, write the data to the buffer and free the msr structures.
recordCurrent = segmentCurrent->firstRecord;
data_offset = (long long)(segmentCurrent->datasamples);
while (recordCurrent != NULL) {
datasize = recordCurrent->record->samplecnt * ms_samplesize(recordCurrent->record->sampletype);
memcpy((void *)data_offset, recordCurrent->record->datasamples, datasize);
// Free the record.
msr_free(&(recordCurrent->record));
// Increase the data_offset and the record.
data_offset += (long long)datasize;
recordCurrent = recordCurrent->next;
}
segmentCurrent = segmentCurrent->next;
}
idListCurrent = idListCurrent->next;
}
return idListHead;
}
// Function that reads from a MiniSEED binary file from a char buffer and
// returns a LinkedIDList.
LinkedIDList *
readMSEEDBuffer (char *mseed, int buflen, Selections *selections, flag
unpack_data, int reclen, flag verbose, flag details,
long (*allocData) (int, char))
{
int retcode = 0;
int retval = 0;
flag swapflag = 0;
// current offset of mseed char pointer
int offset = 0;
// Unpack without reading the data first
flag dataflag = 0;
// the timing_qual of BLK 1001
uint8_t timing_qual = 0xFF;
// the calibration type, availability of BLK 300, 310, 320, 390, 395
int8_t calibration_type = -1;
// Init all the pointers to NULL. Most compilers should do this anyway.
LinkedIDList * idListHead = NULL;
LinkedIDList * idListCurrent = NULL;
LinkedIDList * idListLast = NULL;
MSRecord *msr = NULL;
ContinuousSegment * segmentCurrent = NULL;
hptime_t lastgap;
hptime_t hptimetol;
hptime_t nhptimetol;
long data_offset;
LinkedRecordList *recordHead = NULL;
LinkedRecordList *recordPrevious = NULL;
LinkedRecordList *recordCurrent = NULL;
int datasize;
//
// Read all records and save them in a linked list.
//
int record_count = 0;
while (offset < buflen) {
msr = msr_init(NULL);
retcode = msr_parse ( (mseed+offset), buflen, &msr, reclen, dataflag, verbose);
if ( ! (retcode == MS_NOERROR)) {
msr_free(&msr);
break;
}
// Test against selections if supplied
if ( selections ) {
char srcname[50];
hptime_t endtime;
msr_srcname (msr, srcname, 1);
endtime = msr_endtime (msr);
if ( ms_matchselect (selections, srcname, msr->starttime, endtime, NULL) == NULL ) {
// Add the record length for the next iteration
offset += msr->reclen;
// Free record.
msr_free(&msr);
continue;
}
}
record_count += 1;
recordCurrent = lrl_init ();
// Append to linked record list if one exists.
if ( recordHead != NULL ) {
recordPrevious->next = recordCurrent;
recordCurrent->previous = recordPrevious;
recordCurrent->next = NULL;
recordPrevious = recordCurrent;
}
// Otherwise create a new one.
else {
recordHead = recordCurrent;
recordCurrent->previous = NULL;
recordPrevious = recordCurrent;
}
recordCurrent->record = msr;
// Determine the byteorder swapflag only for the very first record. The byteorder
// should not change within the file.
// XXX: Maybe check for every record?
if (swapflag <= 0) {
// Returns 0 if the host is little endian, otherwise 1.
flag bigendianhost = ms_bigendianhost();
// Set the swapbyteflag if it is needed.
if ( msr->Blkt1000 != 0) {
/* If BE host and LE data need swapping */
if ( bigendianhost && msr->byteorder == 0 ) {
swapflag = 1;
}
/* If LE host and BE data (or bad byte order value) need swapping */
if ( !bigendianhost && msr->byteorder > 0 ) {
swapflag = 1;
}
}
}
// Actually unpack the data if the flag is not set.
if (unpack_data != 0) {
retval = msr_unpack_data (msr, swapflag, verbose);
}
if ( retval > 0 ) {
msr->numsamples = retval;
}
// Add the record length for the next iteration
offset += msr->reclen;
}
// Return empty id list if no records could be found.
if (record_count == 0) {
idListHead = lil_init();
return idListHead;
}
// All records that match the selection are now stored in a LinkedRecordList
// that starts at recordHead. The next step is to sort them by matching ids
// and then by time.
recordCurrent = recordHead;
while (recordCurrent != NULL) {
// Check if the ID of the record is already available and if not create a
// new one.
// Start with the last id as it is most likely to be the correct one.
idListCurrent = idListLast;
while (idListCurrent != NULL) {
if (strcmp(idListCurrent->network, recordCurrent->record->network) == 0 &&
strcmp(idListCurrent->station, recordCurrent->record->station) == 0 &&
strcmp(idListCurrent->location, recordCurrent->record->location) == 0 &&
strcmp(idListCurrent->channel, recordCurrent->record->channel) == 0 &&
idListCurrent->dataquality == recordCurrent->record->dataquality) {
break;
}
else {
idListCurrent = idListCurrent->previous;
}
}
// Create a new id list if one is needed.
if (idListCurrent == NULL) {
idListCurrent = lil_init();
idListCurrent->previous = idListLast;
if (idListLast != NULL) {
idListLast->next = idListCurrent;
}
idListLast = idListCurrent;
if (idListHead == NULL) {
idListHead = idListCurrent;
}
// Set the IdList attributes.
strcpy(idListCurrent->network, recordCurrent->record->network);
strcpy(idListCurrent->station, recordCurrent->record->station);
strcpy(idListCurrent->location, recordCurrent->record->location);
strcpy(idListCurrent->channel, recordCurrent->record->channel);
idListCurrent->dataquality = recordCurrent->record->dataquality;
}
// Now check if the current record fits exactly to the end of the last
// segment of the current id. If not create a new segment. Therefore
// if records with the same id are in wrong order a new segment will be
// created. This is on purpose.
segmentCurrent = idListCurrent->lastSegment;
if (segmentCurrent != NULL) {
hptimetol = (hptime_t) (0.5 * segmentCurrent->hpdelta);
nhptimetol = ( hptimetol ) ? -hptimetol : 0;
lastgap = recordCurrent->record->starttime - segmentCurrent->endtime - segmentCurrent->hpdelta;
}
if (details == 1) {
/* extract information on calibration BLKs */
calibration_type = -1;
if (recordCurrent->record->blkts) {
BlktLink *cur_blkt = recordCurrent->record->blkts;
while (cur_blkt) {
switch (cur_blkt->blkt_type) {
case 300:
calibration_type = 1;
break;
case 310:
calibration_type = 2;
break;
case 320:
calibration_type = 3;
break;
case 390:
calibration_type = 4;
break;
case 395:
calibration_type = -2;
break;
default:
break;
}
cur_blkt = cur_blkt->next;
}
}
/* extract information based on timing quality */
timing_qual = 0xFF;
if (recordCurrent->record->Blkt1001 != 0) {
timing_qual = recordCurrent->record->Blkt1001->timing_qual;
}
}
if ( segmentCurrent != NULL &&
segmentCurrent->sampletype == recordCurrent->record->sampletype &&
// Test the default sample rate tolerance: abs(1-sr1/sr2) < 0.0001
MS_ISRATETOLERABLE (segmentCurrent->samprate, recordCurrent->record->samprate) &&
// Check if the times are within the time tolerance
lastgap <= hptimetol && lastgap >= nhptimetol &&
segmentCurrent->timing_qual == timing_qual &&
segmentCurrent->calibration_type == calibration_type) {
recordCurrent->previous = segmentCurrent->lastRecord;
segmentCurrent->lastRecord = segmentCurrent->lastRecord->next = recordCurrent;
segmentCurrent->samplecnt += recordCurrent->record->samplecnt;
segmentCurrent->endtime = msr_endtime(recordCurrent->record);
}
// Otherwise create a new segment and add the current record.
else {
segmentCurrent = seg_init();
segmentCurrent->previous = idListCurrent->lastSegment;
if (idListCurrent->lastSegment != NULL) {
idListCurrent->lastSegment->next = segmentCurrent;
}
else {
idListCurrent->firstSegment = segmentCurrent;
}
idListCurrent->lastSegment = segmentCurrent;
segmentCurrent->starttime = recordCurrent->record->starttime;
segmentCurrent->endtime = msr_endtime(recordCurrent->record);
segmentCurrent->samprate = recordCurrent->record->samprate;
segmentCurrent->sampletype = recordCurrent->record->sampletype;
segmentCurrent->samplecnt = recordCurrent->record->samplecnt;
// Calculate high-precision sample period
segmentCurrent->hpdelta = (hptime_t) (( recordCurrent->record->samprate ) ?
(HPTMODULUS / recordCurrent->record->samprate) : 0.0);
segmentCurrent->timing_qual = timing_qual;
segmentCurrent->calibration_type = calibration_type;
segmentCurrent->firstRecord = segmentCurrent->lastRecord = recordCurrent;
recordCurrent->previous = NULL;
}
recordPrevious = recordCurrent->next;
recordCurrent->next = NULL;
recordCurrent = recordPrevious;
}
// Now loop over all segments, combine the records and free the msr
// structures.
idListCurrent = idListHead;
while (idListCurrent != NULL)
{
segmentCurrent = idListCurrent->firstSegment;
while (segmentCurrent != NULL) {
if (segmentCurrent->datasamples) {
free(segmentCurrent->datasamples);
}
// Allocate data via a callback function.
if (unpack_data != 0) {
segmentCurrent->datasamples = (void *) allocData(segmentCurrent->samplecnt, segmentCurrent->sampletype);
}
// Loop over all records, write the data to the buffer and free the msr structures.
recordCurrent = segmentCurrent->firstRecord;
data_offset = (long)(segmentCurrent->datasamples);
while (recordCurrent != NULL) {
datasize = recordCurrent->record->samplecnt * ms_samplesize(recordCurrent->record->sampletype);
memcpy((void *)data_offset, recordCurrent->record->datasamples, datasize);
// Free the record.
msr_free(&(recordCurrent->record));
// Increase the data_offset and the record.
data_offset += (long)datasize;
recordCurrent = recordCurrent->next;
}
segmentCurrent = segmentCurrent->next;
}
idListCurrent = idListCurrent->next;
}
return idListHead;
}
int main( int argc, char **argv )
{
SHM_INFO region;
long RingKey; /* Key to the transport ring to read from */
MSG_LOGO getlogo[NLOGO], logo;
long gotsize;
char msg[MAX_TRACEBUF_SIZ];
char *getSta, *getComp, *getNet, *getLoc, *inRing;
char wildSta, wildComp, wildNet, wildLoc;
unsigned char Type_Mseed;
unsigned char Type_TraceBuf,Type_TraceBuf2;
unsigned char Type_TraceComp, Type_TraceComp2;
unsigned char InstWildcard, ModWildcard;
short *short_data;
int *long_data;
TRACE2_HEADER *trh;
char orig_datatype[3];
char stime[256];
char etime[256];
int i;
int rc;
int nLogo = NLOGO;
static unsigned char InstId; /* local installation id */
char ModName[MAX_MOD_STR];
char *modid_name;
unsigned char sequence_number = 0;
// int statistics_flag;
time_t monitor_start_time = 0;
double start_time, end_time;
unsigned long packet_total=0;
unsigned long packet_total_size=0;
MSRecord *msr = NULL; /* mseed record */
logit_init("ring2mongo", 200, 256, 1);
char *mongo_user = getenv("MONGO_USER");
char *mongo_passwd=getenv("MONGO_PASSWD");
char *mongo_host=getenv("MONGO_HOST");
char *mongo_port=getenv("MONGO_PORT");
if(mongo_user == NULL ||
mongo_passwd==NULL ||
mongo_host== NULL ||
mongo_port==NULL){
printf("Ensure the following ENV varibles are set\n *MONGO_USER\n *MONGO_PASSWD\n *MONGO_HOST\n *MONGO_PORT\n");
exit(1);
return(1);
}
char mongo_str[128]="mongodb://";
strcat(mongo_str, mongo_user);
strcat(mongo_str, ":");
strcat(mongo_str, mongo_passwd);
strcat(mongo_str, "@");
strcat(mongo_str, mongo_host);
strcat(mongo_str, ":");
strcat(mongo_str, mongo_port);
strcat(mongo_str, "/");
//mongo stuff
mongoc_client_t *m_client;
mongoc_collection_t *m_collection;
mongoc_init ();
m_client = mongoc_client_new (mongo_str);
m_collection = mongoc_client_get_collection (m_client, "waveforms", "ring");
mongoc_bulk_operation_t *m_bulk;
bson_error_t m_error;
bson_t *m_doc;
bson_t m_reply;
bson_t *m_data;
/*wait for this many messages before performing bulk write
For real time continuous data it shoudl be ~ number of channels on ring*/
int MONGO_BULK_MAX = 30;
int m_count = 0; /* counter for mongo bulk */
bool m_ret;
/*end mongo stuff*/
/* Initialize pointers
**********************/
trh = (TRACE2_HEADER *) msg;
long_data = (int *)( msg + sizeof(TRACE2_HEADER) );
short_data = (short *)( msg + sizeof(TRACE2_HEADER) );
/* Check command line argument
*****************************/
if ( argc < 2 || argc > 6)
{
if(argc > 6)
fprintf(stderr, "ring2mongo: Too many parameters\n");
fprintf(stderr,"Usage: %s <ring name> [sta] [comp] [net] [loc] \n", argv[0]);
fprintf(stderr, " Note: All parameters are positional, and all but first are optional.\n");
fprintf(stderr, " the full data contained in the packet is printed out.\n");
fprintf(stderr, " If sta comp net (but not loc) are specified then only TraceBuf\n");
fprintf(stderr, " packets will be fetched (not TraceBuf2); otherwise both are fetched.\n");
fprintf(stderr, " Example: %s WAVE_RING PHOB wild NC wild n\n", argv[0]);
fprintf(stderr, " MSEED capability starting with version 2.5.1, prints mseed headers\n");
fprintf(stderr, " of TYPE_MSEED packets (no filtering yet).\n");
exit( 1 );
return 1;
}
/* process given parameters */
inRing = argv[1]; /* first parameter is ring name */
/* any parameters not given are set as wildcards */
getSta = getComp = getNet = getLoc = "";
wildSta = wildComp = wildNet = wildLoc = 1;
if(argc > 2)
{ /* at least station parameter given */
getSta = argv[2];
wildSta = IsWild(getSta);
if(argc > 3)
{ /* channel (component) parameter given */
getComp = argv[3];
wildComp = IsWild(getComp);
if(argc > 4)
{ /* network parameter given */
getNet = argv[4];
wildNet = IsWild(getNet);
if(argc > 5)
{ /* location parameter given (SCNL) */
getLoc = argv[5];
wildLoc = IsWild(getLoc);
}
else
{ /* SCN without location parameter given */
nLogo = 3; /* do not include tracebuf2s in search */
}
}
}
}
/* Attach to ring
*****************/
if ((RingKey = GetKey( inRing )) == -1 )
{
fprintf( stderr, "Invalid RingName; exiting!\n" );
exit( -1 );
return -1;
}
tport_attach( ®ion, RingKey );
/* Look up local installation id
*****************************/
if ( GetLocalInst( &InstId ) != 0 )
{
fprintf(stderr, "ring2mongo: error getting local installation id; exiting!\n" );
exit( -1 );
return -1;
}
/* Specify logos to get
***********************/
if ( GetType( "TYPE_MSEED", &Type_Mseed ) != 0 ) {
fprintf(stderr, "%s: WARNING: Invalid message type <TYPE_MSEED>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
Type_Mseed = TYPE_NOTUSED;
}
if ( GetType( "TYPE_TRACEBUF", &Type_TraceBuf ) != 0 ) {
fprintf(stderr, "%s: Invalid message type <TYPE_TRACEBUF>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
exit( -1 );
return -1;
}
if ( GetType( "TYPE_TRACE_COMP_UA", &Type_TraceComp ) != 0 ) {
fprintf(stderr, "%s: Invalid message type <TYPE_TRACE_COMP_UA>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
exit( -1 );
return -1;
}
if ( GetType( "TYPE_TRACEBUF2", &Type_TraceBuf2 ) != 0 ) {
fprintf(stderr, "%s: Invalid message type <TYPE_TRACEBUF2>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
exit( -1 );
return -1;
}
if ( GetType( "TYPE_TRACE2_COMP_UA", &Type_TraceComp2 ) != 0 ) {
fprintf(stderr,"%s: Invalid message type <TYPE_TRACE2_COMP_UA>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
exit( -1 );
return -1;
}
if ( GetModId( "MOD_WILDCARD", &ModWildcard ) != 0 ) {
fprintf(stderr, "%s: Invalid moduleid <MOD_WILDCARD>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
exit( -1 );
return -1;
}
if ( GetInst( "INST_WILDCARD", &InstWildcard ) != 0 ) {
fprintf(stderr, "%s: Invalid instid <INST_WILDCARD>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
exit( -1 );
return -1;
}
for( i=0; i<nLogo; i++ ) {
getlogo[i].instid = InstWildcard;
getlogo[i].mod = ModWildcard;
}
getlogo[0].type = Type_Mseed;
getlogo[1].type = Type_TraceBuf;
getlogo[2].type = Type_TraceComp;
if (nLogo >= 4) { /* if nLogo=5 then include TraceBuf2s */
getlogo[3].type = Type_TraceBuf2;
getlogo[4].type = Type_TraceComp2;
}
logit("", "Starting ring2mongo");
/* Flush the ring
*****************/
while ( tport_copyfrom( ®ion, getlogo, nLogo, &logo, &gotsize,
(char *)&msg, MAX_TRACEBUF_SIZ, &sequence_number ) != GET_NONE ){
packet_total++;
packet_total_size+=gotsize;
}
logit( "et", "ring2mongo: inRing flushed %ld packets of %ld bytes total.\n",
packet_total, packet_total_size);
while (tport_getflag( ®ion ) != TERMINATE ) {
rc = tport_copyfrom( ®ion, getlogo, nLogo,
&logo, &gotsize, msg, MAX_TRACEBUF_SIZ, &sequence_number );
if ( rc == GET_NONE ){
sleep_ew( 200 );
continue;
}
if ( rc == GET_TOOBIG ){
logit("et", "ring2mongo: retrieved message too big (%ld) for msg\n",
gotsize );
continue;
}
if ( rc == GET_NOTRACK )
logit("et", "ring2mongo: Tracking error.\n");
if ( rc == GET_MISS_LAPPED )
logit("et", "ring2mongo: Got lapped on the ring.\n");
if ( rc == GET_MISS_SEQGAP )
logit( "et", "ring2mongo: Gap in sequence numbers\n");
if ( rc == GET_MISS )
logit( "et", "ring2mongo: Missed messages\n");
/* Check SCNL of the retrieved message */
if (Type_Mseed != TYPE_NOTUSED && logo.type == Type_Mseed) {
/* Unpack record header and not data samples */
/*hard coded zero for dataflag(1) and verbose(0)for ring2mongo*/
if ( msr_unpack (msg, gotsize, &msr, 1, 0) != MS_NOERROR) {
logit("et", "Error parsing mseed record\n");
continue;
}
/* Print record information */
msr_print (msr, 0);
msr_free (&msr);
continue;
}
/*end Mseed*/
if ( (wildSta || (strcmp(getSta,trh->sta) ==0)) &&
(wildComp || (strcmp(getComp,trh->chan)==0)) &&
(wildNet || (strcmp(getNet,trh->net) ==0)) &&
(((logo.type == Type_TraceBuf2 ||
logo.type == Type_TraceComp2) &&
(wildLoc || (strcmp(getLoc,trh->loc) == 0))) ||
( (logo.type == Type_TraceBuf ||
logo.type == Type_TraceComp))))
{
strcpy(orig_datatype, trh->datatype);
char scnl[20];
scnl[0] = 0;
strcat( scnl, trh->sta);
strcat( scnl, ".");
strcat( scnl, trh->chan);
strcat( scnl, ".");
strcat( scnl, trh->net);
strcat( scnl, ".");
strcat( scnl, trh->loc);
if(WaveMsg2MakeLocal( trh ) < 0){
char dt[3];
/* now put a space if there are any punctuation marks */
for ( i=0; i<15; i++ ) {
if ( !isalnum(scnl[i]) && !ispunct(scnl[i]))
scnl[i] = ' ';
}
strncpy( dt, trh->datatype, 2 );
for ( i=0; i<2; i++ ) {
if ( !isalnum(dt[i]) && !ispunct(dt[i]))
dt[i] = ' ';
}
dt[i] = 0;
logit("et", "WARNING: WaveMsg2MakeLocal rejected tracebuf. Discard (%s).\n",
scnl );
logit("et", "\tdatatype=[%s]\n", dt);
continue;
}
/*lets get ready to mongo....*/
if(m_count < MONGO_BULK_MAX){
/*initialize when m_count ==0 */
if(m_count==0){
m_bulk = mongoc_collection_create_bulk_operation (m_collection, true, NULL);
}
m_data = bson_new ();
m_doc = BCON_NEW ("key", BCON_UTF8 (scnl),
"nsamp", BCON_INT32 (trh->nsamp),
"starttime", BCON_DOUBLE (trh->starttime*1000),
"endtime", BCON_DOUBLE (trh->endtime*1000),
"samprate", BCON_DOUBLE (trh->samprate),
"datatype", BCON_UTF8 (trh->datatype)
);
char index[4];
bson_append_array_begin (m_doc, "data", -1, m_data);
for ( i = 0; i < trh->nsamp; i++ ){
snprintf(index, 4, "%d", i);
if ( (strcmp (trh->datatype, "s2")==0) || (strcmp (trh->datatype, "i2")==0) ){
bson_append_int32 (m_data, index, -1, short_data[i]);
}else{
bson_append_int32 (m_data, index, -1, long_data[i]);
}
}
bson_append_array_end (m_doc, m_data);
mongoc_bulk_operation_insert (m_bulk, m_doc);
bson_destroy (m_doc);
bson_destroy (m_data);
m_count++;
if(m_count==MONGO_BULK_MAX){
/*write and destroty above stuff */
m_ret = mongoc_bulk_operation_execute (m_bulk, &m_reply, &m_error);
if (!m_ret){
logit ("et", "Error: %s\n", m_error.message);
}
bson_destroy (&m_reply);
mongoc_bulk_operation_destroy (m_bulk);
m_count = 0;
}
}
}
} /* end of while loop */
logit("et", "signing off");
exit (0);
return 0;
}
/***************************************************************************
* msr_duplicate:
*
* Duplicate an MSRecord struct
* including the fixed-section data
* header and blockette chain. If
* the datadup flag is true and the
* source MSRecord has associated
* data samples copy them as well.
*
* Returns a pointer to a new MSRecord on success and NULL on error.
***************************************************************************/
MSRecord *
msr_duplicate (MSRecord *msr, flag datadup)
{
MSRecord *dupmsr = 0;
int samplesize = 0;
if ( ! msr )
return NULL;
/* Allocate target MSRecord structure */
if ( (dupmsr = msr_init (NULL)) == NULL )
return NULL;
/* Copy MSRecord structure */
memcpy (dupmsr, msr, sizeof(MSRecord));
/* Copy fixed-section data header structure */
if ( msr->fsdh )
{
/* Allocate memory for new FSDH structure */
if ( (dupmsr->fsdh = (struct fsdh_s *) malloc (sizeof(struct fsdh_s))) == NULL )
{
ms_log (2, "msr_duplicate(): Error allocating memory\n");
free (dupmsr);
return NULL;
}
/* Copy the contents */
memcpy (dupmsr->fsdh, msr->fsdh, sizeof(struct fsdh_s));
}
/* Copy the blockette chain */
if ( msr->blkts )
{
BlktLink *blkt = msr->blkts;
BlktLink *next = NULL;
dupmsr->blkts = 0;
while ( blkt )
{
next = blkt->next;
/* Add blockette to chain of new MSRecord */
if ( msr_addblockette (dupmsr, blkt->blktdata, blkt->blktdatalen,
blkt->blkt_type, 0) == NULL )
{
ms_log (2, "msr_duplicate(): Error adding blockettes\n");
msr_free (&dupmsr);
return NULL;
}
blkt = next;
}
}
/* Copy data samples if requested and available */
if ( datadup && msr->datasamples )
{
/* Determine size of samples in bytes */
samplesize = ms_samplesize (msr->sampletype);
if ( samplesize == 0 )
{
ms_log (2, "msr_duplicate(): unrecognized sample type: '%c'\n",
msr->sampletype);
free (dupmsr);
return NULL;
}
/* Allocate memory for new data array */
if ( (dupmsr->datasamples = (void *) malloc ((size_t)(msr->numsamples * samplesize))) == NULL )
{
ms_log (2, "msr_duplicate(): Error allocating memory\n");
free (dupmsr);
return NULL;
}
/* Copy the data array */
memcpy (dupmsr->datasamples, msr->datasamples, ((size_t)(msr->numsamples * samplesize)));
}
/* Otherwise make sure the sample array and count are zero */
else
{
dupmsr->datasamples = 0;
dupmsr->numsamples = 0;
}
return dupmsr;
} /* End of msr_duplicate() */
/**********************************************************************
* msr_parse:
*
* This routine will attempt to parse (detect and unpack) a Mini-SEED
* record from a specified memory buffer and populate a supplied
* MSRecord structure.
*
* If reclen is less than or equal to 0 the length of record is
* automatically detected otherwise reclen should be the correct
* record length.
*
* For auto detection of record length the record should include a
* 1000 blockette or be followed by another record header in the
* buffer.
*
* dataflag will be passed directly to msr_unpack().
*
* Return values:
* 0 : Success, populates the supplied MSRecord.
* >0 : Data record detected but not enough data is present, the
* return value is a hint of how many more bytes are needed.
* <0 : libmseed error code (listed in libmseed.h) is returned.
*********************************************************************/
int
msr_parse ( char *record, int recbuflen, MSRecord **ppmsr, int reclen,
flag dataflag, flag verbose )
{
int detlen = 0;
int retcode = 0;
if ( ! ppmsr )
return MS_GENERROR;
if ( ! record )
return MS_GENERROR;
/* Sanity check: record length cannot be larger than buffer */
if ( reclen > 0 && reclen > recbuflen )
{
ms_log (2, "ms_parse() Record length (%d) cannot be larger than buffer (%d)\n",
reclen, recbuflen);
return MS_GENERROR;
}
/* Autodetect the record length */
if ( reclen <= 0 )
{
detlen = ms_detect (record, recbuflen);
/* No data record detected */
if ( detlen < 0 )
{
return MS_NOTSEED;
}
/* Found record but could not determine length */
if ( detlen == 0 )
{
return 256;
}
if ( verbose > 2 )
{
ms_log (1, "Detected record length of %d bytes\n", detlen);
}
reclen = detlen;
}
/* Check that record length is in supported range */
if ( reclen < MINRECLEN || reclen > MAXRECLEN )
{
ms_log (2, "Record length is out of range: %d (allowed: %d to %d)\n",
reclen, MINRECLEN, MAXRECLEN);
return MS_OUTOFRANGE;
}
/* Check if more data is required, return hint */
if ( reclen > recbuflen )
{
if ( verbose > 2 )
ms_log (1, "Detected %d byte record, need %d more bytes\n",
reclen, (reclen - recbuflen));
return (reclen - recbuflen);
}
/* Unpack record */
if ( (retcode = msr_unpack (record, reclen, ppmsr, dataflag, verbose)) != MS_NOERROR )
{
msr_free (ppmsr);
return retcode;
}
return MS_NOERROR;
} /* End of msr_parse() */