int main( int argc, char **argv ) { SHM_INFO region; long RingKey; /* Key to the transport ring to read from */ MSG_LOGO getlogo[NLOGO], logo; long gotsize; char msg[MAX_TRACEBUF_SIZ]; char *getSta, *getComp, *getNet, *getLoc, *inRing; char wildSta, wildComp, wildNet, wildLoc; unsigned char Type_Mseed; unsigned char Type_TraceBuf,Type_TraceBuf2; unsigned char Type_TraceComp, Type_TraceComp2; unsigned char InstWildcard, ModWildcard; short *short_data; int *long_data; TRACE2_HEADER *trh; char orig_datatype[3]; char stime[256]; char etime[256]; int i; int rc; int nLogo = NLOGO; static unsigned char InstId; /* local installation id */ char ModName[MAX_MOD_STR]; char *modid_name; unsigned char sequence_number = 0; // int statistics_flag; time_t monitor_start_time = 0; double start_time, end_time; unsigned long packet_total=0; unsigned long packet_total_size=0; MSRecord *msr = NULL; /* mseed record */ logit_init("ring2mongo", 200, 256, 1); char *mongo_user = getenv("MONGO_USER"); char *mongo_passwd=getenv("MONGO_PASSWD"); char *mongo_host=getenv("MONGO_HOST"); char *mongo_port=getenv("MONGO_PORT"); if(mongo_user == NULL || mongo_passwd==NULL || mongo_host== NULL || mongo_port==NULL){ printf("Ensure the following ENV varibles are set\n *MONGO_USER\n *MONGO_PASSWD\n *MONGO_HOST\n *MONGO_PORT\n"); exit(1); return(1); } char mongo_str[128]="mongodb://"; strcat(mongo_str, mongo_user); strcat(mongo_str, ":"); strcat(mongo_str, mongo_passwd); strcat(mongo_str, "@"); strcat(mongo_str, mongo_host); strcat(mongo_str, ":"); strcat(mongo_str, mongo_port); strcat(mongo_str, "/"); //mongo stuff mongoc_client_t *m_client; mongoc_collection_t *m_collection; mongoc_init (); m_client = mongoc_client_new (mongo_str); m_collection = mongoc_client_get_collection (m_client, "waveforms", "ring"); mongoc_bulk_operation_t *m_bulk; bson_error_t m_error; bson_t *m_doc; bson_t m_reply; bson_t *m_data; /*wait for this many messages before performing bulk write For real time continuous data it shoudl be ~ number of channels on ring*/ int MONGO_BULK_MAX = 30; int m_count = 0; /* counter for mongo bulk */ bool m_ret; /*end mongo stuff*/ /* Initialize pointers **********************/ trh = (TRACE2_HEADER *) msg; long_data = (int *)( msg + sizeof(TRACE2_HEADER) ); short_data = (short *)( msg + sizeof(TRACE2_HEADER) ); /* Check command line argument *****************************/ if ( argc < 2 || argc > 6) { if(argc > 6) fprintf(stderr, "ring2mongo: Too many parameters\n"); fprintf(stderr,"Usage: %s <ring name> [sta] [comp] [net] [loc] \n", argv[0]); fprintf(stderr, " Note: All parameters are positional, and all but first are optional.\n"); fprintf(stderr, " the full data contained in the packet is printed out.\n"); fprintf(stderr, " If sta comp net (but not loc) are specified then only TraceBuf\n"); fprintf(stderr, " packets will be fetched (not TraceBuf2); otherwise both are fetched.\n"); fprintf(stderr, " Example: %s WAVE_RING PHOB wild NC wild n\n", argv[0]); fprintf(stderr, " MSEED capability starting with version 2.5.1, prints mseed headers\n"); fprintf(stderr, " of TYPE_MSEED packets (no filtering yet).\n"); exit( 1 ); return 1; } /* process given parameters */ inRing = argv[1]; /* first parameter is ring name */ /* any parameters not given are set as wildcards */ getSta = getComp = getNet = getLoc = ""; wildSta = wildComp = wildNet = wildLoc = 1; if(argc > 2) { /* at least station parameter given */ getSta = argv[2]; wildSta = IsWild(getSta); if(argc > 3) { /* channel (component) parameter given */ getComp = argv[3]; wildComp = IsWild(getComp); if(argc > 4) { /* network parameter given */ getNet = argv[4]; wildNet = IsWild(getNet); if(argc > 5) { /* location parameter given (SCNL) */ getLoc = argv[5]; wildLoc = IsWild(getLoc); } else { /* SCN without location parameter given */ nLogo = 3; /* do not include tracebuf2s in search */ } } } } /* Attach to ring *****************/ if ((RingKey = GetKey( inRing )) == -1 ) { fprintf( stderr, "Invalid RingName; exiting!\n" ); exit( -1 ); return -1; } tport_attach( ®ion, RingKey ); /* Look up local installation id *****************************/ if ( GetLocalInst( &InstId ) != 0 ) { fprintf(stderr, "ring2mongo: error getting local installation id; exiting!\n" ); exit( -1 ); return -1; } /* Specify logos to get ***********************/ if ( GetType( "TYPE_MSEED", &Type_Mseed ) != 0 ) { fprintf(stderr, "%s: WARNING: Invalid message type <TYPE_MSEED>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); Type_Mseed = TYPE_NOTUSED; } if ( GetType( "TYPE_TRACEBUF", &Type_TraceBuf ) != 0 ) { fprintf(stderr, "%s: Invalid message type <TYPE_TRACEBUF>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); exit( -1 ); return -1; } if ( GetType( "TYPE_TRACE_COMP_UA", &Type_TraceComp ) != 0 ) { fprintf(stderr, "%s: Invalid message type <TYPE_TRACE_COMP_UA>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); exit( -1 ); return -1; } if ( GetType( "TYPE_TRACEBUF2", &Type_TraceBuf2 ) != 0 ) { fprintf(stderr, "%s: Invalid message type <TYPE_TRACEBUF2>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); exit( -1 ); return -1; } if ( GetType( "TYPE_TRACE2_COMP_UA", &Type_TraceComp2 ) != 0 ) { fprintf(stderr,"%s: Invalid message type <TYPE_TRACE2_COMP_UA>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); exit( -1 ); return -1; } if ( GetModId( "MOD_WILDCARD", &ModWildcard ) != 0 ) { fprintf(stderr, "%s: Invalid moduleid <MOD_WILDCARD>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); exit( -1 ); return -1; } if ( GetInst( "INST_WILDCARD", &InstWildcard ) != 0 ) { fprintf(stderr, "%s: Invalid instid <INST_WILDCARD>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); exit( -1 ); return -1; } for( i=0; i<nLogo; i++ ) { getlogo[i].instid = InstWildcard; getlogo[i].mod = ModWildcard; } getlogo[0].type = Type_Mseed; getlogo[1].type = Type_TraceBuf; getlogo[2].type = Type_TraceComp; if (nLogo >= 4) { /* if nLogo=5 then include TraceBuf2s */ getlogo[3].type = Type_TraceBuf2; getlogo[4].type = Type_TraceComp2; } logit("", "Starting ring2mongo"); /* Flush the ring *****************/ while ( tport_copyfrom( ®ion, getlogo, nLogo, &logo, &gotsize, (char *)&msg, MAX_TRACEBUF_SIZ, &sequence_number ) != GET_NONE ){ packet_total++; packet_total_size+=gotsize; } logit( "et", "ring2mongo: inRing flushed %ld packets of %ld bytes total.\n", packet_total, packet_total_size); while (tport_getflag( ®ion ) != TERMINATE ) { rc = tport_copyfrom( ®ion, getlogo, nLogo, &logo, &gotsize, msg, MAX_TRACEBUF_SIZ, &sequence_number ); if ( rc == GET_NONE ){ sleep_ew( 200 ); continue; } if ( rc == GET_TOOBIG ){ logit("et", "ring2mongo: retrieved message too big (%ld) for msg\n", gotsize ); continue; } if ( rc == GET_NOTRACK ) logit("et", "ring2mongo: Tracking error.\n"); if ( rc == GET_MISS_LAPPED ) logit("et", "ring2mongo: Got lapped on the ring.\n"); if ( rc == GET_MISS_SEQGAP ) logit( "et", "ring2mongo: Gap in sequence numbers\n"); if ( rc == GET_MISS ) logit( "et", "ring2mongo: Missed messages\n"); /* Check SCNL of the retrieved message */ if (Type_Mseed != TYPE_NOTUSED && logo.type == Type_Mseed) { /* Unpack record header and not data samples */ /*hard coded zero for dataflag(1) and verbose(0)for ring2mongo*/ if ( msr_unpack (msg, gotsize, &msr, 1, 0) != MS_NOERROR) { logit("et", "Error parsing mseed record\n"); continue; } /* Print record information */ msr_print (msr, 0); msr_free (&msr); continue; } /*end Mseed*/ if ( (wildSta || (strcmp(getSta,trh->sta) ==0)) && (wildComp || (strcmp(getComp,trh->chan)==0)) && (wildNet || (strcmp(getNet,trh->net) ==0)) && (((logo.type == Type_TraceBuf2 || logo.type == Type_TraceComp2) && (wildLoc || (strcmp(getLoc,trh->loc) == 0))) || ( (logo.type == Type_TraceBuf || logo.type == Type_TraceComp)))) { strcpy(orig_datatype, trh->datatype); char scnl[20]; scnl[0] = 0; strcat( scnl, trh->sta); strcat( scnl, "."); strcat( scnl, trh->chan); strcat( scnl, "."); strcat( scnl, trh->net); strcat( scnl, "."); strcat( scnl, trh->loc); if(WaveMsg2MakeLocal( trh ) < 0){ char dt[3]; /* now put a space if there are any punctuation marks */ for ( i=0; i<15; i++ ) { if ( !isalnum(scnl[i]) && !ispunct(scnl[i])) scnl[i] = ' '; } strncpy( dt, trh->datatype, 2 ); for ( i=0; i<2; i++ ) { if ( !isalnum(dt[i]) && !ispunct(dt[i])) dt[i] = ' '; } dt[i] = 0; logit("et", "WARNING: WaveMsg2MakeLocal rejected tracebuf. Discard (%s).\n", scnl ); logit("et", "\tdatatype=[%s]\n", dt); continue; } /*lets get ready to mongo....*/ if(m_count < MONGO_BULK_MAX){ /*initialize when m_count ==0 */ if(m_count==0){ m_bulk = mongoc_collection_create_bulk_operation (m_collection, true, NULL); } m_data = bson_new (); m_doc = BCON_NEW ("key", BCON_UTF8 (scnl), "nsamp", BCON_INT32 (trh->nsamp), "starttime", BCON_DOUBLE (trh->starttime*1000), "endtime", BCON_DOUBLE (trh->endtime*1000), "samprate", BCON_DOUBLE (trh->samprate), "datatype", BCON_UTF8 (trh->datatype) ); char index[4]; bson_append_array_begin (m_doc, "data", -1, m_data); for ( i = 0; i < trh->nsamp; i++ ){ snprintf(index, 4, "%d", i); if ( (strcmp (trh->datatype, "s2")==0) || (strcmp (trh->datatype, "i2")==0) ){ bson_append_int32 (m_data, index, -1, short_data[i]); }else{ bson_append_int32 (m_data, index, -1, long_data[i]); } } bson_append_array_end (m_doc, m_data); mongoc_bulk_operation_insert (m_bulk, m_doc); bson_destroy (m_doc); bson_destroy (m_data); m_count++; if(m_count==MONGO_BULK_MAX){ /*write and destroty above stuff */ m_ret = mongoc_bulk_operation_execute (m_bulk, &m_reply, &m_error); if (!m_ret){ logit ("et", "Error: %s\n", m_error.message); } bson_destroy (&m_reply); mongoc_bulk_operation_destroy (m_bulk); m_count = 0; } } } } /* end of while loop */ logit("et", "signing off"); exit (0); return 0; }
int main (int argc, char **argv) { struct filelink *flp; MSRecord *msr = 0; MSTraceList *mstl = 0; FILE *bfp = 0; FILE *ofp = 0; int retcode = MS_NOERROR; char envvariable[100]; int dataflag = 0; long long int totalrecs = 0; long long int totalsamps = 0; long long int totalfiles = 0; off_t filepos = 0; char srcname[50]; char stime[30]; /* Set default error message prefix */ ms_loginit (NULL, NULL, NULL, "ERROR: "); /* Process given parameters (command line and parameter file) */ if ( processparam (argc, argv) < 0 ) return 1; /* Setup encoding environment variable if specified, ugly kludge */ if ( encodingstr ) { snprintf (envvariable, sizeof(envvariable), "UNPACK_DATA_FORMAT=%s", encodingstr); if ( putenv (envvariable) ) { ms_log (2, "Cannot set environment variable UNPACK_DATA_FORMAT\n"); return 1; } } /* Open the integer output file if specified */ if ( binfile ) { if ( strcmp (binfile, "-") == 0 ) { bfp = stdout; } else if ( (bfp = fopen (binfile, "wb")) == NULL ) { ms_log (2, "Cannot open binary data output file: %s (%s)\n", binfile, strerror(errno)); return 1; } } /* Open the output file if specified */ if ( outfile ) { if ( strcmp (outfile, "-") == 0 ) { ofp = stdout; } else if ( (ofp = fopen (outfile, "wb")) == NULL ) { ms_log (2, "Cannot open output file: %s (%s)\n", outfile, strerror(errno)); return 1; } } if ( printdata || binfile ) dataflag = 1; if ( tracegapsum || tracegaponly ) mstl = mstl_init (NULL); flp = filelist; while ( flp != 0 ) { if ( verbose >= 2 ) { if ( flp->offset ) ms_log (1, "Processing: %s (starting at byte %lld)\n", flp->filename, flp->offset); else ms_log (1, "Processing: %s\n", flp->filename); } /* Set starting byte offset if supplied as negative file position */ filepos = - flp->offset; /* Loop over the input file */ while ( reccntdown != 0 ) { if ( (retcode = ms_readmsr (&msr, flp->filename, reclen, &filepos, NULL, skipnotdata, 0, verbose)) != MS_NOERROR ) break; /* Check if record matches start/end time criteria */ if ( starttime != HPTERROR || endtime != HPTERROR ) { hptime_t recendtime = msr_endtime (msr); if ( starttime != HPTERROR && (msr->starttime < starttime && ! (msr->starttime <= starttime && recendtime >= starttime)) ) { if ( verbose >= 3 ) { msr_srcname (msr, srcname, 1); ms_hptime2seedtimestr (msr->starttime, stime, 1); ms_log (1, "Skipping (starttime) %s, %s\n", srcname, stime); } continue; } if ( endtime != HPTERROR && (recendtime > endtime && ! (msr->starttime <= endtime && recendtime >= endtime)) ) { if ( verbose >= 3 ) { msr_srcname (msr, srcname, 1); ms_hptime2seedtimestr (msr->starttime, stime, 1); ms_log (1, "Skipping (starttime) %s, %s\n", srcname, stime); } continue; } } if ( match || reject ) { /* Generate the srcname with the quality code */ msr_srcname (msr, srcname, 1); /* Check if record is matched by the match regex */ if ( match ) { if ( regexec ( match, srcname, 0, 0, 0) != 0 ) { if ( verbose >= 3 ) { ms_hptime2seedtimestr (msr->starttime, stime, 1); ms_log (1, "Skipping (match) %s, %s\n", srcname, stime); } continue; } } /* Check if record is rejected by the reject regex */ if ( reject ) { if ( regexec ( reject, srcname, 0, 0, 0) == 0 ) { if ( verbose >= 3 ) { ms_hptime2seedtimestr (msr->starttime, stime, 1); ms_log (1, "Skipping (reject) %s, %s\n", srcname, stime); } continue; } } } if ( reccntdown > 0 ) reccntdown--; totalrecs++; totalsamps += msr->samplecnt; if ( ! tracegaponly ) { if ( printoffset ) ms_log (0, "%-10lld", (long long) filepos); if ( printlatency ) ms_log (0, "%-10.6g secs ", msr_host_latency(msr)); if ( printraw ) ms_parse_raw (msr->record, msr->reclen, ppackets, -1); else msr_print (msr, ppackets); } if ( tracegapsum || tracegaponly ) mstl_addmsr (mstl, msr, dataquality, 1, timetol, sampratetol); if ( dataflag ) { /* Parse the record (again) and unpack the data */ int rv = msr_unpack (msr->record, msr->reclen, &msr, 1, verbose); if ( rv == MS_NOERROR && printdata && ! tracegaponly ) { int line, col, cnt, samplesize; int lines = (msr->numsamples / 6) + 1; void *sptr; if ( (samplesize = ms_samplesize(msr->sampletype)) == 0 ) { ms_log (2, "Unrecognized sample type: %c\n", msr->sampletype); } if ( msr->sampletype == 'a' ) { char *ascii = (char *)msr->datasamples; int length = msr->numsamples; ms_log (0, "ASCII Data:\n"); /* Print maximum log message segments */ while ( length > (MAX_LOG_MSG_LENGTH-1) ) { ms_log (0, "%.*s", (MAX_LOG_MSG_LENGTH-1), ascii); ascii += MAX_LOG_MSG_LENGTH-1; length -= MAX_LOG_MSG_LENGTH-1; } /* Print any remaining ASCII and add a newline */ if ( length > 0 ) { ms_log (0, "%.*s\n", length, ascii); } else { ms_log (0, "\n"); } } else for ( cnt = 0, line = 0; line < lines; line++ ) { for ( col = 0; col < 6 ; col ++ ) { if ( cnt < msr->numsamples ) { sptr = (char*)msr->datasamples + (cnt * samplesize); if ( msr->sampletype == 'i' ) ms_log (0, "%10d ", *(int32_t *)sptr); else if ( msr->sampletype == 'f' ) ms_log (0, "%10.8g ", *(float *)sptr); else if ( msr->sampletype == 'd' ) ms_log (0, "%10.10g ", *(double *)sptr); cnt++; } } ms_log (0, "\n"); /* If only printing the first 6 samples break out here */ if ( printdata == 1 ) break; } } if ( binfile ) { uint8_t samplesize = ms_samplesize (msr->sampletype); if ( samplesize ) { fwrite (msr->datasamples, samplesize, msr->numsamples, bfp); } else { ms_log (1, "Cannot write to binary file, unknown sample type: %c\n", msr->sampletype); } } } if ( outfile ) { fwrite (msr->record, 1, msr->reclen, ofp); } } /* Print error if not EOF and not counting down records */ if ( retcode != MS_ENDOFFILE && reccntdown != 0 ) { ms_log (2, "Cannot read %s: %s\n", flp->filename, ms_errorstr(retcode)); ms_readmsr (&msr, NULL, 0, NULL, NULL, 0, 0, 0); exit (1); } /* Make sure everything is cleaned up */ ms_readmsr (&msr, NULL, 0, NULL, NULL, 0, 0, 0); totalfiles++; flp = flp->next; } /* End of looping over file list */ if ( binfile ) fclose (bfp); if ( outfile ) fclose (ofp); if ( basicsum ) ms_log (0, "Files: %lld, Records: %lld, Samples: %lld\n", totalfiles, totalrecs, totalsamps); if ( tracegapsum || tracegaponly ) { if ( tracegapsum == 1 || tracegaponly == 1 ) { mstl_printtracelist (mstl, timeformat, 1, tracegaps); } if ( tracegapsum == 2 || tracegaponly == 2 ) { mstl_printgaplist (mstl, timeformat, mingapptr, maxgapptr); } if ( tracegaponly == 3 ) { mstl_printsynclist (mstl, NULL, 1); } } if ( mstl ) mstl_free (&mstl, 0); return 0; } /* End of main() */
/********************************************************************** * msr_parse: * * This routine will attempt to parse (detect and unpack) a Mini-SEED * record from a specified memory buffer and populate a supplied * MSRecord structure. * * If reclen is less than or equal to 0 the length of record is * automatically detected otherwise reclen should be the correct * record length. * * For auto detection of record length the record should include a * 1000 blockette or be followed by another record header in the * buffer. * * dataflag will be passed directly to msr_unpack(). * * Return values: * 0 : Success, populates the supplied MSRecord. * >0 : Data record detected but not enough data is present, the * return value is a hint of how many more bytes are needed. * <0 : libmseed error code (listed in libmseed.h) is returned. *********************************************************************/ int msr_parse ( char *record, int recbuflen, MSRecord **ppmsr, int reclen, flag dataflag, flag verbose ) { int detlen = 0; int retcode = 0; if ( ! ppmsr ) return MS_GENERROR; if ( ! record ) return MS_GENERROR; /* Sanity check: record length cannot be larger than buffer */ if ( reclen > 0 && reclen > recbuflen ) { ms_log (2, "ms_parse() Record length (%d) cannot be larger than buffer (%d)\n", reclen, recbuflen); return MS_GENERROR; } /* Autodetect the record length */ if ( reclen <= 0 ) { detlen = ms_detect (record, recbuflen); /* No data record detected */ if ( detlen < 0 ) { return MS_NOTSEED; } /* Found record but could not determine length */ if ( detlen == 0 ) { return 256; } if ( verbose > 2 ) { ms_log (1, "Detected record length of %d bytes\n", detlen); } reclen = detlen; } /* Check that record length is in supported range */ if ( reclen < MINRECLEN || reclen > MAXRECLEN ) { ms_log (2, "Record length is out of range: %d (allowed: %d to %d)\n", reclen, MINRECLEN, MAXRECLEN); return MS_OUTOFRANGE; } /* Check if more data is required, return hint */ if ( reclen > recbuflen ) { if ( verbose > 2 ) ms_log (1, "Detected %d byte record, need %d more bytes\n", reclen, (reclen - recbuflen)); return (reclen - recbuflen); } /* Unpack record */ if ( (retcode = msr_unpack (record, reclen, ppmsr, dataflag, verbose)) != MS_NOERROR ) { msr_free (ppmsr); return retcode; } return MS_NOERROR; } /* End of msr_parse() */