static void
bulk_collation (mongoc_collection_t *collection)
{
mongoc_bulk_operation_t *bulk;
bson_t *opts;
bson_t *doc;
bson_t *selector;
bson_t *update;
bson_error_t error;
bson_t reply;
char *str;
uint32_t ret;
/* insert {_id: "one"} and {_id: "One"} */
bulk = mongoc_collection_create_bulk_operation_with_opts (
collection, NULL);
doc = BCON_NEW ("_id", BCON_UTF8 ("one"));
mongoc_bulk_operation_insert (bulk, doc);
bson_destroy (doc);
doc = BCON_NEW ("_id", BCON_UTF8 ("One"));
mongoc_bulk_operation_insert (bulk, doc);
bson_destroy (doc);
/* "One" normally sorts before "one"; make "one" come first */
opts = BCON_NEW ("collation",
"{",
"locale",
BCON_UTF8 ("en_US"),
"caseFirst",
BCON_UTF8 ("lower"),
"}");
/* set x=1 on the document with _id "One", which now sorts after "one" */
update = BCON_NEW ("$set", "{", "x", BCON_INT64 (1), "}");
selector = BCON_NEW ("_id", "{", "$gt", BCON_UTF8 ("one"), "}");
mongoc_bulk_operation_update_one_with_opts (
bulk, selector, update, opts, &error);
ret = mongoc_bulk_operation_execute (bulk, &reply, &error);
str = bson_as_canonical_extended_json (&reply, NULL);
printf ("%s\n", str);
bson_free (str);
if (!ret) {
printf ("Error: %s\n", error.message);
}
bson_destroy (&reply);
bson_destroy (update);
bson_destroy (selector);
bson_destroy (opts);
mongoc_bulk_operation_destroy (bulk);
}
static void
bulk3 (mongoc_collection_t *collection)
{
mongoc_bulk_operation_t *bulk;
bson_error_t error;
bson_t *query;
bson_t *doc;
bson_t reply;
char *str;
bool ret;
/* false indicates unordered */
bulk = mongoc_collection_create_bulk_operation (collection, false, NULL);
/* Add a document */
doc = BCON_NEW ("_id", BCON_INT32 (1));
mongoc_bulk_operation_insert (bulk, doc);
bson_destroy (doc);
/* remove {_id: 2} */
query = BCON_NEW ("_id", BCON_INT32 (2));
mongoc_bulk_operation_remove_one (bulk, query);
bson_destroy (query);
/* insert {_id: 3} */
doc = BCON_NEW ("_id", BCON_INT32 (3));
mongoc_bulk_operation_insert (bulk, doc);
bson_destroy (doc);
/* replace {_id:4} {'i': 1} */
query = BCON_NEW ("_id", BCON_INT32 (4));
doc = BCON_NEW ("i", BCON_INT32 (1));
mongoc_bulk_operation_replace_one (bulk, query, doc, false);
bson_destroy (query);
bson_destroy (doc);
ret = mongoc_bulk_operation_execute (bulk, &reply, &error);
str = bson_as_json (&reply, NULL);
printf ("%s\n", str);
bson_free (str);
if (!ret) {
printf ("Error: %s\n", error.message);
}
bson_destroy (&reply);
mongoc_bulk_operation_destroy (bulk);
}
int mongodb_module_process(fieldset_t *fs)
{
bson_t *doc;
if (!bulk) {
return EXIT_FAILURE;
}
if (buffer_fill == BUFFER_SIZE) {
if (mongodb_module_flush()) {
return EXIT_FAILURE;
}
}
doc = bson_new();
for (int i=0; i < fs->len; i++) {
field_t *f = &(fs->fields[i]);
if (f->type == FS_STRING) {
BSON_APPEND_UTF8(doc,f->name,f->value.ptr);
} else if (f->type == FS_UINT64) {
BSON_APPEND_INT64(doc,f->name,(uint64_t) f->value.num);
} else if (f->type == FS_BINARY) {
BSON_APPEND_BINARY(doc,f->name, BSON_SUBTYPE_BINARY,f->value.ptr, f->len);
} else if (f->type == FS_NULL) {
// do nothing
} else {
log_fatal("mongodb", "received unknown output type");
}
}
mongoc_bulk_operation_insert(bulk,doc);
buffer_fill++;
return EXIT_SUCCESS;
}
static void
bulk1 (mongoc_collection_t *collection)
{
mongoc_bulk_operation_t *bulk;
bson_error_t error;
bson_t *doc;
bson_t reply;
char *str;
bool ret;
int i;
bulk = mongoc_collection_create_bulk_operation (collection, true, NULL);
for (i = 0; i < 10000; i++) {
doc = BCON_NEW ("i", BCON_INT32 (i));
mongoc_bulk_operation_insert (bulk, doc);
bson_destroy (doc);
}
ret = mongoc_bulk_operation_execute (bulk, &reply, &error);
str = bson_as_json (&reply, NULL);
printf ("%s\n", str);
bson_free (str);
if (!ret) {
fprintf (stderr, "Error: %s\n", error.message);
}
bson_destroy (&reply);
mongoc_bulk_operation_destroy (bulk);
}
int
lua_mongo_collection_insert_one (lua_State *L)
{
collection_t *collection;
mongoc_bulk_operation_t *bulk_insert = NULL;
bson_t bson_doc = BSON_INITIALIZER;
bson_error_t error;
bson_t reply = BSON_INITIALIZER;
bool ret;
bool throw_error = false;
collection = (collection_t *)luaL_checkudata(L, 1, "lua_mongoc_collection");
if (!(lua_istable(L, 2))) {
luaL_error(L, "second input must be a table");
} else {
if (!(lua_table_to_bson(L, &bson_doc, 2, true, &error))) {
throw_error = true;
goto DONE;
}
}
bulk_insert = mongoc_collection_create_bulk_operation(collection->c_collection,
true, NULL);
mongoc_bulk_operation_insert(bulk_insert, &bson_doc);
ret = mongoc_bulk_operation_execute (bulk_insert, &reply, &error);
if (!ret) {
throw_error = true;
goto DONE;
}
if (!(generate_InsertOneResult(L, ret, 2, &error))) {
throw_error = true;
goto DONE;
}
DONE:
bson_destroy(&bson_doc);
bson_destroy(&reply);
if (bulk_insert) {
mongoc_bulk_operation_destroy(bulk_insert);
}
if (throw_error) {
luaL_error(L, error.message);
}
return 1;
}
static void
bulk4 (mongoc_collection_t *collection)
{
mongoc_write_concern_t *wc;
mongoc_bulk_operation_t *bulk;
bson_error_t error;
bson_t *doc;
bson_t reply;
char *str;
bool ret;
wc = mongoc_write_concern_new ();
mongoc_write_concern_set_w (wc, 4);
mongoc_write_concern_set_wtimeout (wc, 100); /* milliseconds */
bulk = mongoc_collection_create_bulk_operation (collection, true, wc);
/* Two inserts */
doc = BCON_NEW ("_id", BCON_INT32 (10));
mongoc_bulk_operation_insert (bulk, doc);
bson_destroy (doc);
doc = BCON_NEW ("_id", BCON_INT32 (11));
mongoc_bulk_operation_insert (bulk, doc);
bson_destroy (doc);
ret = mongoc_bulk_operation_execute (bulk, &reply, &error);
str = bson_as_json (&reply, NULL);
printf ("%s\n", str);
bson_free (str);
if (!ret) {
printf ("Error: %s\n", error.message);
}
bson_destroy (&reply);
mongoc_bulk_operation_destroy (bulk);
mongoc_write_concern_destroy (wc);
}
static void
bulk5_success (mongoc_collection_t *collection)
{
mongoc_bulk_operation_t *bulk;
bson_error_t error;
bson_t *doc;
bson_t reply;
char *str;
bool ret;
bulk = mongoc_collection_create_bulk_operation (collection, true, NULL);
/* Allow this document to bypass document validation.
* NOTE: When authentication is enabled, the authenticated user must have
* either the "dbadmin" or "restore" roles to bypass document validation */
mongoc_bulk_operation_set_bypass_document_validation (bulk, true);
/* Two inserts */
doc = BCON_NEW ("_id", BCON_INT32 (31));
mongoc_bulk_operation_insert (bulk, doc);
bson_destroy (doc);
doc = BCON_NEW ("_id", BCON_INT32 (32));
mongoc_bulk_operation_insert (bulk, doc);
bson_destroy (doc);
ret = mongoc_bulk_operation_execute (bulk, &reply, &error);
str = bson_as_json (&reply, NULL);
printf ("%s\n", str);
bson_free (str);
if (!ret) {
printf ("Error: %s\n", error.message);
}
bson_destroy (&reply);
mongoc_bulk_operation_destroy (bulk);
}
static void
bulk5_fail (mongoc_collection_t *collection)
{
mongoc_bulk_operation_t *bulk;
bson_error_t error;
bson_t *doc;
bson_t reply;
char *str;
bool ret;
bulk = mongoc_collection_create_bulk_operation (collection, true, NULL);
/* Two inserts */
doc = BCON_NEW ("_id", BCON_INT32 (31));
mongoc_bulk_operation_insert (bulk, doc);
bson_destroy (doc);
doc = BCON_NEW ("_id", BCON_INT32 (32));
mongoc_bulk_operation_insert (bulk, doc);
bson_destroy (doc);
/* The above documents do not comply to the schema validation rules
* we created previously, so this will result in an error */
ret = mongoc_bulk_operation_execute (bulk, &reply, &error);
str = bson_as_json (&reply, NULL);
printf ("%s\n", str);
bson_free (str);
if (!ret) {
printf ("Error: %s\n", error.message);
}
bson_destroy (&reply);
mongoc_bulk_operation_destroy (bulk);
}
SEXP R_mongo_restore(SEXP con, SEXP ptr_col, SEXP verb) {
bool verbose = Rf_asLogical(verb);
mongoc_collection_t *col = r2col(ptr_col);
bson_reader_t *reader = bson_reader_new_from_handle(con, bson_reader_feed, bson_reader_finalize);
mongoc_bulk_operation_t *bulk = NULL;
const bson_t *b;
bson_error_t err;
int count = 0;
int i = 0;
bool done = false;
bson_t reply;
while(!done) {
//note: default opts uses {ordered:true}
bulk = mongoc_collection_create_bulk_operation_with_opts(col, NULL);
for(i = 0; i < 1000; i++){
if(!(b = bson_reader_read (reader, &done)))
break;
mongoc_bulk_operation_insert (bulk, b);
count++;
}
if(i == 0)
break;
if(!mongoc_bulk_operation_execute (bulk, &reply, &err)){
bson_reader_destroy(reader);
mongoc_bulk_operation_destroy (bulk);
Rf_error(err.message);
}
if(verbose)
Rprintf("\rRestored %d records...", count);
}
if(verbose)
Rprintf("\rDone! Inserted total of %d records.\n", count);
if (!done)
Rf_warning("Failed to read all documents.\n");
bson_reader_destroy(reader);
mongoc_bulk_operation_destroy (bulk);
return Rf_ScalarInteger(count);
}
SEXP R_mongo_collection_insert_page(SEXP ptr_col, SEXP json_vec, SEXP stop_on_error){
if(!Rf_isString(json_vec) || !Rf_length(json_vec))
stop("json_vec must be character string of at least length 1");
//ordered means serial execution
bool ordered = Rf_asLogical(stop_on_error);
//create bulk operation
bson_error_t err;
bson_t *b;
bson_t reply;
mongoc_bulk_operation_t *bulk = mongoc_collection_create_bulk_operation_with_opts (r2col(ptr_col), NULL);
for(int i = 0; i < Rf_length(json_vec); i++){
b = bson_new_from_json ((uint8_t*) Rf_translateCharUTF8(Rf_asChar(STRING_ELT(json_vec, i))), -1, &err);
if(!b){
mongoc_bulk_operation_destroy (bulk);
stop(err.message);
}
mongoc_bulk_operation_insert(bulk, b);
bson_destroy (b);
b = NULL;
}
//execute bulk operation
bool success = mongoc_bulk_operation_execute (bulk, &reply, &err);
mongoc_bulk_operation_destroy (bulk);
//check for errors
if(!success){
if(ordered){
Rf_errorcall(R_NilValue, err.message);
} else {
Rf_warningcall(R_NilValue, "Not all inserts were successful: %s\n", err.message);
}
}
//get output
SEXP out = PROTECT(bson2list(&reply));
bson_destroy (&reply);
UNPROTECT(1);
return out;
}
/* {{{ proto mixed BulkWrite::insert(array|object $document)
Adds an insert operation to the bulk */
PHP_METHOD(BulkWrite, insert)
{
php_phongo_bulkwrite_t *intern;
zval *document;
bson_t *bson;
bson_t *bson_out = NULL;
int bson_flags = PHONGO_BSON_ADD_ID;
DECLARE_RETURN_VALUE_USED
SUPPRESS_UNUSED_WARNING(return_value_ptr)
intern = Z_BULKWRITE_OBJ_P(getThis());
if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "A", &document) == FAILURE) {
return;
}
if (return_value_used) {
bson_flags |= PHONGO_BSON_RETURN_ID;
}
bson = bson_new();
phongo_zval_to_bson(document, bson_flags, bson, &bson_out TSRMLS_CC);
mongoc_bulk_operation_insert(intern->bulk, bson);
bson_clear(&bson);
intern->num_ops++;
if (bson_out && return_value_used) {
bson_iter_t iter;
if (bson_iter_init_find(&iter, bson_out, "_id")) {
php_phongo_objectid_new_from_oid(return_value, bson_iter_oid(&iter) TSRMLS_CC);
bson_clear(&bson_out);
return;
}
bson_clear(&bson_out);
}
}
int
lua_mongo_collection_insert_many (lua_State *L)
{
collection_t *collection;
mongoc_bulk_operation_t *bulk_insert = NULL;
bool ordered;
bson_error_t error;
bson_t reply = BSON_INITIALIZER;
bool ret;
bool throw_error = false;
collection = (collection_t *)luaL_checkudata(L, 1, "lua_mongoc_collection");
if (!(lua_istable(L, 2))) {
luaL_error(L, "second input must be a table");
}
if (!(lua_isboolean(L, 3))) {
luaL_error(L, "ordered parameter must be a boolean");
} else {
ordered = lua_toboolean(L, 3);
}
bulk_insert = mongoc_collection_create_bulk_operation(collection->c_collection,
ordered, NULL);
lua_pushnil(L);
int num_elements, lua_index;
for (num_elements = 0, lua_index = 1;
lua_next(L, 2) != 0;
num_elements++, lua_index++)
{
bson_t bson_doc = BSON_INITIALIZER;
if (!(lua_isnumber(L, -2)) || lua_tonumber(L, -2) != lua_index) {
throw_error = true;
strncpy(error.message,
"malformed array of documents",
sizeof(error.message));
goto DONE;
}
if (!(lua_table_to_bson(L, &bson_doc, -1, true, &error))) {
throw_error = true;
goto DONE;
}
mongoc_bulk_operation_insert(bulk_insert, &bson_doc);
lua_pop(L, 1);
bson_destroy(&bson_doc);
}
if (!(mongoc_bulk_operation_execute (bulk_insert, &reply, &error))) {
throw_error = true;
goto DONE;
}
if (!(generate_InsertManyResult(L, &reply, 2, num_elements, &error))) {
throw_error = true;
goto DONE;
}
DONE:
bson_destroy(&reply);
if (bulk_insert){
mongoc_bulk_operation_destroy (bulk_insert);
}
if (throw_error) {
luaL_error(L, error.message);
}
return 1;
}
void run_loader(int thread, char *filename) {
bson_t *record;
mongoc_client_t *conn;
mongoc_collection_t *collection;
bson_error_t error;
bson_t reply;
int count;
FILE *infile;
char *rptr;
char ilinebuf[BUFSIZ];
char rlinebuf[BUFSIZ];
char *ritem;
char *rlast = NULL;
int rfcount = 0;
int batchcount = 0;
char *str;
int total = 0;
//Get the highest used value on that shard so far
conn = mongoc_client_new(DEFAULT_URI);
if (!conn) {
fprintf(stderr, "Failed to parse URI.\n");
exit(1);
}
collection = mongoc_client_get_collection(conn, DATA_DB, DATA_COLLECTION);
long long chunkno = carve_chunk(conn, collection);
printf("Thread %d reading %s\n", thread, filename);
infile = fopen(filename, "r");
if (infile == NULL) {
perror("Opening results file");
exit(1);
}
mongoc_bulk_operation_t *bulk = mongoc_collection_create_bulk_operation(
collection, true, NULL);
if(!bulk)
{
printf("Failed to create bulk op\n");
}
rptr = fgets(rlinebuf, BUFSIZ, infile);
rlinebuf[strlen(rlinebuf) - 1] = '\0';
//Read the Results Line
while (rptr) {
total++;
if (total % (INSERT_THREADS_PER_SHARD * nshards) == thread) {
ritem = strtok_r(rptr, "|", &rlast);
rfcount = 0;
record = bson_new();
//Two part ID - a loader (32 bits for that) and a one_up
bson_append_int64(record, "_id", -1, (chunkno << 32) + total);
while (ritem) {
switch (resulttype[rfcount]) {
case 0:
//printf("%s\n",ritem);
bson_append_utf8(record, resultfields[rfcount], -1, ritem,
-1);
break;
case 1:
bson_append_int32(record, resultfields[rfcount], -1,
atoi(ritem));
break;
case 2:
if (strncmp(ritem, "NULL", 4)) {
struct tm tm;
if (strptime(ritem, "%Y-%m-%d", &tm)) {
time_t t = mktime(&tm); // t is now your desired time_t
bson_append_date_time(record, resultfields[rfcount],
-1, (long long) t * 1000);
}
}
break;
default:
printf("Unknown type col %d = %d\n", rfcount,
resulttype[rfcount]);
}
ritem = strtok_r(NULL, "|", &rlast);
rfcount++;
}
mongoc_bulk_operation_insert(bulk, record);
bson_destroy(record);
if (batchcount == (BATCHSIZE - 1)) {
int ret = mongoc_bulk_operation_execute(bulk, &reply, &error);
if (!ret) {
printf( "Error: %s\n", error.message);
}
if (thread == 0)
printf("%s %d\n", filename, total);
bson_destroy(&reply);
mongoc_bulk_operation_destroy(bulk);
batchcount = 0;
bulk = mongoc_collection_create_bulk_operation(collection, true,
NULL);
} else {
batchcount++;
}
}
//Read next line from file
rptr = fgets(rlinebuf, BUFSIZ, infile);
rlinebuf[strlen(rlinebuf) - 1] = '\0';
}
int ret = mongoc_bulk_operation_execute(bulk, &reply, &error);
if (!ret) {
fprintf(stderr, "Error: %s\n", error.message);
}
if (thread == 0)
printf("%s %d\n", filename, total);
bson_destroy(&reply);
mongoc_collection_destroy(collection);
mongoc_bulk_operation_destroy(bulk);
mongoc_client_destroy(conn);
}
static void
test_bypass_validation (void *context)
{
mongoc_collection_t *collection;
bson_t reply = BSON_INITIALIZER;
mongoc_bulk_operation_t *bulk;
mongoc_database_t *database;
mongoc_write_concern_t *wr;
mongoc_client_t *client;
bson_error_t error;
bson_t *options;
char *collname;
char *dbname;
int r;
int i;
client = test_framework_client_new ();
assert (client);
dbname = gen_collection_name ("dbtest");
collname = gen_collection_name ("bypass");
database = mongoc_client_get_database (client, dbname);
collection = mongoc_database_get_collection (database, collname);
assert (collection);
options = tmp_bson ("{'validator': {'number': {'$gte': 5}}, 'validationAction': 'error'}");
ASSERT_OR_PRINT (mongoc_database_create_collection (database, collname, options, &error), error);
/* {{{ Default fails validation */
bulk = mongoc_collection_create_bulk_operation(collection, true, NULL);
for (i = 0; i < 3; i++) {
bson_t *doc = tmp_bson (bson_strdup_printf ("{'number': 3, 'high': %d }", i));
mongoc_bulk_operation_insert (bulk, doc);
}
r = mongoc_bulk_operation_execute (bulk, &reply, &error);
ASSERT(!r);
ASSERT_ERROR_CONTAINS (error, MONGOC_ERROR_COMMAND, 121, "Document failed validation");
mongoc_bulk_operation_destroy (bulk);
/* }}} */
/* {{{ bypass_document_validation=false Fails validation */
bulk = mongoc_collection_create_bulk_operation(collection, true, NULL);
mongoc_bulk_operation_set_bypass_document_validation (bulk, false);
for (i = 0; i < 3; i++) {
bson_t *doc = tmp_bson (bson_strdup_printf ("{'number': 3, 'high': %d }", i));
mongoc_bulk_operation_insert (bulk, doc);
}
r = mongoc_bulk_operation_execute (bulk, &reply, &error);
ASSERT(!r);
ASSERT_ERROR_CONTAINS (error, MONGOC_ERROR_COMMAND, 121, "Document failed validation");
mongoc_bulk_operation_destroy (bulk);
/* }}} */
/* {{{ bypass_document_validation=true ignores validation */
bulk = mongoc_collection_create_bulk_operation(collection, true, NULL);
mongoc_bulk_operation_set_bypass_document_validation (bulk, true);
for (i = 0; i < 3; i++) {
bson_t *doc = tmp_bson (bson_strdup_printf ("{'number': 3, 'high': %d }", i));
mongoc_bulk_operation_insert (bulk, doc);
}
r = mongoc_bulk_operation_execute (bulk, &reply, &error);
ASSERT_OR_PRINT(r, error);
mongoc_bulk_operation_destroy (bulk);
/* }}} */
/* {{{ w=0 and bypass_document_validation=set fails */
bulk = mongoc_collection_create_bulk_operation(collection, true, NULL);
wr = mongoc_write_concern_new ();
mongoc_write_concern_set_w (wr, 0);
mongoc_bulk_operation_set_write_concern (bulk, wr);
mongoc_bulk_operation_set_bypass_document_validation (bulk, true);
for (i = 0; i < 3; i++) {
bson_t *doc = tmp_bson (bson_strdup_printf ("{'number': 3, 'high': %d }", i));
mongoc_bulk_operation_insert (bulk, doc);
}
r = mongoc_bulk_operation_execute (bulk, &reply, &error);
ASSERT_OR_PRINT(!r, error);
ASSERT_ERROR_CONTAINS (error, MONGOC_ERROR_COMMAND, MONGOC_ERROR_COMMAND_INVALID_ARG,
"Cannot set bypassDocumentValidation for unacknowledged writes");
mongoc_bulk_operation_destroy (bulk);
mongoc_write_concern_destroy (wr);
/* }}} */
ASSERT_OR_PRINT (mongoc_collection_drop (collection, &error), error);
mongoc_collection_destroy (collection);
mongoc_client_destroy (client);
}
int main( int argc, char **argv )
{
SHM_INFO region;
long RingKey; /* Key to the transport ring to read from */
MSG_LOGO getlogo[NLOGO], logo;
long gotsize;
char msg[MAX_TRACEBUF_SIZ];
char *getSta, *getComp, *getNet, *getLoc, *inRing;
char wildSta, wildComp, wildNet, wildLoc;
unsigned char Type_Mseed;
unsigned char Type_TraceBuf,Type_TraceBuf2;
unsigned char Type_TraceComp, Type_TraceComp2;
unsigned char InstWildcard, ModWildcard;
short *short_data;
int *long_data;
TRACE2_HEADER *trh;
char orig_datatype[3];
char stime[256];
char etime[256];
int i;
int rc;
int nLogo = NLOGO;
static unsigned char InstId; /* local installation id */
char ModName[MAX_MOD_STR];
char *modid_name;
unsigned char sequence_number = 0;
// int statistics_flag;
time_t monitor_start_time = 0;
double start_time, end_time;
unsigned long packet_total=0;
unsigned long packet_total_size=0;
MSRecord *msr = NULL; /* mseed record */
logit_init("ring2mongo", 200, 256, 1);
char *mongo_user = getenv("MONGO_USER");
char *mongo_passwd=getenv("MONGO_PASSWD");
char *mongo_host=getenv("MONGO_HOST");
char *mongo_port=getenv("MONGO_PORT");
if(mongo_user == NULL ||
mongo_passwd==NULL ||
mongo_host== NULL ||
mongo_port==NULL){
printf("Ensure the following ENV varibles are set\n *MONGO_USER\n *MONGO_PASSWD\n *MONGO_HOST\n *MONGO_PORT\n");
exit(1);
return(1);
}
char mongo_str[128]="mongodb://";
strcat(mongo_str, mongo_user);
strcat(mongo_str, ":");
strcat(mongo_str, mongo_passwd);
strcat(mongo_str, "@");
strcat(mongo_str, mongo_host);
strcat(mongo_str, ":");
strcat(mongo_str, mongo_port);
strcat(mongo_str, "/");
//mongo stuff
mongoc_client_t *m_client;
mongoc_collection_t *m_collection;
mongoc_init ();
m_client = mongoc_client_new (mongo_str);
m_collection = mongoc_client_get_collection (m_client, "waveforms", "ring");
mongoc_bulk_operation_t *m_bulk;
bson_error_t m_error;
bson_t *m_doc;
bson_t m_reply;
bson_t *m_data;
/*wait for this many messages before performing bulk write
For real time continuous data it shoudl be ~ number of channels on ring*/
int MONGO_BULK_MAX = 30;
int m_count = 0; /* counter for mongo bulk */
bool m_ret;
/*end mongo stuff*/
/* Initialize pointers
**********************/
trh = (TRACE2_HEADER *) msg;
long_data = (int *)( msg + sizeof(TRACE2_HEADER) );
short_data = (short *)( msg + sizeof(TRACE2_HEADER) );
/* Check command line argument
*****************************/
if ( argc < 2 || argc > 6)
{
if(argc > 6)
fprintf(stderr, "ring2mongo: Too many parameters\n");
fprintf(stderr,"Usage: %s <ring name> [sta] [comp] [net] [loc] \n", argv[0]);
fprintf(stderr, " Note: All parameters are positional, and all but first are optional.\n");
fprintf(stderr, " the full data contained in the packet is printed out.\n");
fprintf(stderr, " If sta comp net (but not loc) are specified then only TraceBuf\n");
fprintf(stderr, " packets will be fetched (not TraceBuf2); otherwise both are fetched.\n");
fprintf(stderr, " Example: %s WAVE_RING PHOB wild NC wild n\n", argv[0]);
fprintf(stderr, " MSEED capability starting with version 2.5.1, prints mseed headers\n");
fprintf(stderr, " of TYPE_MSEED packets (no filtering yet).\n");
exit( 1 );
return 1;
}
/* process given parameters */
inRing = argv[1]; /* first parameter is ring name */
/* any parameters not given are set as wildcards */
getSta = getComp = getNet = getLoc = "";
wildSta = wildComp = wildNet = wildLoc = 1;
if(argc > 2)
{ /* at least station parameter given */
getSta = argv[2];
wildSta = IsWild(getSta);
if(argc > 3)
{ /* channel (component) parameter given */
getComp = argv[3];
wildComp = IsWild(getComp);
if(argc > 4)
{ /* network parameter given */
getNet = argv[4];
wildNet = IsWild(getNet);
if(argc > 5)
{ /* location parameter given (SCNL) */
getLoc = argv[5];
wildLoc = IsWild(getLoc);
}
else
{ /* SCN without location parameter given */
nLogo = 3; /* do not include tracebuf2s in search */
}
}
}
}
/* Attach to ring
*****************/
if ((RingKey = GetKey( inRing )) == -1 )
{
fprintf( stderr, "Invalid RingName; exiting!\n" );
exit( -1 );
return -1;
}
tport_attach( ®ion, RingKey );
/* Look up local installation id
*****************************/
if ( GetLocalInst( &InstId ) != 0 )
{
fprintf(stderr, "ring2mongo: error getting local installation id; exiting!\n" );
exit( -1 );
return -1;
}
/* Specify logos to get
***********************/
if ( GetType( "TYPE_MSEED", &Type_Mseed ) != 0 ) {
fprintf(stderr, "%s: WARNING: Invalid message type <TYPE_MSEED>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
Type_Mseed = TYPE_NOTUSED;
}
if ( GetType( "TYPE_TRACEBUF", &Type_TraceBuf ) != 0 ) {
fprintf(stderr, "%s: Invalid message type <TYPE_TRACEBUF>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
exit( -1 );
return -1;
}
if ( GetType( "TYPE_TRACE_COMP_UA", &Type_TraceComp ) != 0 ) {
fprintf(stderr, "%s: Invalid message type <TYPE_TRACE_COMP_UA>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
exit( -1 );
return -1;
}
if ( GetType( "TYPE_TRACEBUF2", &Type_TraceBuf2 ) != 0 ) {
fprintf(stderr, "%s: Invalid message type <TYPE_TRACEBUF2>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
exit( -1 );
return -1;
}
if ( GetType( "TYPE_TRACE2_COMP_UA", &Type_TraceComp2 ) != 0 ) {
fprintf(stderr,"%s: Invalid message type <TYPE_TRACE2_COMP_UA>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
exit( -1 );
return -1;
}
if ( GetModId( "MOD_WILDCARD", &ModWildcard ) != 0 ) {
fprintf(stderr, "%s: Invalid moduleid <MOD_WILDCARD>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
exit( -1 );
return -1;
}
if ( GetInst( "INST_WILDCARD", &InstWildcard ) != 0 ) {
fprintf(stderr, "%s: Invalid instid <INST_WILDCARD>! Missing from earthworm.d or earthworm_global.d\n", argv[0] );
exit( -1 );
return -1;
}
for( i=0; i<nLogo; i++ ) {
getlogo[i].instid = InstWildcard;
getlogo[i].mod = ModWildcard;
}
getlogo[0].type = Type_Mseed;
getlogo[1].type = Type_TraceBuf;
getlogo[2].type = Type_TraceComp;
if (nLogo >= 4) { /* if nLogo=5 then include TraceBuf2s */
getlogo[3].type = Type_TraceBuf2;
getlogo[4].type = Type_TraceComp2;
}
logit("", "Starting ring2mongo");
/* Flush the ring
*****************/
while ( tport_copyfrom( ®ion, getlogo, nLogo, &logo, &gotsize,
(char *)&msg, MAX_TRACEBUF_SIZ, &sequence_number ) != GET_NONE ){
packet_total++;
packet_total_size+=gotsize;
}
logit( "et", "ring2mongo: inRing flushed %ld packets of %ld bytes total.\n",
packet_total, packet_total_size);
while (tport_getflag( ®ion ) != TERMINATE ) {
rc = tport_copyfrom( ®ion, getlogo, nLogo,
&logo, &gotsize, msg, MAX_TRACEBUF_SIZ, &sequence_number );
if ( rc == GET_NONE ){
sleep_ew( 200 );
continue;
}
if ( rc == GET_TOOBIG ){
logit("et", "ring2mongo: retrieved message too big (%ld) for msg\n",
gotsize );
continue;
}
if ( rc == GET_NOTRACK )
logit("et", "ring2mongo: Tracking error.\n");
if ( rc == GET_MISS_LAPPED )
logit("et", "ring2mongo: Got lapped on the ring.\n");
if ( rc == GET_MISS_SEQGAP )
logit( "et", "ring2mongo: Gap in sequence numbers\n");
if ( rc == GET_MISS )
logit( "et", "ring2mongo: Missed messages\n");
/* Check SCNL of the retrieved message */
if (Type_Mseed != TYPE_NOTUSED && logo.type == Type_Mseed) {
/* Unpack record header and not data samples */
/*hard coded zero for dataflag(1) and verbose(0)for ring2mongo*/
if ( msr_unpack (msg, gotsize, &msr, 1, 0) != MS_NOERROR) {
logit("et", "Error parsing mseed record\n");
continue;
}
/* Print record information */
msr_print (msr, 0);
msr_free (&msr);
continue;
}
/*end Mseed*/
if ( (wildSta || (strcmp(getSta,trh->sta) ==0)) &&
(wildComp || (strcmp(getComp,trh->chan)==0)) &&
(wildNet || (strcmp(getNet,trh->net) ==0)) &&
(((logo.type == Type_TraceBuf2 ||
logo.type == Type_TraceComp2) &&
(wildLoc || (strcmp(getLoc,trh->loc) == 0))) ||
( (logo.type == Type_TraceBuf ||
logo.type == Type_TraceComp))))
{
strcpy(orig_datatype, trh->datatype);
char scnl[20];
scnl[0] = 0;
strcat( scnl, trh->sta);
strcat( scnl, ".");
strcat( scnl, trh->chan);
strcat( scnl, ".");
strcat( scnl, trh->net);
strcat( scnl, ".");
strcat( scnl, trh->loc);
if(WaveMsg2MakeLocal( trh ) < 0){
char dt[3];
/* now put a space if there are any punctuation marks */
for ( i=0; i<15; i++ ) {
if ( !isalnum(scnl[i]) && !ispunct(scnl[i]))
scnl[i] = ' ';
}
strncpy( dt, trh->datatype, 2 );
for ( i=0; i<2; i++ ) {
if ( !isalnum(dt[i]) && !ispunct(dt[i]))
dt[i] = ' ';
}
dt[i] = 0;
logit("et", "WARNING: WaveMsg2MakeLocal rejected tracebuf. Discard (%s).\n",
scnl );
logit("et", "\tdatatype=[%s]\n", dt);
continue;
}
/*lets get ready to mongo....*/
if(m_count < MONGO_BULK_MAX){
/*initialize when m_count ==0 */
if(m_count==0){
m_bulk = mongoc_collection_create_bulk_operation (m_collection, true, NULL);
}
m_data = bson_new ();
m_doc = BCON_NEW ("key", BCON_UTF8 (scnl),
"nsamp", BCON_INT32 (trh->nsamp),
"starttime", BCON_DOUBLE (trh->starttime*1000),
"endtime", BCON_DOUBLE (trh->endtime*1000),
"samprate", BCON_DOUBLE (trh->samprate),
"datatype", BCON_UTF8 (trh->datatype)
);
char index[4];
bson_append_array_begin (m_doc, "data", -1, m_data);
for ( i = 0; i < trh->nsamp; i++ ){
snprintf(index, 4, "%d", i);
if ( (strcmp (trh->datatype, "s2")==0) || (strcmp (trh->datatype, "i2")==0) ){
bson_append_int32 (m_data, index, -1, short_data[i]);
}else{
bson_append_int32 (m_data, index, -1, long_data[i]);
}
}
bson_append_array_end (m_doc, m_data);
mongoc_bulk_operation_insert (m_bulk, m_doc);
bson_destroy (m_doc);
bson_destroy (m_data);
m_count++;
if(m_count==MONGO_BULK_MAX){
/*write and destroty above stuff */
m_ret = mongoc_bulk_operation_execute (m_bulk, &m_reply, &m_error);
if (!m_ret){
logit ("et", "Error: %s\n", m_error.message);
}
bson_destroy (&m_reply);
mongoc_bulk_operation_destroy (m_bulk);
m_count = 0;
}
}
}
} /* end of while loop */
logit("et", "signing off");
exit (0);
return 0;
}
static void
test_bulk (void)
{
mongoc_bulk_operation_t *bulk;
mongoc_collection_t *collection;
mongoc_client_t *client;
bson_error_t error;
bson_iter_t iter;
bson_t reply;
bson_t child;
bson_t del;
bson_t up;
bson_t doc = BSON_INITIALIZER;
bool r;
client = mongoc_client_new (gTestUri);
assert (client);
collection = get_test_collection (client, "test_bulk");
assert (collection);
bulk = mongoc_collection_create_bulk_operation (collection, true, NULL);
assert (bulk);
mongoc_bulk_operation_insert (bulk, &doc);
mongoc_bulk_operation_insert (bulk, &doc);
mongoc_bulk_operation_insert (bulk, &doc);
mongoc_bulk_operation_insert (bulk, &doc);
bson_init (&up);
bson_append_document_begin (&up, "$set", -1, &child);
bson_append_int32 (&child, "hello", -1, 123);
bson_append_document_end (&up, &child);
mongoc_bulk_operation_update (bulk, &doc, &up, false);
bson_destroy (&up);
bson_init (&del);
BSON_APPEND_INT32 (&del, "hello", 123);
mongoc_bulk_operation_delete (bulk, &del);
bson_destroy (&del);
r = mongoc_bulk_operation_execute (bulk, &reply, &error);
assert (r);
assert (bson_iter_init_find (&iter, &reply, "nInserted"));
assert (BSON_ITER_HOLDS_INT32 (&iter));
assert (bson_iter_int32 (&iter) == 4);
/*
* This may be omitted if we talked to a (<= 2.4.x) node, or a mongos
* talked to a (<= 2.4.x) node.
*/
if (bson_iter_init_find (&iter, &reply, "nModified")) {
assert (BSON_ITER_HOLDS_INT32 (&iter));
assert (bson_iter_int32 (&iter) == 4);
}
assert (bson_iter_init_find (&iter, &reply, "nRemoved"));
assert (BSON_ITER_HOLDS_INT32 (&iter));
assert (4 == bson_iter_int32 (&iter));
assert (bson_iter_init_find (&iter, &reply, "nMatched"));
assert (BSON_ITER_HOLDS_INT32 (&iter));
assert (4 == bson_iter_int32 (&iter));
assert (bson_iter_init_find (&iter, &reply, "nUpserted"));
assert (BSON_ITER_HOLDS_INT32 (&iter));
assert (!bson_iter_int32 (&iter));
bson_destroy (&reply);
r = mongoc_collection_drop (collection, &error);
assert (r);
mongoc_bulk_operation_destroy (bulk);
mongoc_collection_destroy (collection);
mongoc_client_destroy (client);
bson_destroy (&doc);
}
int64_t
load_table (mongoc_database_t *db,
const char *table_name,
bson_t *bson_schema)
{
int64_t ret = true;
column_map_t *column_map, *column_map_p;
int column_map_size, i;
double start_time, end_time, delta_time;
FILE *fp;
mongoc_collection_t *collection;
mongoc_bulk_operation_t *bulk;
size_t n_docs = 0;
char *token;
bson_t bson, reply;
int64_t count = 0;
bson_error_t error;
fprintf (stderr, "load_table table_name: \"%s\"\n", table_name);
get_column_map (bson_schema, table_name, &column_map, &column_map_size) || DIE;
snprintf (mbdump_file, MAXPATHLEN, "%s/%s", mbdump_dir, table_name);
/* fprintf (stderr, "mbdump_file: \"%s\"\n", mbdump_file); */
start_time = dtimeofday ();
fp = fopen (mbdump_file, "r");
if (!fp) DIE;
collection = mongoc_database_get_collection (db, table_name);
bulk = mongoc_collection_create_bulk_operation (collection, true, NULL);
bson_init (&bson);
while (ret && fgets (buf, BUFSIZ, fp)) {
/*
fputs (buf, stdout);
*/
chomp (buf);
for (i = 0, column_map_p = column_map, token = strtok_single (buf, "\t");
i < column_map_size;
i++, column_map_p++, token = strtok_single (NULL, "\t")) {
bool ret;
/*
fprintf (stderr, "%s: \"%s\" [%d/%d](%s)\n", column_map_p->column_name, token, i, column_map_size, column_map_p->data_type);
fflush (stdout);
*/
ret = (*column_map_p->bson_append_from_s) (&bson, column_map_p->column_name, token);
ret || fprintf (stderr, "WARNING: column_map_p->bson_append_from_s failed column %s: \"%s\" [%d/%d](%s)\n",
column_map_p->column_name, token, i, column_map_size, column_map_p->data_type);
}
/*
bson_printf ("bson: %s\n", &bson);
*/
mongoc_bulk_operation_insert (bulk, &bson);
bson_reinit (&bson);
if (++n_docs == BULK_OPS_SIZE) {
ret = mongoc_bulk_operation_execute (bulk, &reply, &error);
if (ret) {
count += n_docs;
if (count % PROGRESS_SIZE == 0) {
fputc('.', stdout);
fflush(stdout);
}
}
else
fprintf (stderr, "mongoc_cursor_bulk_insert execute failure: %s\n", error.message);
n_docs = 0;
mongoc_bulk_operation_destroy (bulk);
bulk = mongoc_collection_create_bulk_operation (collection, true, NULL);
}
}
if (ret && n_docs > 0) {
ret = mongoc_bulk_operation_execute (bulk, &reply, &error);
if (ret)
count += n_docs;
else
fprintf (stderr, "mongoc_cursor_bulk_insert execute failure: %s\n", error.message);
}
fputc('.', stdout);
fputc('\n', stdout);
fflush(stdout);
bson_destroy (&bson);
mongoc_bulk_operation_destroy (bulk);
mongoc_collection_destroy (collection);
fclose (fp);
end_time = dtimeofday ();
delta_time = end_time - start_time + 0.0000001;
fprintf (stderr, "info: real: %.2f, count: %"PRId64", %"PRId64" docs/sec\n", delta_time, count, (int64_t)round (count/delta_time));
fflush (stderr);
free (column_map);
return ret ? count : -1;
}
