static int mongodb_module_flush(void) { int ret; uint32_t bulk_ret; bson_error_t error; bson_t reply; mongoc_bulk_operation_t *old_bulk; if (buffer_fill == 0){ mongoc_bulk_operation_destroy(bulk); return EXIT_SUCCESS; } bulk_ret = mongoc_bulk_operation_execute(bulk, &reply, &error); old_bulk = bulk; if (bulk_ret == 0) { mongoc_log(MONGOC_LOG_LEVEL_ERROR, "zmap", "Error executing bulk insert: %s", error.message); ret = EXIT_FAILURE; } else { bulk = mongoc_collection_create_bulk_operation(collection,false,NULL); ret = EXIT_SUCCESS; } bson_destroy(&reply); mongoc_bulk_operation_destroy(old_bulk); return ret; }
static void bulk1 (mongoc_collection_t *collection) { mongoc_bulk_operation_t *bulk; bson_error_t error; bson_t *doc; bson_t reply; char *str; bool ret; int i; bulk = mongoc_collection_create_bulk_operation (collection, true, NULL); for (i = 0; i < 10000; i++) { doc = BCON_NEW ("i", BCON_INT32 (i)); mongoc_bulk_operation_insert (bulk, doc); bson_destroy (doc); } ret = mongoc_bulk_operation_execute (bulk, &reply, &error); str = bson_as_json (&reply, NULL); printf ("%s\n", str); bson_free (str); if (!ret) { fprintf (stderr, "Error: %s\n", error.message); } bson_destroy (&reply); mongoc_bulk_operation_destroy (bulk); }
int mongodb_module_init(struct state_conf *conf, UNUSED char **fields, UNUSED int fieldlens) { char *uri_str = NULL; buffer_fill = 0; const char *db; if (conf->output_args) { uri_str = conf->output_args; } mongoc_init(); mongoc_log_set_handler(mongodb_module_log, NULL); mongoc_uri_t *uri = mongoc_uri_new(uri_str); if (uri == NULL) { log_fatal("mongodb-module", "URI %s not valid!", uri_str); } client = mongoc_client_new_from_uri(uri); db = mongoc_uri_get_database(uri); collection = mongoc_client_get_collection(client, db ? db : strdup("zmap_output"), conf->output_filename ? conf->output_filename : strdup("zmap_output")); bulk = mongoc_collection_create_bulk_operation(collection,false,NULL); return EXIT_SUCCESS; }
int lua_mongo_collection_insert_one (lua_State *L) { collection_t *collection; mongoc_bulk_operation_t *bulk_insert = NULL; bson_t bson_doc = BSON_INITIALIZER; bson_error_t error; bson_t reply = BSON_INITIALIZER; bool ret; bool throw_error = false; collection = (collection_t *)luaL_checkudata(L, 1, "lua_mongoc_collection"); if (!(lua_istable(L, 2))) { luaL_error(L, "second input must be a table"); } else { if (!(lua_table_to_bson(L, &bson_doc, 2, true, &error))) { throw_error = true; goto DONE; } } bulk_insert = mongoc_collection_create_bulk_operation(collection->c_collection, true, NULL); mongoc_bulk_operation_insert(bulk_insert, &bson_doc); ret = mongoc_bulk_operation_execute (bulk_insert, &reply, &error); if (!ret) { throw_error = true; goto DONE; } if (!(generate_InsertOneResult(L, ret, 2, &error))) { throw_error = true; goto DONE; } DONE: bson_destroy(&bson_doc); bson_destroy(&reply); if (bulk_insert) { mongoc_bulk_operation_destroy(bulk_insert); } if (throw_error) { luaL_error(L, error.message); } return 1; }
int lua_mongo_collection_delete_many(lua_State *L) { collection_t *collection; bson_t selector = BSON_INITIALIZER; mongoc_bulk_operation_t *bulk_remove = NULL; bson_error_t error; bson_t reply; bool throw_error = false; bool ret; collection = (collection_t *)luaL_checkudata(L, 1, "lua_mongoc_collection"); if ((lua_istable(L, 2))) { if (!(lua_table_to_bson(L, &selector, 2, false, &error))) { throw_error = true; goto DONE; } } else { luaL_error(L, "second input must be a table"); } bulk_remove = mongoc_collection_create_bulk_operation(collection->c_collection, true, NULL); mongoc_bulk_operation_remove(bulk_remove, &selector); ret = mongoc_bulk_operation_execute(bulk_remove, &reply, &error); if (!(ret)) { throw_error = true; goto DONE; } if (!(generate_DeleteResult(L, &reply, ret, &error))) { throw_error = true; goto DONE; } DONE: bson_destroy(&selector); if (bulk_remove) { mongoc_bulk_operation_destroy (bulk_remove); } if (throw_error) { luaL_error(L, error.message); } return 1; }
static void bulk3 (mongoc_collection_t *collection) { mongoc_bulk_operation_t *bulk; bson_error_t error; bson_t *query; bson_t *doc; bson_t reply; char *str; bool ret; /* false indicates unordered */ bulk = mongoc_collection_create_bulk_operation (collection, false, NULL); /* Add a document */ doc = BCON_NEW ("_id", BCON_INT32 (1)); mongoc_bulk_operation_insert (bulk, doc); bson_destroy (doc); /* remove {_id: 2} */ query = BCON_NEW ("_id", BCON_INT32 (2)); mongoc_bulk_operation_remove_one (bulk, query); bson_destroy (query); /* insert {_id: 3} */ doc = BCON_NEW ("_id", BCON_INT32 (3)); mongoc_bulk_operation_insert (bulk, doc); bson_destroy (doc); /* replace {_id:4} {'i': 1} */ query = BCON_NEW ("_id", BCON_INT32 (4)); doc = BCON_NEW ("i", BCON_INT32 (1)); mongoc_bulk_operation_replace_one (bulk, query, doc, false); bson_destroy (query); bson_destroy (doc); ret = mongoc_bulk_operation_execute (bulk, &reply, &error); str = bson_as_json (&reply, NULL); printf ("%s\n", str); bson_free (str); if (!ret) { printf ("Error: %s\n", error.message); } bson_destroy (&reply); mongoc_bulk_operation_destroy (bulk); }
SEXP R_mongo_collection_insert_page(SEXP ptr_col, SEXP json_vec, SEXP stop_on_error){ if(!isString(json_vec) || !length(json_vec)) stop("json_vec must be character string of at least length 1"); //ordered means serial execution bool ordered = asLogical(stop_on_error); //create bulk operation bson_error_t err; bson_t *b; bson_t reply; mongoc_bulk_operation_t *bulk = mongoc_collection_create_bulk_operation (r2col(ptr_col), ordered, NULL); for(int i = 0; i < length(json_vec); i++){ b = bson_new_from_json ((uint8_t*)translateCharUTF8(asChar(STRING_ELT(json_vec, i))), -1, &err); if(!b){ mongoc_bulk_operation_destroy (bulk); stop(err.message); } mongoc_bulk_operation_insert(bulk, b); bson_destroy (b); b = NULL; } //execute bulk operation bool success = mongoc_bulk_operation_execute (bulk, &reply, &err); mongoc_bulk_operation_destroy (bulk); //check for errors if(!success){ if(ordered){ Rf_errorcall(R_NilValue, err.message); } else { Rf_warningcall(R_NilValue, "Not all inserts were successful: %s\n", err.message); } } //get output SEXP out = PROTECT(bson2list(&reply)); bson_destroy (&reply); UNPROTECT(1); return out; }
static void bulk4 (mongoc_collection_t *collection) { mongoc_write_concern_t *wc; mongoc_bulk_operation_t *bulk; bson_error_t error; bson_t *doc; bson_t reply; char *str; bool ret; wc = mongoc_write_concern_new (); mongoc_write_concern_set_w (wc, 4); mongoc_write_concern_set_wtimeout (wc, 100); /* milliseconds */ bulk = mongoc_collection_create_bulk_operation (collection, true, wc); /* Two inserts */ doc = BCON_NEW ("_id", BCON_INT32 (10)); mongoc_bulk_operation_insert (bulk, doc); bson_destroy (doc); doc = BCON_NEW ("_id", BCON_INT32 (11)); mongoc_bulk_operation_insert (bulk, doc); bson_destroy (doc); ret = mongoc_bulk_operation_execute (bulk, &reply, &error); str = bson_as_json (&reply, NULL); printf ("%s\n", str); bson_free (str); if (!ret) { printf ("Error: %s\n", error.message); } bson_destroy (&reply); mongoc_bulk_operation_destroy (bulk); mongoc_write_concern_destroy (wc); }
static void bulk5_success (mongoc_collection_t *collection) { mongoc_bulk_operation_t *bulk; bson_error_t error; bson_t *doc; bson_t reply; char *str; bool ret; bulk = mongoc_collection_create_bulk_operation (collection, true, NULL); /* Allow this document to bypass document validation. * NOTE: When authentication is enabled, the authenticated user must have * either the "dbadmin" or "restore" roles to bypass document validation */ mongoc_bulk_operation_set_bypass_document_validation (bulk, true); /* Two inserts */ doc = BCON_NEW ("_id", BCON_INT32 (31)); mongoc_bulk_operation_insert (bulk, doc); bson_destroy (doc); doc = BCON_NEW ("_id", BCON_INT32 (32)); mongoc_bulk_operation_insert (bulk, doc); bson_destroy (doc); ret = mongoc_bulk_operation_execute (bulk, &reply, &error); str = bson_as_json (&reply, NULL); printf ("%s\n", str); bson_free (str); if (!ret) { printf ("Error: %s\n", error.message); } bson_destroy (&reply); mongoc_bulk_operation_destroy (bulk); }
static void bulk5_fail (mongoc_collection_t *collection) { mongoc_bulk_operation_t *bulk; bson_error_t error; bson_t *doc; bson_t reply; char *str; bool ret; bulk = mongoc_collection_create_bulk_operation (collection, true, NULL); /* Two inserts */ doc = BCON_NEW ("_id", BCON_INT32 (31)); mongoc_bulk_operation_insert (bulk, doc); bson_destroy (doc); doc = BCON_NEW ("_id", BCON_INT32 (32)); mongoc_bulk_operation_insert (bulk, doc); bson_destroy (doc); /* The above documents do not comply to the schema validation rules * we created previously, so this will result in an error */ ret = mongoc_bulk_operation_execute (bulk, &reply, &error); str = bson_as_json (&reply, NULL); printf ("%s\n", str); bson_free (str); if (!ret) { printf ("Error: %s\n", error.message); } bson_destroy (&reply); mongoc_bulk_operation_destroy (bulk); }
static void test_bulk (void) { mongoc_bulk_operation_t *bulk; mongoc_collection_t *collection; mongoc_client_t *client; bson_error_t error; bson_iter_t iter; bson_t reply; bson_t child; bson_t del; bson_t up; bson_t doc = BSON_INITIALIZER; bool r; client = mongoc_client_new (gTestUri); assert (client); collection = get_test_collection (client, "test_bulk"); assert (collection); bulk = mongoc_collection_create_bulk_operation (collection, true, NULL); assert (bulk); mongoc_bulk_operation_insert (bulk, &doc); mongoc_bulk_operation_insert (bulk, &doc); mongoc_bulk_operation_insert (bulk, &doc); mongoc_bulk_operation_insert (bulk, &doc); bson_init (&up); bson_append_document_begin (&up, "$set", -1, &child); bson_append_int32 (&child, "hello", -1, 123); bson_append_document_end (&up, &child); mongoc_bulk_operation_update (bulk, &doc, &up, false); bson_destroy (&up); bson_init (&del); BSON_APPEND_INT32 (&del, "hello", 123); mongoc_bulk_operation_delete (bulk, &del); bson_destroy (&del); r = mongoc_bulk_operation_execute (bulk, &reply, &error); assert (r); assert (bson_iter_init_find (&iter, &reply, "nInserted")); assert (BSON_ITER_HOLDS_INT32 (&iter)); assert (bson_iter_int32 (&iter) == 4); /* * This may be omitted if we talked to a (<= 2.4.x) node, or a mongos * talked to a (<= 2.4.x) node. */ if (bson_iter_init_find (&iter, &reply, "nModified")) { assert (BSON_ITER_HOLDS_INT32 (&iter)); assert (bson_iter_int32 (&iter) == 4); } assert (bson_iter_init_find (&iter, &reply, "nRemoved")); assert (BSON_ITER_HOLDS_INT32 (&iter)); assert (4 == bson_iter_int32 (&iter)); assert (bson_iter_init_find (&iter, &reply, "nMatched")); assert (BSON_ITER_HOLDS_INT32 (&iter)); assert (4 == bson_iter_int32 (&iter)); assert (bson_iter_init_find (&iter, &reply, "nUpserted")); assert (BSON_ITER_HOLDS_INT32 (&iter)); assert (!bson_iter_int32 (&iter)); bson_destroy (&reply); r = mongoc_collection_drop (collection, &error); assert (r); mongoc_bulk_operation_destroy (bulk); mongoc_collection_destroy (collection); mongoc_client_destroy (client); bson_destroy (&doc); }
int lua_mongo_collection_insert_many (lua_State *L) { collection_t *collection; mongoc_bulk_operation_t *bulk_insert = NULL; bool ordered; bson_error_t error; bson_t reply = BSON_INITIALIZER; bool ret; bool throw_error = false; collection = (collection_t *)luaL_checkudata(L, 1, "lua_mongoc_collection"); if (!(lua_istable(L, 2))) { luaL_error(L, "second input must be a table"); } if (!(lua_isboolean(L, 3))) { luaL_error(L, "ordered parameter must be a boolean"); } else { ordered = lua_toboolean(L, 3); } bulk_insert = mongoc_collection_create_bulk_operation(collection->c_collection, ordered, NULL); lua_pushnil(L); int num_elements, lua_index; for (num_elements = 0, lua_index = 1; lua_next(L, 2) != 0; num_elements++, lua_index++) { bson_t bson_doc = BSON_INITIALIZER; if (!(lua_isnumber(L, -2)) || lua_tonumber(L, -2) != lua_index) { throw_error = true; strncpy(error.message, "malformed array of documents", sizeof(error.message)); goto DONE; } if (!(lua_table_to_bson(L, &bson_doc, -1, true, &error))) { throw_error = true; goto DONE; } mongoc_bulk_operation_insert(bulk_insert, &bson_doc); lua_pop(L, 1); bson_destroy(&bson_doc); } if (!(mongoc_bulk_operation_execute (bulk_insert, &reply, &error))) { throw_error = true; goto DONE; } if (!(generate_InsertManyResult(L, &reply, 2, num_elements, &error))) { throw_error = true; goto DONE; } DONE: bson_destroy(&reply); if (bulk_insert){ mongoc_bulk_operation_destroy (bulk_insert); } if (throw_error) { luaL_error(L, error.message); } return 1; }
int main( int argc, char **argv ) { SHM_INFO region; long RingKey; /* Key to the transport ring to read from */ MSG_LOGO getlogo[NLOGO], logo; long gotsize; char msg[MAX_TRACEBUF_SIZ]; char *getSta, *getComp, *getNet, *getLoc, *inRing; char wildSta, wildComp, wildNet, wildLoc; unsigned char Type_Mseed; unsigned char Type_TraceBuf,Type_TraceBuf2; unsigned char Type_TraceComp, Type_TraceComp2; unsigned char InstWildcard, ModWildcard; short *short_data; int *long_data; TRACE2_HEADER *trh; char orig_datatype[3]; char stime[256]; char etime[256]; int i; int rc; int nLogo = NLOGO; static unsigned char InstId; /* local installation id */ char ModName[MAX_MOD_STR]; char *modid_name; unsigned char sequence_number = 0; // int statistics_flag; time_t monitor_start_time = 0; double start_time, end_time; unsigned long packet_total=0; unsigned long packet_total_size=0; MSRecord *msr = NULL; /* mseed record */ logit_init("ring2mongo", 200, 256, 1); char *mongo_user = getenv("MONGO_USER"); char *mongo_passwd=getenv("MONGO_PASSWD"); char *mongo_host=getenv("MONGO_HOST"); char *mongo_port=getenv("MONGO_PORT"); if(mongo_user == NULL || mongo_passwd==NULL || mongo_host== NULL || mongo_port==NULL){ printf("Ensure the following ENV varibles are set\n *MONGO_USER\n *MONGO_PASSWD\n *MONGO_HOST\n *MONGO_PORT\n"); exit(1); return(1); } char mongo_str[128]="mongodb://"; strcat(mongo_str, mongo_user); strcat(mongo_str, ":"); strcat(mongo_str, mongo_passwd); strcat(mongo_str, "@"); strcat(mongo_str, mongo_host); strcat(mongo_str, ":"); strcat(mongo_str, mongo_port); strcat(mongo_str, "/"); //mongo stuff mongoc_client_t *m_client; mongoc_collection_t *m_collection; mongoc_init (); m_client = mongoc_client_new (mongo_str); m_collection = mongoc_client_get_collection (m_client, "waveforms", "ring"); mongoc_bulk_operation_t *m_bulk; bson_error_t m_error; bson_t *m_doc; bson_t m_reply; bson_t *m_data; /*wait for this many messages before performing bulk write For real time continuous data it shoudl be ~ number of channels on ring*/ int MONGO_BULK_MAX = 30; int m_count = 0; /* counter for mongo bulk */ bool m_ret; /*end mongo stuff*/ /* Initialize pointers **********************/ trh = (TRACE2_HEADER *) msg; long_data = (int *)( msg + sizeof(TRACE2_HEADER) ); short_data = (short *)( msg + sizeof(TRACE2_HEADER) ); /* Check command line argument *****************************/ if ( argc < 2 || argc > 6) { if(argc > 6) fprintf(stderr, "ring2mongo: Too many parameters\n"); fprintf(stderr,"Usage: %s <ring name> [sta] [comp] [net] [loc] \n", argv[0]); fprintf(stderr, " Note: All parameters are positional, and all but first are optional.\n"); fprintf(stderr, " the full data contained in the packet is printed out.\n"); fprintf(stderr, " If sta comp net (but not loc) are specified then only TraceBuf\n"); fprintf(stderr, " packets will be fetched (not TraceBuf2); otherwise both are fetched.\n"); fprintf(stderr, " Example: %s WAVE_RING PHOB wild NC wild n\n", argv[0]); fprintf(stderr, " MSEED capability starting with version 2.5.1, prints mseed headers\n"); fprintf(stderr, " of TYPE_MSEED packets (no filtering yet).\n"); exit( 1 ); return 1; } /* process given parameters */ inRing = argv[1]; /* first parameter is ring name */ /* any parameters not given are set as wildcards */ getSta = getComp = getNet = getLoc = ""; wildSta = wildComp = wildNet = wildLoc = 1; if(argc > 2) { /* at least station parameter given */ getSta = argv[2]; wildSta = IsWild(getSta); if(argc > 3) { /* channel (component) parameter given */ getComp = argv[3]; wildComp = IsWild(getComp); if(argc > 4) { /* network parameter given */ getNet = argv[4]; wildNet = IsWild(getNet); if(argc > 5) { /* location parameter given (SCNL) */ getLoc = argv[5]; wildLoc = IsWild(getLoc); } else { /* SCN without location parameter given */ nLogo = 3; /* do not include tracebuf2s in search */ } } } } /* Attach to ring *****************/ if ((RingKey = GetKey( inRing )) == -1 ) { fprintf( stderr, "Invalid RingName; exiting!\n" ); exit( -1 ); return -1; } tport_attach( ®ion, RingKey ); /* Look up local installation id *****************************/ if ( GetLocalInst( &InstId ) != 0 ) { fprintf(stderr, "ring2mongo: error getting local installation id; exiting!\n" ); exit( -1 ); return -1; } /* Specify logos to get ***********************/ if ( GetType( "TYPE_MSEED", &Type_Mseed ) != 0 ) { fprintf(stderr, "%s: WARNING: Invalid message type <TYPE_MSEED>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); Type_Mseed = TYPE_NOTUSED; } if ( GetType( "TYPE_TRACEBUF", &Type_TraceBuf ) != 0 ) { fprintf(stderr, "%s: Invalid message type <TYPE_TRACEBUF>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); exit( -1 ); return -1; } if ( GetType( "TYPE_TRACE_COMP_UA", &Type_TraceComp ) != 0 ) { fprintf(stderr, "%s: Invalid message type <TYPE_TRACE_COMP_UA>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); exit( -1 ); return -1; } if ( GetType( "TYPE_TRACEBUF2", &Type_TraceBuf2 ) != 0 ) { fprintf(stderr, "%s: Invalid message type <TYPE_TRACEBUF2>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); exit( -1 ); return -1; } if ( GetType( "TYPE_TRACE2_COMP_UA", &Type_TraceComp2 ) != 0 ) { fprintf(stderr,"%s: Invalid message type <TYPE_TRACE2_COMP_UA>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); exit( -1 ); return -1; } if ( GetModId( "MOD_WILDCARD", &ModWildcard ) != 0 ) { fprintf(stderr, "%s: Invalid moduleid <MOD_WILDCARD>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); exit( -1 ); return -1; } if ( GetInst( "INST_WILDCARD", &InstWildcard ) != 0 ) { fprintf(stderr, "%s: Invalid instid <INST_WILDCARD>! Missing from earthworm.d or earthworm_global.d\n", argv[0] ); exit( -1 ); return -1; } for( i=0; i<nLogo; i++ ) { getlogo[i].instid = InstWildcard; getlogo[i].mod = ModWildcard; } getlogo[0].type = Type_Mseed; getlogo[1].type = Type_TraceBuf; getlogo[2].type = Type_TraceComp; if (nLogo >= 4) { /* if nLogo=5 then include TraceBuf2s */ getlogo[3].type = Type_TraceBuf2; getlogo[4].type = Type_TraceComp2; } logit("", "Starting ring2mongo"); /* Flush the ring *****************/ while ( tport_copyfrom( ®ion, getlogo, nLogo, &logo, &gotsize, (char *)&msg, MAX_TRACEBUF_SIZ, &sequence_number ) != GET_NONE ){ packet_total++; packet_total_size+=gotsize; } logit( "et", "ring2mongo: inRing flushed %ld packets of %ld bytes total.\n", packet_total, packet_total_size); while (tport_getflag( ®ion ) != TERMINATE ) { rc = tport_copyfrom( ®ion, getlogo, nLogo, &logo, &gotsize, msg, MAX_TRACEBUF_SIZ, &sequence_number ); if ( rc == GET_NONE ){ sleep_ew( 200 ); continue; } if ( rc == GET_TOOBIG ){ logit("et", "ring2mongo: retrieved message too big (%ld) for msg\n", gotsize ); continue; } if ( rc == GET_NOTRACK ) logit("et", "ring2mongo: Tracking error.\n"); if ( rc == GET_MISS_LAPPED ) logit("et", "ring2mongo: Got lapped on the ring.\n"); if ( rc == GET_MISS_SEQGAP ) logit( "et", "ring2mongo: Gap in sequence numbers\n"); if ( rc == GET_MISS ) logit( "et", "ring2mongo: Missed messages\n"); /* Check SCNL of the retrieved message */ if (Type_Mseed != TYPE_NOTUSED && logo.type == Type_Mseed) { /* Unpack record header and not data samples */ /*hard coded zero for dataflag(1) and verbose(0)for ring2mongo*/ if ( msr_unpack (msg, gotsize, &msr, 1, 0) != MS_NOERROR) { logit("et", "Error parsing mseed record\n"); continue; } /* Print record information */ msr_print (msr, 0); msr_free (&msr); continue; } /*end Mseed*/ if ( (wildSta || (strcmp(getSta,trh->sta) ==0)) && (wildComp || (strcmp(getComp,trh->chan)==0)) && (wildNet || (strcmp(getNet,trh->net) ==0)) && (((logo.type == Type_TraceBuf2 || logo.type == Type_TraceComp2) && (wildLoc || (strcmp(getLoc,trh->loc) == 0))) || ( (logo.type == Type_TraceBuf || logo.type == Type_TraceComp)))) { strcpy(orig_datatype, trh->datatype); char scnl[20]; scnl[0] = 0; strcat( scnl, trh->sta); strcat( scnl, "."); strcat( scnl, trh->chan); strcat( scnl, "."); strcat( scnl, trh->net); strcat( scnl, "."); strcat( scnl, trh->loc); if(WaveMsg2MakeLocal( trh ) < 0){ char dt[3]; /* now put a space if there are any punctuation marks */ for ( i=0; i<15; i++ ) { if ( !isalnum(scnl[i]) && !ispunct(scnl[i])) scnl[i] = ' '; } strncpy( dt, trh->datatype, 2 ); for ( i=0; i<2; i++ ) { if ( !isalnum(dt[i]) && !ispunct(dt[i])) dt[i] = ' '; } dt[i] = 0; logit("et", "WARNING: WaveMsg2MakeLocal rejected tracebuf. Discard (%s).\n", scnl ); logit("et", "\tdatatype=[%s]\n", dt); continue; } /*lets get ready to mongo....*/ if(m_count < MONGO_BULK_MAX){ /*initialize when m_count ==0 */ if(m_count==0){ m_bulk = mongoc_collection_create_bulk_operation (m_collection, true, NULL); } m_data = bson_new (); m_doc = BCON_NEW ("key", BCON_UTF8 (scnl), "nsamp", BCON_INT32 (trh->nsamp), "starttime", BCON_DOUBLE (trh->starttime*1000), "endtime", BCON_DOUBLE (trh->endtime*1000), "samprate", BCON_DOUBLE (trh->samprate), "datatype", BCON_UTF8 (trh->datatype) ); char index[4]; bson_append_array_begin (m_doc, "data", -1, m_data); for ( i = 0; i < trh->nsamp; i++ ){ snprintf(index, 4, "%d", i); if ( (strcmp (trh->datatype, "s2")==0) || (strcmp (trh->datatype, "i2")==0) ){ bson_append_int32 (m_data, index, -1, short_data[i]); }else{ bson_append_int32 (m_data, index, -1, long_data[i]); } } bson_append_array_end (m_doc, m_data); mongoc_bulk_operation_insert (m_bulk, m_doc); bson_destroy (m_doc); bson_destroy (m_data); m_count++; if(m_count==MONGO_BULK_MAX){ /*write and destroty above stuff */ m_ret = mongoc_bulk_operation_execute (m_bulk, &m_reply, &m_error); if (!m_ret){ logit ("et", "Error: %s\n", m_error.message); } bson_destroy (&m_reply); mongoc_bulk_operation_destroy (m_bulk); m_count = 0; } } } } /* end of while loop */ logit("et", "signing off"); exit (0); return 0; }
int lua_mongo_collection_update (lua_State *L) { collection_t *collection; bson_t filter = BSON_INITIALIZER; bson_t update = BSON_INITIALIZER; bool upsert; bool update_many; bson_error_t error; bool ret; mongoc_bulk_operation_t *bulk_update = NULL; bson_t reply = BSON_INITIALIZER; bool throw_error = false; collection = (collection_t *)luaL_checkudata(L, 1, "lua_mongoc_collection"); if (!(lua_isnil(L, 2))) { if (!(lua_table_to_bson(L, &filter, 2, false, &error))) { throw_error = true; goto DONE; } } if (lua_isnil(L, 3)) { luaL_error(L, "update parameters must be included"); } else { if (!(lua_table_to_bson(L, &update, 3, false, &error))) { throw_error = true; goto DONE; } } if ((lua_isboolean(L, 4))) { upsert = lua_toboolean(L, 4); } else { luaL_error(L, "upsert parameter must be a boolean"); } if ((lua_isboolean(L, 5))) { update_many = lua_toboolean(L, 5); } else { luaL_error(L, "update_many parameter must be a boolean"); } bulk_update = mongoc_collection_create_bulk_operation (collection->c_collection, false, NULL); if (update_many) { mongoc_bulk_operation_update(bulk_update, &filter, &update, upsert); } else { mongoc_bulk_operation_update_one(bulk_update, &filter, &update, upsert); } ret = mongoc_bulk_operation_execute (bulk_update, &reply, &error); if (!ret) { throw_error = true; goto DONE; } if (!(bson_document_or_array_to_table(L, &reply, true, &error))) { throw_error = true; goto DONE; } DONE: bson_destroy(&filter); bson_destroy(&update); bson_destroy(&reply); if (bulk_update) { mongoc_bulk_operation_destroy (bulk_update); } if (throw_error) { luaL_error(L, error.message); } return 1; }
bool mongoc_gridfs_remove_by_filename (mongoc_gridfs_t *gridfs, const char *filename, bson_error_t *error) { mongoc_bulk_operation_t *bulk_files = NULL; mongoc_bulk_operation_t *bulk_chunks = NULL; mongoc_cursor_t *cursor = NULL; bson_error_t files_error; bson_error_t chunks_error; const bson_t *doc; const char *key; char keybuf[16]; int count = 0; bool chunks_ret; bool files_ret; bool ret = false; bson_iter_t iter; bson_t *files_q = NULL; bson_t *chunks_q = NULL; bson_t q = BSON_INITIALIZER; bson_t fields = BSON_INITIALIZER; bson_t ar = BSON_INITIALIZER; BSON_ASSERT (gridfs); if (!filename) { bson_set_error (error, MONGOC_ERROR_GRIDFS, MONGOC_ERROR_GRIDFS_INVALID_FILENAME, "A non-NULL filename must be specified."); return false; } /* * Find all files matching this filename. Hopefully just one, but not * strictly required! */ BSON_APPEND_UTF8 (&q, "filename", filename); BSON_APPEND_INT32 (&fields, "_id", 1); cursor = _mongoc_cursor_new (gridfs->client, gridfs->files->ns, MONGOC_QUERY_NONE, 0, 0, 0, false /* is command */, &q, &fields, NULL, NULL); BSON_ASSERT (cursor); while (mongoc_cursor_next (cursor, &doc)) { if (bson_iter_init_find (&iter, doc, "_id")) { const bson_value_t *value = bson_iter_value (&iter); bson_uint32_to_string (count, &key, keybuf, sizeof keybuf); BSON_APPEND_VALUE (&ar, key, value); } } if (mongoc_cursor_error (cursor, error)) { goto failure; } bulk_files = mongoc_collection_create_bulk_operation (gridfs->files, false, NULL); bulk_chunks = mongoc_collection_create_bulk_operation (gridfs->chunks, false, NULL); files_q = BCON_NEW ("_id", "{", "$in", BCON_ARRAY (&ar), "}"); chunks_q = BCON_NEW ("files_id", "{", "$in", BCON_ARRAY (&ar), "}"); mongoc_bulk_operation_remove (bulk_files, files_q); mongoc_bulk_operation_remove (bulk_chunks, chunks_q); files_ret = mongoc_bulk_operation_execute (bulk_files, NULL, &files_error); chunks_ret = mongoc_bulk_operation_execute (bulk_chunks, NULL, &chunks_error); if (error) { if (!files_ret) { memcpy (error, &files_error, sizeof *error); } else if (!chunks_ret) { memcpy (error, &chunks_error, sizeof *error); } } ret = (files_ret && chunks_ret); failure: if (cursor) { mongoc_cursor_destroy (cursor); } if (bulk_files) { mongoc_bulk_operation_destroy (bulk_files); } if (bulk_chunks) { mongoc_bulk_operation_destroy (bulk_chunks); } bson_destroy (&q); bson_destroy (&fields); bson_destroy (&ar); if (files_q) { bson_destroy (files_q); } if (chunks_q) { bson_destroy (chunks_q); } return ret; }
static void test_bypass_validation (void *context) { mongoc_collection_t *collection; bson_t reply = BSON_INITIALIZER; mongoc_bulk_operation_t *bulk; mongoc_database_t *database; mongoc_write_concern_t *wr; mongoc_client_t *client; bson_error_t error; bson_t *options; char *collname; char *dbname; int r; int i; client = test_framework_client_new (); assert (client); dbname = gen_collection_name ("dbtest"); collname = gen_collection_name ("bypass"); database = mongoc_client_get_database (client, dbname); collection = mongoc_database_get_collection (database, collname); assert (collection); options = tmp_bson ("{'validator': {'number': {'$gte': 5}}, 'validationAction': 'error'}"); ASSERT_OR_PRINT (mongoc_database_create_collection (database, collname, options, &error), error); /* {{{ Default fails validation */ bulk = mongoc_collection_create_bulk_operation(collection, true, NULL); for (i = 0; i < 3; i++) { bson_t *doc = tmp_bson (bson_strdup_printf ("{'number': 3, 'high': %d }", i)); mongoc_bulk_operation_insert (bulk, doc); } r = mongoc_bulk_operation_execute (bulk, &reply, &error); ASSERT(!r); ASSERT_ERROR_CONTAINS (error, MONGOC_ERROR_COMMAND, 121, "Document failed validation"); mongoc_bulk_operation_destroy (bulk); /* }}} */ /* {{{ bypass_document_validation=false Fails validation */ bulk = mongoc_collection_create_bulk_operation(collection, true, NULL); mongoc_bulk_operation_set_bypass_document_validation (bulk, false); for (i = 0; i < 3; i++) { bson_t *doc = tmp_bson (bson_strdup_printf ("{'number': 3, 'high': %d }", i)); mongoc_bulk_operation_insert (bulk, doc); } r = mongoc_bulk_operation_execute (bulk, &reply, &error); ASSERT(!r); ASSERT_ERROR_CONTAINS (error, MONGOC_ERROR_COMMAND, 121, "Document failed validation"); mongoc_bulk_operation_destroy (bulk); /* }}} */ /* {{{ bypass_document_validation=true ignores validation */ bulk = mongoc_collection_create_bulk_operation(collection, true, NULL); mongoc_bulk_operation_set_bypass_document_validation (bulk, true); for (i = 0; i < 3; i++) { bson_t *doc = tmp_bson (bson_strdup_printf ("{'number': 3, 'high': %d }", i)); mongoc_bulk_operation_insert (bulk, doc); } r = mongoc_bulk_operation_execute (bulk, &reply, &error); ASSERT_OR_PRINT(r, error); mongoc_bulk_operation_destroy (bulk); /* }}} */ /* {{{ w=0 and bypass_document_validation=set fails */ bulk = mongoc_collection_create_bulk_operation(collection, true, NULL); wr = mongoc_write_concern_new (); mongoc_write_concern_set_w (wr, 0); mongoc_bulk_operation_set_write_concern (bulk, wr); mongoc_bulk_operation_set_bypass_document_validation (bulk, true); for (i = 0; i < 3; i++) { bson_t *doc = tmp_bson (bson_strdup_printf ("{'number': 3, 'high': %d }", i)); mongoc_bulk_operation_insert (bulk, doc); } r = mongoc_bulk_operation_execute (bulk, &reply, &error); ASSERT_OR_PRINT(!r, error); ASSERT_ERROR_CONTAINS (error, MONGOC_ERROR_COMMAND, MONGOC_ERROR_COMMAND_INVALID_ARG, "Cannot set bypassDocumentValidation for unacknowledged writes"); mongoc_bulk_operation_destroy (bulk); mongoc_write_concern_destroy (wr); /* }}} */ ASSERT_OR_PRINT (mongoc_collection_drop (collection, &error), error); mongoc_collection_destroy (collection); mongoc_client_destroy (client); }
void run_loader(int thread, char *filename) { bson_t *record; mongoc_client_t *conn; mongoc_collection_t *collection; bson_error_t error; bson_t reply; int count; FILE *infile; char *rptr; char ilinebuf[BUFSIZ]; char rlinebuf[BUFSIZ]; char *ritem; char *rlast = NULL; int rfcount = 0; int batchcount = 0; char *str; int total = 0; //Get the highest used value on that shard so far conn = mongoc_client_new(DEFAULT_URI); if (!conn) { fprintf(stderr, "Failed to parse URI.\n"); exit(1); } collection = mongoc_client_get_collection(conn, DATA_DB, DATA_COLLECTION); long long chunkno = carve_chunk(conn, collection); printf("Thread %d reading %s\n", thread, filename); infile = fopen(filename, "r"); if (infile == NULL) { perror("Opening results file"); exit(1); } mongoc_bulk_operation_t *bulk = mongoc_collection_create_bulk_operation( collection, true, NULL); if(!bulk) { printf("Failed to create bulk op\n"); } rptr = fgets(rlinebuf, BUFSIZ, infile); rlinebuf[strlen(rlinebuf) - 1] = '\0'; //Read the Results Line while (rptr) { total++; if (total % (INSERT_THREADS_PER_SHARD * nshards) == thread) { ritem = strtok_r(rptr, "|", &rlast); rfcount = 0; record = bson_new(); //Two part ID - a loader (32 bits for that) and a one_up bson_append_int64(record, "_id", -1, (chunkno << 32) + total); while (ritem) { switch (resulttype[rfcount]) { case 0: //printf("%s\n",ritem); bson_append_utf8(record, resultfields[rfcount], -1, ritem, -1); break; case 1: bson_append_int32(record, resultfields[rfcount], -1, atoi(ritem)); break; case 2: if (strncmp(ritem, "NULL", 4)) { struct tm tm; if (strptime(ritem, "%Y-%m-%d", &tm)) { time_t t = mktime(&tm); // t is now your desired time_t bson_append_date_time(record, resultfields[rfcount], -1, (long long) t * 1000); } } break; default: printf("Unknown type col %d = %d\n", rfcount, resulttype[rfcount]); } ritem = strtok_r(NULL, "|", &rlast); rfcount++; } mongoc_bulk_operation_insert(bulk, record); bson_destroy(record); if (batchcount == (BATCHSIZE - 1)) { int ret = mongoc_bulk_operation_execute(bulk, &reply, &error); if (!ret) { printf( "Error: %s\n", error.message); } if (thread == 0) printf("%s %d\n", filename, total); bson_destroy(&reply); mongoc_bulk_operation_destroy(bulk); batchcount = 0; bulk = mongoc_collection_create_bulk_operation(collection, true, NULL); } else { batchcount++; } } //Read next line from file rptr = fgets(rlinebuf, BUFSIZ, infile); rlinebuf[strlen(rlinebuf) - 1] = '\0'; } int ret = mongoc_bulk_operation_execute(bulk, &reply, &error); if (!ret) { fprintf(stderr, "Error: %s\n", error.message); } if (thread == 0) printf("%s %d\n", filename, total); bson_destroy(&reply); mongoc_collection_destroy(collection); mongoc_bulk_operation_destroy(bulk); mongoc_client_destroy(conn); }
static void test_index_offset (void) { mongoc_bulk_operation_t *bulk; mongoc_collection_t *collection; mongoc_client_t *client; bson_error_t error; bson_iter_t iter; bson_iter_t ar; bson_iter_t citer; bson_t reply; bson_t *sel; bson_t *doc; bool r; client = mongoc_client_new (gTestUri); assert (client); collection = get_test_collection (client, "test_index_offset"); assert (collection); doc = bson_new (); BSON_APPEND_INT32 (doc, "abcd", 1234); r = mongoc_collection_insert (collection, MONGOC_INSERT_NONE, doc, NULL, &error); assert (r); bson_destroy (doc); bulk = mongoc_collection_create_bulk_operation (collection, true, NULL); assert (bulk); sel = BCON_NEW ("abcd", BCON_INT32 (1234)); doc = BCON_NEW ("$set", "{", "hello", "there", "}"); mongoc_bulk_operation_delete_one (bulk, sel); mongoc_bulk_operation_update (bulk, sel, doc, true); r = mongoc_bulk_operation_execute (bulk, &reply, &error); assert (r); assert (bson_iter_init_find (&iter, &reply, "nUpserted")); assert (BSON_ITER_HOLDS_INT32 (&iter)); assert (bson_iter_int32 (&iter) == 1); assert (bson_iter_init_find (&iter, &reply, "nMatched")); assert (BSON_ITER_HOLDS_INT32 (&iter)); assert (bson_iter_int32 (&iter) == 0); assert (bson_iter_init_find (&iter, &reply, "nRemoved")); assert (BSON_ITER_HOLDS_INT32 (&iter)); assert (bson_iter_int32 (&iter) == 1); assert (bson_iter_init_find (&iter, &reply, "nInserted")); assert (BSON_ITER_HOLDS_INT32 (&iter)); assert (bson_iter_int32 (&iter) == 0); if (bson_iter_init_find (&iter, &reply, "nModified")) { assert (BSON_ITER_HOLDS_INT32 (&iter)); assert (bson_iter_int32 (&iter) == 0); } assert (bson_iter_init_find (&iter, &reply, "upserted")); assert (BSON_ITER_HOLDS_ARRAY (&iter)); assert (bson_iter_recurse (&iter, &ar)); assert (bson_iter_next (&ar)); assert (BSON_ITER_HOLDS_DOCUMENT (&ar)); assert (bson_iter_recurse (&ar, &citer)); assert (bson_iter_next (&citer)); assert (BSON_ITER_IS_KEY (&citer, "index")); assert (bson_iter_int32 (&citer) == 1); assert (bson_iter_next (&citer)); assert (BSON_ITER_IS_KEY (&citer, "_id")); assert (BSON_ITER_HOLDS_OID (&citer)); assert (!bson_iter_next (&citer)); assert (!bson_iter_next (&ar)); assert (bson_iter_init_find (&iter, &reply, "writeErrors")); assert (BSON_ITER_HOLDS_ARRAY (&iter)); assert (bson_iter_recurse (&iter, &ar)); assert (!bson_iter_next (&ar)); bson_destroy (&reply); r = mongoc_collection_drop (collection, &error); assert (r); mongoc_bulk_operation_destroy (bulk); mongoc_collection_destroy (collection); mongoc_client_destroy (client); bson_destroy (doc); bson_destroy (sel); }
int64_t load_table (mongoc_database_t *db, const char *table_name, bson_t *bson_schema) { int64_t ret = true; column_map_t *column_map, *column_map_p; int column_map_size, i; double start_time, end_time, delta_time; FILE *fp; mongoc_collection_t *collection; mongoc_bulk_operation_t *bulk; size_t n_docs = 0; char *token; bson_t bson, reply; int64_t count = 0; bson_error_t error; fprintf (stderr, "load_table table_name: \"%s\"\n", table_name); get_column_map (bson_schema, table_name, &column_map, &column_map_size) || DIE; snprintf (mbdump_file, MAXPATHLEN, "%s/%s", mbdump_dir, table_name); /* fprintf (stderr, "mbdump_file: \"%s\"\n", mbdump_file); */ start_time = dtimeofday (); fp = fopen (mbdump_file, "r"); if (!fp) DIE; collection = mongoc_database_get_collection (db, table_name); bulk = mongoc_collection_create_bulk_operation (collection, true, NULL); bson_init (&bson); while (ret && fgets (buf, BUFSIZ, fp)) { /* fputs (buf, stdout); */ chomp (buf); for (i = 0, column_map_p = column_map, token = strtok_single (buf, "\t"); i < column_map_size; i++, column_map_p++, token = strtok_single (NULL, "\t")) { bool ret; /* fprintf (stderr, "%s: \"%s\" [%d/%d](%s)\n", column_map_p->column_name, token, i, column_map_size, column_map_p->data_type); fflush (stdout); */ ret = (*column_map_p->bson_append_from_s) (&bson, column_map_p->column_name, token); ret || fprintf (stderr, "WARNING: column_map_p->bson_append_from_s failed column %s: \"%s\" [%d/%d](%s)\n", column_map_p->column_name, token, i, column_map_size, column_map_p->data_type); } /* bson_printf ("bson: %s\n", &bson); */ mongoc_bulk_operation_insert (bulk, &bson); bson_reinit (&bson); if (++n_docs == BULK_OPS_SIZE) { ret = mongoc_bulk_operation_execute (bulk, &reply, &error); if (ret) { count += n_docs; if (count % PROGRESS_SIZE == 0) { fputc('.', stdout); fflush(stdout); } } else fprintf (stderr, "mongoc_cursor_bulk_insert execute failure: %s\n", error.message); n_docs = 0; mongoc_bulk_operation_destroy (bulk); bulk = mongoc_collection_create_bulk_operation (collection, true, NULL); } } if (ret && n_docs > 0) { ret = mongoc_bulk_operation_execute (bulk, &reply, &error); if (ret) count += n_docs; else fprintf (stderr, "mongoc_cursor_bulk_insert execute failure: %s\n", error.message); } fputc('.', stdout); fputc('\n', stdout); fflush(stdout); bson_destroy (&bson); mongoc_bulk_operation_destroy (bulk); mongoc_collection_destroy (collection); fclose (fp); end_time = dtimeofday (); delta_time = end_time - start_time + 0.0000001; fprintf (stderr, "info: real: %.2f, count: %"PRId64", %"PRId64" docs/sec\n", delta_time, count, (int64_t)round (count/delta_time)); fflush (stderr); free (column_map); return ret ? count : -1; }