// PD_TRACE_DECLARE_FUNCTION ( SDB__CLSSPLIT_INIT, "_rtnSplit::init" ) INT32 _rtnSplit::init ( INT32 flags, INT64 numToSkip, INT64 numToReturn, const CHAR * pMatcherBuff, const CHAR * pSelectBuff, const CHAR * pOrderByBuff, const CHAR * pHintBuff ) { INT32 rc = SDB_OK ; PD_TRACE_ENTRY ( SDB__CLSSPLIT_INIT ) ; const CHAR *pCollectionName = NULL ; const CHAR *pTargetName = NULL ; const CHAR *pSourceName = NULL ; try { BSONObj boRequest ( pMatcherBuff ) ; BSONElement beName = boRequest.getField ( CAT_COLLECTION_NAME ) ; BSONElement beTarget = boRequest.getField ( CAT_TARGET_NAME ) ; BSONElement beSplitKey = boRequest.getField ( CAT_SPLITVALUE_NAME ) ; BSONElement beSource = boRequest.getField ( CAT_SOURCE_NAME ) ; BSONElement bePercent = boRequest.getField ( CAT_SPLITPERCENT_NAME ) ; PD_CHECK ( !beName.eoo() && beName.type() == String, SDB_INVALIDARG, error, PDERROR, "Invalid collection name: %s", beName.toString().c_str() ) ; pCollectionName = beName.valuestr() ; PD_CHECK ( ossStrlen ( pCollectionName ) < DMS_COLLECTION_SPACE_NAME_SZ + DMS_COLLECTION_NAME_SZ + 1, SDB_INVALIDARG, error, PDERROR, "Collection name is too long: %s", pCollectionName ) ; ossStrncpy ( _szCollection, pCollectionName, DMS_COLLECTION_SPACE_NAME_SZ + DMS_COLLECTION_NAME_SZ + 1 ) ; PD_CHECK ( !beTarget.eoo() && beTarget.type() == String, SDB_INVALIDARG, error, PDERROR, "Invalid target group name: %s", beTarget.toString().c_str() ) ; pTargetName = beTarget.valuestr() ; PD_CHECK ( ossStrlen ( pTargetName ) < OP_MAXNAMELENGTH, SDB_INVALIDARG, error, PDERROR, "target group name is too long: %s", pTargetName ) ; ossStrncpy ( _szTargetName, pTargetName, OP_MAXNAMELENGTH ) ; PD_CHECK ( !beSource.eoo() && beSource.type() == String, SDB_INVALIDARG, error, PDERROR, "Invalid source group name: %s", beSource.toString().c_str() ) ; pSourceName = beSource.valuestr() ; PD_CHECK ( ossStrlen ( pSourceName ) < OP_MAXNAMELENGTH, SDB_INVALIDARG, error, PDERROR, "source group name is too long: %s", pSourceName ) ; ossStrncpy ( _szSourceName, pSourceName, OP_MAXNAMELENGTH ) ; PD_CHECK ( !beSplitKey.eoo() && beSplitKey.type() == Object, SDB_INVALIDARG, error, PDERROR, "Invalid split key: %s", beSplitKey.toString().c_str() ) ; _splitKey = beSplitKey.embeddedObject () ; _percent = bePercent.numberDouble() ; } catch ( std::exception &e ) { PD_RC_CHECK ( SDB_SYS, PDERROR, "Exception handled when parsing split request: %s", e.what() ) ; } PD_TRACE4 ( SDB__CLSSPLIT_INIT, PD_PACK_STRING ( pCollectionName ), PD_PACK_STRING ( pTargetName ), PD_PACK_STRING ( pSourceName ), PD_PACK_STRING ( _splitKey.toString().c_str() ) ) ; done: PD_TRACE_EXITRC ( SDB__CLSSPLIT_INIT, rc ) ; return rc ; error: goto done ; }
/* slave: pull some data from the master's oplog note: not yet in db mutex at this point. @return -1 error 0 ok, don't sleep 1 ok, sleep */ int ReplSource::sync_pullOpLog(int& nApplied) { int okResultCode = 1; string ns = string("local.oplog.$") + sourceName(); log(2) << "repl: sync_pullOpLog " << ns << " syncedTo:" << syncedTo.toStringLong() << '\n'; bool tailing = true; oplogReader.tailCheck(); bool initial = syncedTo.isNull(); if ( !oplogReader.haveCursor() || initial ) { if ( initial ) { // Important to grab last oplog timestamp before listing databases. syncToTailOfRemoteLog(); BSONObj info; bool ok = oplogReader.conn()->runCommand( "admin", BSON( "listDatabases" << 1 ), info ); massert( 10389 , "Unable to get database list", ok ); BSONObjIterator i( info.getField( "databases" ).embeddedObject() ); while( i.moreWithEOO() ) { BSONElement e = i.next(); if ( e.eoo() ) break; string name = e.embeddedObject().getField( "name" ).valuestr(); if ( !e.embeddedObject().getBoolField( "empty" ) ) { if ( name != "local" ) { if ( only.empty() || only == name ) { log( 2 ) << "adding to 'addDbNextPass': "******"$gte", syncedTo.asDate()); BSONObjBuilder query; query.append("ts", q.done()); if ( !only.empty() ) { // note we may here skip a LOT of data table scanning, a lot of work for the master. query.appendRegex("ns", string("^") + only); // maybe append "\\." here? } BSONObj queryObj = query.done(); // e.g. queryObj = { ts: { $gte: syncedTo } } oplogReader.tailingQuery(ns.c_str(), queryObj); tailing = false; } else { log(2) << "repl: tailing=true\n"; } if( !oplogReader.haveCursor() ) { problem() << "repl: dbclient::query returns null (conn closed?)" << endl; oplogReader.resetConnection(); return -1; } // show any deferred database creates from a previous pass { set<string>::iterator i = addDbNextPass.begin(); if ( i != addDbNextPass.end() ) { BSONObjBuilder b; b.append("ns", *i + '.'); b.append("op", "db"); BSONObj op = b.done(); sync_pullOpLog_applyOperation(op, false); } } if ( !oplogReader.more() ) { if ( tailing ) { log(2) << "repl: tailing & no new activity\n"; if( oplogReader.awaitCapable() ) okResultCode = 0; // don't sleep } else { log() << "repl: " << ns << " oplog is empty\n"; } { dblock lk; save(); } return okResultCode; } OpTime nextOpTime; { BSONObj op = oplogReader.next(); BSONElement ts = op.getField("ts"); if ( ts.type() != Date && ts.type() != Timestamp ) { string err = op.getStringField("$err"); if ( !err.empty() ) { // 13051 is "tailable cursor requested on non capped collection" if (op.getIntField("code") == 13051) { problem() << "trying to slave off of a non-master" << '\n'; massert( 13344 , "trying to slave off of a non-master", false ); } else { problem() << "repl: $err reading remote oplog: " + err << '\n'; massert( 10390 , "got $err reading remote oplog", false ); } } else { problem() << "repl: bad object read from remote oplog: " << op.toString() << '\n'; massert( 10391 , "repl: bad object read from remote oplog", false); } } nextOpTime = OpTime( ts.date() ); log(2) << "repl: first op time received: " << nextOpTime.toString() << '\n'; if ( initial ) { log(1) << "repl: initial run\n"; } if( tailing ) { if( !( syncedTo < nextOpTime ) ) { log() << "repl ASSERTION failed : syncedTo < nextOpTime" << endl; log() << "repl syncTo: " << syncedTo.toStringLong() << endl; log() << "repl nextOpTime: " << nextOpTime.toStringLong() << endl; assert(false); } oplogReader.putBack( op ); // op will be processed in the loop below nextOpTime = OpTime(); // will reread the op below } else if ( nextOpTime != syncedTo ) { // didn't get what we queried for - error Nullstream& l = log(); l << "repl: nextOpTime " << nextOpTime.toStringLong() << ' '; if ( nextOpTime < syncedTo ) l << "<??"; else l << ">"; l << " syncedTo " << syncedTo.toStringLong() << '\n'; log() << "repl: time diff: " << (nextOpTime.getSecs() - syncedTo.getSecs()) << "sec\n"; log() << "repl: tailing: " << tailing << '\n'; log() << "repl: data too stale, halting replication" << endl; replInfo = replAllDead = "data too stale halted replication"; assert( syncedTo < nextOpTime ); throw SyncException(); } else { /* t == syncedTo, so the first op was applied previously or it is the first op of initial query and need not be applied. */ } } // apply operations { int n = 0; time_t saveLast = time(0); while ( 1 ) { bool moreInitialSyncsPending = !addDbNextPass.empty() && n; // we need "&& n" to assure we actually process at least one op to get a sync point recorded in the first place. if ( moreInitialSyncsPending || !oplogReader.more() ) { dblock lk; // NOTE aaron 2011-03-29 This block may be unnecessary, but I'm leaving it in place to avoid changing timing behavior. { dbtemprelease t; if ( !moreInitialSyncsPending && oplogReader.more() ) { continue; } // otherwise, break out of loop so we can set to completed or clone more dbs } if( oplogReader.awaitCapable() && tailing ) okResultCode = 0; // don't sleep syncedTo = nextOpTime; save(); // note how far we are synced up to now log() << "repl: applied " << n << " operations" << endl; nApplied = n; log() << "repl: end sync_pullOpLog syncedTo: " << syncedTo.toStringLong() << endl; break; } else { } OCCASIONALLY if( n > 0 && ( n > 100000 || time(0) - saveLast > 60 ) ) { // periodically note our progress, in case we are doing a lot of work and crash dblock lk; syncedTo = nextOpTime; // can't update local log ts since there are pending operations from our peer save(); log() << "repl: checkpoint applied " << n << " operations" << endl; log() << "repl: syncedTo: " << syncedTo.toStringLong() << endl; saveLast = time(0); n = 0; } BSONObj op = oplogReader.next(); unsigned b = replApplyBatchSize; bool justOne = b == 1; scoped_ptr<writelock> lk( justOne ? 0 : new writelock() ); while( 1 ) { BSONElement ts = op.getField("ts"); if( !( ts.type() == Date || ts.type() == Timestamp ) ) { log() << "sync error: problem querying remote oplog record" << endl; log() << "op: " << op.toString() << endl; log() << "halting replication" << endl; replInfo = replAllDead = "sync error: no ts found querying remote oplog record"; throw SyncException(); } OpTime last = nextOpTime; nextOpTime = OpTime( ts.date() ); if ( !( last < nextOpTime ) ) { log() << "sync error: last applied optime at slave >= nextOpTime from master" << endl; log() << " last: " << last.toStringLong() << endl; log() << " nextOpTime: " << nextOpTime.toStringLong() << endl; log() << " halting replication" << endl; replInfo = replAllDead = "sync error last >= nextOpTime"; uassert( 10123 , "replication error last applied optime at slave >= nextOpTime from master", false); } if ( replSettings.slavedelay && ( unsigned( time( 0 ) ) < nextOpTime.getSecs() + replSettings.slavedelay ) ) { assert( justOne ); oplogReader.putBack( op ); _sleepAdviceTime = nextOpTime.getSecs() + replSettings.slavedelay + 1; dblock lk; if ( n > 0 ) { syncedTo = last; save(); } log() << "repl: applied " << n << " operations" << endl; log() << "repl: syncedTo: " << syncedTo.toStringLong() << endl; log() << "waiting until: " << _sleepAdviceTime << " to continue" << endl; return okResultCode; } sync_pullOpLog_applyOperation(op, !justOne); n++; if( --b == 0 ) break; // if to here, we are doing mulpile applications in a singel write lock acquisition if( !oplogReader.moreInCurrentBatch() ) { // break if no more in batch so we release lock while reading from the master break; } op = oplogReader.next(); getDur().commitIfNeeded(); } } } return okResultCode; }
int run() { if ( hasParam( "repair" ) ){ warning() << "repair is a work in progress" << endl; return repair(); } { string q = getParam("query"); if ( q.size() ) _query = fromjson( q ); } string opLogName = ""; unsigned long long opLogStart = 0; if (hasParam("oplog")) { if (hasParam("query") || hasParam("db") || hasParam("collection")) { cout << "oplog mode is only supported on full dumps" << endl; return -1; } BSONObj isMaster; conn("true").simpleCommand("admin", &isMaster, "isMaster"); if (isMaster.hasField("hosts")) { // if connected to replica set member opLogName = "local.oplog.rs"; } else { opLogName = "local.oplog.$main"; if ( ! isMaster["ismaster"].trueValue() ) { cout << "oplog mode is only supported on master or replica set member" << endl; return -1; } } auth("local"); BSONObj op = conn(true).findOne(opLogName, Query().sort("$natural", -1), 0, QueryOption_SlaveOk); if (op.isEmpty()) { cout << "No operations in oplog. Please ensure you are connecting to a master." << endl; return -1; } assert(op["ts"].type() == Timestamp); opLogStart = op["ts"]._numberLong(); } // check if we're outputting to stdout string out = getParam("out"); if ( out == "-" ) { if ( _db != "*" && _coll != "*" ) { writeCollectionStdout( _db+"."+_coll ); return 0; } else { cout << "You must specify database and collection to print to stdout" << endl; return -1; } } _usingMongos = isMongos(); path root( out ); string db = _db; if ( db == "*" ) { cout << "all dbs" << endl; auth( "admin" ); BSONObj res = conn( true ).findOne( "admin.$cmd" , BSON( "listDatabases" << 1 ) ); if ( ! res["databases"].isABSONObj() ) { error() << "output of listDatabases isn't what we expected, no 'databases' field:\n" << res << endl; return -2; } BSONObj dbs = res["databases"].embeddedObjectUserCheck(); set<string> keys; dbs.getFieldNames( keys ); for ( set<string>::iterator i = keys.begin() ; i != keys.end() ; i++ ) { string key = *i; if ( ! dbs[key].isABSONObj() ) { error() << "database field not an object key: " << key << " value: " << dbs[key] << endl; return -3; } BSONObj dbobj = dbs[key].embeddedObjectUserCheck(); const char * dbName = dbobj.getField( "name" ).valuestr(); if ( (string)dbName == "local" ) continue; go ( dbName , root / dbName ); } } else { auth( db ); go( db , root / db ); } if (!opLogName.empty()) { BSONObjBuilder b; b.appendTimestamp("$gt", opLogStart); _query = BSON("ts" << b.obj()); writeCollectionFile( opLogName , root / "oplog.bson" ); } return 0; }
bool _userCreateNS(const char *ns, const BSONObj& options, string& err, bool *deferIdIndex) { LOG(1) << "create collection " << ns << ' ' << options << endl; if ( nsdetails(ns) ) { err = "collection already exists"; return false; } long long size = Extent::initialSize(128); { BSONElement e = options.getField("size"); if ( e.isNumber() ) { size = e.numberLong(); uassert( 10083 , "create collection invalid size spec", size >= 0 ); size += 0xff; size &= 0xffffffffffffff00LL; if ( size < Extent::minSize() ) size = Extent::minSize(); } } bool newCapped = false; long long mx = 0; if( options["capped"].trueValue() ) { newCapped = true; BSONElement e = options.getField("max"); if ( e.isNumber() ) { mx = e.numberLong(); uassert( 16495, "max in a capped collection has to be < 2^31 or not set", NamespaceDetails::validMaxCappedDocs(&mx) ); } } cc().database()->createCollection( ns, options["capped"].trueValue(), &options ); Collection* collection = cc().database()->getCollection( ns ); verify( collection ); // $nExtents just for debug/testing. BSONElement e = options.getField( "$nExtents" ); if ( e.type() == Array ) { // We create one extent per array entry, with size specified // by the array value. BSONObjIterator i( e.embeddedObject() ); while( i.more() ) { BSONElement e = i.next(); int size = int( e.number() ); verify( size <= 0x7fffffff ); // $nExtents is just for testing - always allocate new extents // rather than reuse existing extents so we have some predictibility // in the extent size used by our tests collection->increaseStorageSize( (int)size, false ); } } else if ( int( e.number() ) > 0 ) { // We create '$nExtents' extents, each of size 'size'. int nExtents = int( e.number() ); verify( size <= 0x7fffffff ); for ( int i = 0; i < nExtents; ++i ) { verify( size <= 0x7fffffff ); // $nExtents is just for testing - always allocate new extents // rather than reuse existing extents so we have some predictibility // in the extent size used by our tests collection->increaseStorageSize( (int)size, false ); } } else { // This is the non test case, where we don't have a $nExtents spec. while ( size > 0 ) { const int max = Extent::maxSize(); const int min = Extent::minSize(); int desiredExtentSize = static_cast<int> (size > max ? max : size); desiredExtentSize = static_cast<int> (desiredExtentSize < min ? min : desiredExtentSize); desiredExtentSize &= 0xffffff00; Extent* e = collection->increaseStorageSize( (int)desiredExtentSize, true ); size -= e->length; } } NamespaceDetails *d = nsdetails(ns); verify(d); bool ensure = true; // respect autoIndexId if set. otherwise, create an _id index for all colls, except for // capped ones in local w/o autoIndexID (reason for the exception is for the oplog and // non-replicated capped colls) if( options.hasField( "autoIndexId" ) || (newCapped && nsToDatabase( ns ) == "local" ) ) { ensure = options.getField( "autoIndexId" ).trueValue(); } if( ensure ) { if( deferIdIndex ) *deferIdIndex = true; else ensureIdIndexForNewNs( ns ); } if ( mx > 0 ) d->setMaxCappedDocs( mx ); if ( options["flags"].numberInt() ) { d->replaceUserFlags( options["flags"].numberInt() ); } return true; }
int run() { if (mongoDumpGlobalParams.repair) { return repair(); } { if (mongoDumpGlobalParams.query.size()) { _query = fromjson(mongoDumpGlobalParams.query); } } if (mongoDumpGlobalParams.dumpUsersAndRoles) { uassertStatusOK(auth::getRemoteStoredAuthorizationVersion(&conn(true), &_serverAuthzVersion)); uassert(17369, mongoutils::str::stream() << "Backing up users and roles is only supported for " "clusters with auth schema versions 1 or 3, found: " << _serverAuthzVersion, _serverAuthzVersion == AuthorizationManager::schemaVersion24 || _serverAuthzVersion == AuthorizationManager::schemaVersion26Final); } string opLogName = ""; unsigned long long opLogStart = 0; if (mongoDumpGlobalParams.useOplog) { BSONObj isMaster; conn("true").simpleCommand("admin", &isMaster, "isMaster"); if (isMaster.hasField("hosts")) { // if connected to replica set member opLogName = "local.oplog.rs"; } else { opLogName = "local.oplog.$main"; if ( ! isMaster["ismaster"].trueValue() ) { toolError() << "oplog mode is only supported on master or replica set member" << std::endl; return -1; } } BSONObj op = conn(true).findOne(opLogName, Query().sort("$natural", -1), 0, QueryOption_SlaveOk); if (op.isEmpty()) { toolError() << "No operations in oplog. Please ensure you are connecting to a " << "master." << std::endl; return -1; } verify(op["ts"].type() == Timestamp); opLogStart = op["ts"]._numberLong(); } // check if we're outputting to stdout if (mongoDumpGlobalParams.outputDirectory == "-") { if (toolGlobalParams.db != "" && toolGlobalParams.coll != "") { writeCollectionStdout(toolGlobalParams.db + "." + toolGlobalParams.coll); return 0; } else { toolError() << "You must specify database and collection to print to stdout" << std::endl; return -1; } } _usingMongos = isMongos(); boost::filesystem::path root(mongoDumpGlobalParams.outputDirectory); if (toolGlobalParams.db == "") { if (toolGlobalParams.coll != "") { toolError() << "--db must be specified with --collection" << std::endl; return -1; } toolInfoLog() << "all dbs" << std::endl; BSONObj res = conn( true ).findOne( "admin.$cmd" , BSON( "listDatabases" << 1 ) ); if ( ! res["databases"].isABSONObj() ) { toolError() << "output of listDatabases isn't what we expected, no 'databases' " << "field:\n" << res << std::endl; return -2; } BSONObj dbs = res["databases"].embeddedObjectUserCheck(); set<string> keys; dbs.getFieldNames( keys ); for ( set<string>::iterator i = keys.begin() ; i != keys.end() ; i++ ) { string key = *i; if ( ! dbs[key].isABSONObj() ) { toolError() << "database field not an document key: " << key << " value: " << dbs[key] << std::endl; return -3; } BSONObj dbobj = dbs[key].embeddedObjectUserCheck(); const char * dbName = dbobj.getField( "name" ).valuestr(); if ( (string)dbName == "local" ) continue; boost::filesystem::path outdir = root / dbName; toolInfoLog() << "DATABASE: " << dbName << "\t to \t" << outdir.string() << std::endl; go ( dbName , "", _query, outdir, "" ); } } else { boost::filesystem::path outdir = root / toolGlobalParams.db; toolInfoLog() << "DATABASE: " << toolGlobalParams.db << "\t to \t" << outdir.string() << std::endl; go(toolGlobalParams.db, toolGlobalParams.coll, _query, outdir, ""); if (mongoDumpGlobalParams.dumpUsersAndRoles && _serverAuthzVersion == AuthorizationManager::schemaVersion26Final && toolGlobalParams.db != "admin") { toolInfoLog() << "Backing up user and role data for the " << toolGlobalParams.db << " database"; Query query = Query(BSON("db" << toolGlobalParams.db)); go("admin", "system.users", query, outdir, "$admin.system.users"); go("admin", "system.roles", query, outdir, "$admin.system.roles"); } } if (!opLogName.empty()) { BSONObjBuilder b; b.appendTimestamp("$gt", opLogStart); _query = BSON("ts" << b.obj()); writeCollectionFile( opLogName , _query, root / "oplog.bson" ); } return 0; }
// PD_TRACE_DECLARE_FUNCTION ( SDB_RTNEXPLAIN, "rtnExplain" ) INT32 rtnExplain( const CHAR *pCollectionName, const BSONObj &selector, const BSONObj &matcher, const BSONObj &orderBy, const BSONObj &hint, SINT32 flags, SINT64 numToSkip, SINT64 numToReturn, pmdEDUCB *cb, SDB_DMSCB *dmsCB, SDB_RTNCB *rtnCB, INT64 &contextID, rtnContextBase **ppContext ) { INT32 rc = SDB_OK ; PD_TRACE_ENTRY ( SDB_RTNEXPLAIN ) ; SDB_ASSERT ( cb, "educb can't be NULL" ) ; SDB_ASSERT ( dmsCB, "dmsCB can't be NULL" ) ; SDB_ASSERT ( rtnCB, "rtnCB can't be NULL" ) ; BSONObj explainOptions ; BSONObj realHint ; BSONElement ele = hint.getField( FIELD_NAME_OPTIONS ) ; if ( Object == ele.type() ) { explainOptions = ele.embeddedObject() ; } ele = hint.getField( FIELD_NAME_HINT ) ; if ( Object == ele.type() ) { realHint = ele.embeddedObject() ; } rtnQueryOptions options( matcher, selector, orderBy, realHint, pCollectionName, numToSkip, numToReturn, OSS_BIT_CLEAR( flags, FLG_QUERY_EXPLAIN), FALSE ) ; rtnContextExplain *context = NULL ; rc = rtnCB->contextNew( RTN_CONTEXT_EXPLAIN, ( rtnContext **)( &context ), contextID, cb ) ; if ( SDB_OK != rc ) { PD_LOG( PDERROR, "failed to create explain context:%d", rc ) ; goto error ; } rc = context->open( options, explainOptions ) ; if ( SDB_OK != rc ) { PD_LOG( PDERROR, "failed to open explain context:%d", rc ) ; goto error ; } if ( ppContext ) { *ppContext = context ; } done: PD_TRACE_EXITRC( SDB_RTNEXPLAIN, rc ) ; return rc ; error: if ( -1 != contextID ) { rtnCB->contextDelete( contextID, cb ) ; contextID = -1 ; } goto done ; }
INT32 _spdFMP::_extractMsg( BSONObj &msg, BOOLEAN &extracted ) { INT32 rc = SDB_OK ; extracted = FALSE ; SDB_ASSERT( 0 <= _expect, "impossible" ) ; /// magic has already been found. if ( sizeof( SPD_MAGIC ) == _expect ) { found: if ( (_totalRead - _itr) < (INT32)sizeof(INT32) ) { extracted = FALSE ; goto done ; } else { SINT32 bsonLen = *((SINT32 *)(_readBuf+_itr)) ; if ( (_totalRead - _itr) < bsonLen ) { rc = FALSE ; goto done ; } else if ( (_totalRead - _itr) == bsonLen ) { SDB_ASSERT( _itr >= (INT32)sizeof( SPD_MAGIC ) , "impossible" ) ; BSONObj tmp ; try { tmp = BSONObj( _readBuf + _itr ) ; } catch ( std::exception &e ) { PD_LOG( PDERROR, "unexpected err happened:%s", e.what() ) ; rc = SDB_SYS ; goto error ; } if ( sizeof( SPD_MAGIC ) == _itr ) { /// only a bson msg. msg = tmp ; extracted = TRUE ; } else { /// not only a bson msg. _readBuf[_itr - 4] = '\0' ; BSONElement retCode = tmp.getField( FMP_RES_CODE ) ; BSONElement errMsg = tmp.getField( FMP_ERR_MSG ) ; /// some code like 'print' may return msg more than a bsonobj. /// we must parse it's return code. if it is ok, we ignore /// print. else we put it into errmsg. if ( !retCode.eoo() && NumberInt != retCode.type() ) { rc = SDB_SYS ; PD_LOG( PDERROR, "invalid type of retCode:%d", retCode.type() ) ; goto error ; } else if ( !retCode.eoo() ) { if ( SDB_OK != retCode.Int() ) { if ( !errMsg.eoo() ) { msg = tmp ; } else { BSONObjBuilder builder ; builder.append( FMP_ERR_MSG, _readBuf ) ; builder.append( retCode ) ; msg = builder.obj() ; } } else { msg = tmp ; } } else { /// retCode is eoo. msg = tmp ; } extracted = TRUE ; } } else { SDB_ASSERT( FALSE, "impossible" ) ; rc = SDB_SYS ; PD_LOG( PDERROR, "left len can not be lager than objsize" ) ; goto error ; } } } else { while ( _itr < _totalRead && (UINT32)_expect < sizeof( SPD_MAGIC ) ) { if ( SPD_MAGIC[_expect] == _readBuf[_itr] ) { ++_itr ; ++_expect ; if ( sizeof( SPD_MAGIC ) == _expect ) { goto found ; } } else if ( 0 == _expect ) { ++_itr ; } else { _expect = 0 ; } } } done: return rc ; error: goto done ; }
void go( const string db , const boost::filesystem::path outdir ) { log() << "DATABASE: " << db << "\t to \t" << outdir.string() << endl; boost::filesystem::create_directories( outdir ); map <string, BSONObj> collectionOptions; multimap <string, BSONObj> indexes; vector <string> collections; // Save indexes for database string ins = db + ".system.indexes"; auto_ptr<DBClientCursor> cursor = conn( true ).query( ins.c_str() , Query() , 0 , 0 , 0 , QueryOption_SlaveOk | QueryOption_NoCursorTimeout ); while ( cursor->more() ) { BSONObj obj = cursor->nextSafe(); const string name = obj.getField( "ns" ).valuestr(); indexes.insert( pair<string, BSONObj> (name, obj.getOwned()) ); } string sns = db + ".system.namespaces"; cursor = conn( true ).query( sns.c_str() , Query() , 0 , 0 , 0 , QueryOption_SlaveOk | QueryOption_NoCursorTimeout ); while ( cursor->more() ) { BSONObj obj = cursor->nextSafe(); const string name = obj.getField( "name" ).valuestr(); if (obj.hasField("options")) { collectionOptions[name] = obj.getField("options").embeddedObject().getOwned(); } // skip namespaces with $ in them only if we don't specify a collection to dump if ( _coll == "" && name.find( ".$" ) != string::npos ) { log(1) << "\tskipping collection: " << name << endl; continue; } const string filename = name.substr( db.size() + 1 ); //if a particular collections is specified, and it's not this one, skip it if ( _coll != "" && db + "." + _coll != name && _coll != name ) continue; // raise error before writing collection with non-permitted filename chars in the name size_t hasBadChars = name.find_first_of("/\0"); if (hasBadChars != string::npos){ error() << "Cannot dump " << name << ". Collection has '/' or null in the collection name." << endl; continue; } // Don't dump indexes if ( endsWith(name.c_str(), ".system.indexes") ) { continue; } if ( _coll != "" && db + "." + _coll != name && _coll != name ) continue; collections.push_back(name); } for (vector<string>::iterator it = collections.begin(); it != collections.end(); ++it) { string name = *it; const string filename = name.substr( db.size() + 1 ); writeCollectionFile( name , outdir / ( filename + ".bson" ) ); writeMetadataFile( name, outdir / (filename + ".metadata.json"), collectionOptions, indexes); } }
int LocalMemoryGridFile::flush() { trace() << " -> LocalMemoryGridFile::flush {file: " << _filename << "}" << endl; if (!_dirty) { // Since, there are no dirty chunks, this does not need a flush info() << "buffers are not dirty.. need not flush {filename: " << _filename << "}" << endl; return 0; } size_t bufferLen = 0; boost::shared_array<char> buffer = createFlushBuffer(bufferLen); if (!buffer.get() && bufferLen > 0) { // Failed to create flush buffer return -ENOMEM; } // Get the existing gridfile from GridFS to get metadata and delete the // file from the system try { ScopedDbConnection dbc(globalFSOptions._connectString); GridFS gridFS(dbc.conn(), globalFSOptions._db, globalFSOptions._collPrefix); GridFile origGridFile = gridFS.findFile(BSON("filename" << _filename)); if (!origGridFile.exists()) { dbc.done(); warn() << "Requested file not found for flushing back data {file: " << _filename << "}" << endl; return -EBADF; } //TODO: Make checks for appropriate object correctness //i.e. do not update anything that is not a Regular File //Check what happens in case of a link gridFS.removeFile(_filename); trace() << "Removing the current file from GridFS {file: " << _filename << "}" << endl; //TODO: Check for remove status if that was successfull or not //TODO: Rather have an update along with active / passive flag for the //file try { GridFS gridFS(dbc.conn(), globalFSOptions._db, globalFSOptions._collPrefix); // Create an empty file to signify the file creation and open a local file for the same trace() << "Adding new file to GridFS {file: " << _filename << "}" << endl; BSONObj fileObj = gridFS.storeFile(buffer.get(), bufferLen, _filename); if (!fileObj.isValid()) { warn() << "Failed to save file object in data flush {file: " << _filename << "}" << std::endl; dbc.done(); return -EBADF; } // Update the last updated date for the document BSONObj metadata = origGridFile.getMetadata(); BSONElement fileObjId = fileObj.getField("_id"); dbc->update(globalFSOptions._filesNS, BSON("_id" << fileObjId.OID()), BSON("$set" << BSON( "uploadDate" << origGridFile.getUploadDate() << "metadata.type" << "file" << "metadata.filename" << mgridfs::getPathBasename(_filename) << "metadata.directory" << mgridfs::getPathDirname(_filename) << "metadata.lastUpdated" << jsTime() << "metadata.uid" << metadata["uid"] << "metadata.gid" << metadata["gid"] << "metadata.mode" << metadata["mode"] ) ) ); } catch (DBException& e) { error() << "Caught exception in saving remote file in flush {code: " << e.getCode() << ", what: " << e.what() << ", exception: " << e.toString() << "}" << endl; return -EIO; } dbc.done(); } catch (DBException& e) { // Something failed in getting the file from GridFS error() << "Caught exception in getting remote file for flush {code: " << e.getCode() << ", what: " << e.what() << ", exception: " << e.toString() << "}" << endl; return -EIO; } _dirty = false; debug() << "Completed flushing the file content to GridFS {file: " << _filename << "}" << endl; return 0; }
bool dbEval(const string& dbName, BSONObj& cmd, BSONObjBuilder& result, string& errmsg) { BSONElement e = cmd.firstElement(); uassert( 10046 , "eval needs Code" , e.type() == Code || e.type() == CodeWScope || e.type() == String ); const char *code = 0; switch ( e.type() ) { case String: case Code: code = e.valuestr(); break; case CodeWScope: code = e.codeWScopeCode(); break; default: verify(0); } verify( code ); if ( ! globalScriptEngine ) { errmsg = "db side execution is disabled"; return false; } const string userToken = ClientBasic::getCurrent()->getAuthorizationManager() ->getAuthenticatedPrincipalNamesToken(); auto_ptr<Scope> s = globalScriptEngine->getPooledScope( dbName, "dbeval" + userToken ); ScriptingFunction f = s->createFunction(code); if ( f == 0 ) { errmsg = (string)"compile failed: " + s->getError(); return false; } if ( e.type() == CodeWScope ) s->init( e.codeWScopeScopeDataUnsafe() ); s->localConnect( dbName.c_str() ); BSONObj args; { BSONElement argsElement = cmd.getField("args"); if ( argsElement.type() == Array ) { args = argsElement.embeddedObject(); if ( edebug ) { out() << "args:" << args.toString() << endl; out() << "code:\n" << code << endl; } } } int res; { Timer t; res = s->invoke(f, &args, 0, cmdLine.quota ? 10 * 60 * 1000 : 0 ); int m = t.millis(); if ( m > cmdLine.slowMS ) { out() << "dbeval slow, time: " << dec << m << "ms " << dbName << endl; if ( m >= 1000 ) log() << code << endl; else OCCASIONALLY log() << code << endl; } } if (res || s->isLastRetNativeCode()) { result.append("errno", (double) res); errmsg = "invoke failed: "; if (s->isLastRetNativeCode()) errmsg += "cannot return native function"; else errmsg += s->getError(); return false; } s->append( result , "retval" , "__returnValue" ); return true; }
static bool dbEval(OperationContext* txn, const string& dbName, const BSONObj& cmd, BSONObjBuilder& result, string& errmsg) { BSONElement e = cmd.firstElement(); uassert( 10046 , "eval needs Code" , e.type() == Code || e.type() == CodeWScope || e.type() == String ); const char *code = 0; switch ( e.type() ) { case String: case Code: code = e.valuestr(); break; case CodeWScope: code = e.codeWScopeCode(); break; default: verify(0); } verify( code ); if ( ! globalScriptEngine ) { errmsg = "db side execution is disabled"; return false; } scoped_ptr<Scope> s(globalScriptEngine->newScope()); ScriptingFunction f = s->createFunction(code); if ( f == 0 ) { errmsg = (string)"compile failed: " + s->getError(); return false; } s->localConnectForDbEval(txn, dbName.c_str()); if ( e.type() == CodeWScope ) s->init( e.codeWScopeScopeDataUnsafe() ); BSONObj args; { BSONElement argsElement = cmd.getField("args"); if ( argsElement.type() == Array ) { args = argsElement.embeddedObject(); if ( edebug ) { log() << "args:" << args.toString() << endl; log() << "code:\n" << code << endl; } } } int res; { Timer t; res = s->invoke(f, &args, 0, storageGlobalParams.quota ? 10 * 60 * 1000 : 0); int m = t.millis(); if (m > serverGlobalParams.slowMS) { log() << "dbeval slow, time: " << dec << m << "ms " << dbName << endl; if ( m >= 1000 ) log() << code << endl; else OCCASIONALLY log() << code << endl; } } if (res || s->isLastRetNativeCode()) { result.append("errno", (double) res); errmsg = "invoke failed: "; if (s->isLastRetNativeCode()) errmsg += "cannot return native function"; else errmsg += s->getError(); return false; } s->append( result , "retval" , "__returnValue" ); return true; }
/** * This will verify that all updated fields are * 1.) Valid for storage (checking parent to make sure things like DBRefs are valid) * 2.) Compare updated immutable fields do not change values * * If updateFields is empty then it was replacement and/or we need to check all fields */ inline Status validate(const BSONObj& original, const FieldRefSet& updatedFields, const mb::Document& updated, const std::vector<FieldRef*>* immutableAndSingleValueFields, const ModifierInterface::Options& opts) { LOG(3) << "update validate options -- " << " updatedFields: " << updatedFields << " immutableAndSingleValueFields.size:" << (immutableAndSingleValueFields ? immutableAndSingleValueFields->size() : 0) << " validate:" << opts.enforceOkForStorage; // 1.) Loop through each updated field and validate for storage // and detect immutable field updates // The set of possibly changed immutable fields -- we will need to check their vals FieldRefSet changedImmutableFields; // Check to see if there were no fields specified or if we are not validating // The case if a range query, or query that didn't result in saved fields if (updatedFields.empty() || !opts.enforceOkForStorage) { if (opts.enforceOkForStorage) { // No specific fields were updated so the whole doc must be checked Status s = storageValid(updated, true); if (!s.isOK()) return s; } // Check all immutable fields if (immutableAndSingleValueFields) changedImmutableFields.fillFrom(*immutableAndSingleValueFields); } else { // TODO: Change impl so we don't need to create a new FieldRefSet // -- move all conflict logic into static function on FieldRefSet? FieldRefSet immutableFieldRef; if (immutableAndSingleValueFields) immutableFieldRef.fillFrom(*immutableAndSingleValueFields); FieldRefSet::const_iterator where = updatedFields.begin(); const FieldRefSet::const_iterator end = updatedFields.end(); for (; where != end; ++where) { const FieldRef& current = **where; // Find the updated field in the updated document. mutablebson::ConstElement newElem = updated.root(); size_t currentPart = 0; while (newElem.ok() && currentPart < current.numParts()) newElem = newElem[current.getPart(currentPart++)]; // newElem might be missing if $unset/$renamed-away if (newElem.ok()) { // Check element, and its children Status s = storageValid(newElem, true); if (!s.isOK()) return s; // Check parents to make sure they are valid as well. s = storageValidParents(newElem); if (!s.isOK()) return s; } // Check if the updated field conflicts with immutable fields immutableFieldRef.findConflicts(¤t, &changedImmutableFields); } } const bool checkIdField = (updatedFields.empty() && !original.isEmpty()) || updatedFields.findConflicts(&idFieldRef, NULL); // Add _id to fields to check since it too is immutable if (checkIdField) changedImmutableFields.keepShortest(&idFieldRef); else if (changedImmutableFields.empty()) { // Return early if nothing changed which is immutable return Status::OK(); } LOG(4) << "Changed immutable fields: " << changedImmutableFields; // 2.) Now compare values of the changed immutable fields (to make sure they haven't) const mutablebson::ConstElement newIdElem = updated.root()[idFieldName]; FieldRefSet::const_iterator where = changedImmutableFields.begin(); const FieldRefSet::const_iterator end = changedImmutableFields.end(); for (; where != end; ++where) { const FieldRef& current = **where; // Find the updated field in the updated document. mutablebson::ConstElement newElem = updated.root(); size_t currentPart = 0; while (newElem.ok() && currentPart < current.numParts()) newElem = newElem[current.getPart(currentPart++)]; if (!newElem.ok()) { if (original.isEmpty()) { // If the _id is missing and not required, then skip this check if (!(current.dottedField() == idFieldName)) return Status(ErrorCodes::NoSuchKey, mongoutils::str::stream() << "After applying the update, the new" << " document was missing the '" << current.dottedField() << "' (required and immutable) field."); } else { if (current.dottedField() != idFieldName) return Status(ErrorCodes::ImmutableField, mongoutils::str::stream() << "After applying the update to the document with " << newIdElem.toString() << ", the '" << current.dottedField() << "' (required and immutable) field was " "found to have been removed --" << original); } } else { // Find the potentially affected field in the original document. const BSONElement oldElem = dps::extractElementAtPath(original, current.dottedField()); const BSONElement oldIdElem = original.getField(idFieldName); // Ensure no arrays since neither _id nor shard keys can be in an array, or one. mb::ConstElement currElem = newElem; while (currElem.ok()) { if (currElem.getType() == Array) { return Status( ErrorCodes::NotSingleValueField, mongoutils::str::stream() << "After applying the update to the document {" << (oldIdElem.ok() ? oldIdElem.toString() : newIdElem.toString()) << " , ...}, the (immutable) field '" << current.dottedField() << "' was found to be an array or array descendant."); } currElem = currElem.parent(); } // If we have both (old and new), compare them. If we just have new we are good if (oldElem.ok() && newElem.compareWithBSONElement(oldElem, nullptr, false) != 0) { return Status(ErrorCodes::ImmutableField, mongoutils::str::stream() << "After applying the update to the document {" << oldElem.toString() << " , ...}, the (immutable) field '" << current.dottedField() << "' was found to have been altered to " << newElem.toString()); } } } return Status::OK(); }
/* **************************************************************************** * * attributeType - * */ static std::string attributeType ( const std::string& tenant, const std::vector<std::string>& servicePathV, const std::string entityType, const std::string attrName ) { std::string idType = std::string("_id.") + ENT_ENTITY_TYPE; std::string idServicePath = std::string("_id.") + ENT_SERVICE_PATH; std::string attributeName = std::string(ENT_ATTRS) + "." + attrName; BSONObj query = BSON(idType << entityType << idServicePath << fillQueryServicePath(servicePathV) << attributeName << BSON("$exists" << true)); std::auto_ptr<DBClientCursor> cursor; DBClientBase* connection = NULL; LM_T(LmtMongo, ("query() in '%s' collection: '%s'", getEntitiesCollectionName(tenant).c_str(), query.toString().c_str())); try { connection = getMongoConnection(); cursor = connection->query(getEntitiesCollectionName(tenant).c_str(), query); /* * We have observed that in some cases of DB errors (e.g. the database daemon is down) instead of * raising an exception, the query() method sets the cursor to NULL. In this case, we raise the * exception ourselves */ if (cursor.get() == NULL) { throw DBException("Null cursor from mongo (details on this is found in the source code)", 0); } releaseMongoConnection(connection); LM_I(("Database Operation Successful (%s)", query.toString().c_str())); } catch (const DBException &e) { releaseMongoConnection(connection); LM_E(("Database Error ('%s', '%s')", query.toString().c_str(), e.what())); return ""; } catch (...) { releaseMongoConnection(connection); LM_E(("Database Error ('%s', '%s')", query.toString().c_str(), "generic exception")); return ""; } while (cursor->more()) { BSONObj r = cursor->next(); LM_T(LmtMongo, ("retrieved document: '%s'", r.toString().c_str())); /* It could happen that different entities within the same entity type may have attributes with the same name * but different types. In that case, one type (at random) is returned. A list could be returned but the * NGSIv2 operations only allow to set one type */ return r.getField(ENT_ATTRS).embeddedObject().getField(attrName).embeddedObject().getStringField(ENT_ATTRS_TYPE); } return ""; }
/* **************************************************************************** * * mongoEntityTypes - */ HttpStatusCode mongoEntityTypes ( EntityTypesResponse* responseP, const std::string& tenant, const std::vector<std::string>& servicePathV, std::map<std::string, std::string>& uriParams ) { unsigned int offset = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str()); unsigned int limit = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str()); std::string detailsString = uriParams[URI_PARAM_PAGINATION_DETAILS]; bool details = (strcasecmp("on", detailsString.c_str()) == 0)? true : false; DBClientBase* connection = NULL; bool reqSemTaken = false; LM_T(LmtMongo, ("Query Entity Types")); LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false")); reqSemTake(__FUNCTION__, "query types request", SemReadOp, &reqSemTaken); /* Compose query based on this aggregation command: * * db.runCommand({aggregate: "entities", * pipeline: [ {$match: { "_id.servicePath": /.../ } }, * {$project: {_id: 1, "attrNames": 1} }, * {$project: { "attrNames" * {$cond: [ {$eq: [ "$attrNames", [ ] ] }, [null], "$attrNames"] } * } * }, * {$unwind: "$attrNames"}, * {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrNames"}} }, * {$sort: {_id: 1} } * ] * }) * * The $cond part is hard... more information at http://stackoverflow.com/questions/27510143/empty-array-prevents-document-to-appear-in-query * As a consequence, some "null" values may appear in the resulting attrs vector, which are pruned by the result processing logic. * * FIXME P6: in the future, we can interpret the collapse parameter at this layer. If collapse=true so we don't need attributes, the * following command can be used: * * db.runCommand({aggregate: "entities", pipeline: [ {$group: {_id: "$_id.type"} }]}) * * * */ BSONObj result; // // Building the projection part of the query that includes types that have no attributes // See bug: https://github.com/telefonicaid/fiware-orion/issues/686 // BSONArrayBuilder emptyArrayBuilder; BSONArrayBuilder nulledArrayBuilder; nulledArrayBuilder.appendNull(); // We are using the $cond: [ .. ] and not the $cond: { .. } one, as the former is the only one valid in MongoDB 2.4 BSONObj projection = BSON( "$project" << BSON( ENT_ATTRNAMES << BSON( "$cond" << BSON_ARRAY( BSON("$eq" << BSON_ARRAY(S_ATTRNAMES << emptyArrayBuilder.arr()) ) << nulledArrayBuilder.arr() << S_ATTRNAMES ) ) ) ); BSONObj cmd = BSON("aggregate" << COL_ENTITIES << "pipeline" << BSON_ARRAY( BSON("$match" << BSON(C_ID_SERVICEPATH << fillQueryServicePath(servicePathV))) << BSON("$project" << BSON("_id" << 1 << ENT_ATTRNAMES << 1)) << projection << BSON("$unwind" << S_ATTRNAMES) << BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRNAMES))) << BSON("$sort" << BSON("_id" << 1)) ) ); LM_T(LmtMongo, ("runCommand() in '%s' database: '%s'", composeDatabaseName(tenant).c_str(), cmd.toString().c_str())); try { connection = getMongoConnection(); connection->runCommand(composeDatabaseName(tenant).c_str(), cmd, result); releaseMongoConnection(connection); LM_I(("Database Operation Successful (%s)", cmd.toString().c_str())); } catch (const DBException& e) { releaseMongoConnection(connection); std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() + " - command: " + cmd.toString() + " - exception: " + e.what(); LM_E(("Database Error (%s)", err.c_str())); responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request", reqSemTaken); return SccOk; } catch (...) { releaseMongoConnection(connection); std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() + " - command: " + cmd.toString() + " - exception: " + "generic"; LM_E(("Database Error (%s)", err.c_str())); responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request", reqSemTaken); return SccOk; } // Processing result to build response LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str())); std::vector<BSONElement> resultsArray = result.getField("result").Array(); if (resultsArray.size() == 0) { responseP->statusCode.fill(SccContextElementNotFound); reqSemGive(__FUNCTION__, "query types request", reqSemTaken); return SccOk; } /* Another strategy to implement pagination is to use the $skip and $limit operators in the * aggregation framework. However, doing so, we don't know the total number of results, which can * be needed in the case of details=on (using that approach, we need to do two queries: one to get * the count and other to get the actual results with $skip and $limit, in the same "transaction" to * avoid incoherence between both if some entity type is created or deleted in the process). * * However, considering that the number of types will be small compared with the number of entities, * the current approach seems to be ok */ for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix) { BSONObj resultItem = resultsArray[ix].embeddedObject(); TypeEntity* entityType = new TypeEntity(resultItem.getStringField("_id")); std::vector<BSONElement> attrsArray = resultItem.getField("attrs").Array(); entityType->count = countEntities(tenant, servicePathV, entityType->type); if (!attrsArray[0].isNull()) { for (unsigned int jx = 0; jx < attrsArray.size(); ++jx) { /* This is where NULL elements in the resulting attrs vector are pruned */ if (attrsArray[jx].isNull()) { continue; } /* Note that we need and extra query() to the database (inside attributeType() function) to get each attribute type. * This could be unefficient, specially if the number of attributes is large */ std::string attrType = attributeType(tenant, servicePathV, entityType->type , attrsArray[jx].str()); ContextAttribute* ca = new ContextAttribute(attrsArray[jx].str(), attrType, ""); entityType->contextAttributeVector.push_back(ca); } } responseP->typeEntityVector.push_back(entityType); } char detailsMsg[256]; if (responseP->typeEntityVector.size() > 0) { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size()); responseP->statusCode.fill(SccOk, detailsMsg); } else { responseP->statusCode.fill(SccOk); } } else { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Number of types: %d. Offset is %d", (int) resultsArray.size(), offset); responseP->statusCode.fill(SccContextElementNotFound, detailsMsg); } else { responseP->statusCode.fill(SccContextElementNotFound); } } reqSemGive(__FUNCTION__, "query types request", reqSemTaken); return SccOk; }
/* **************************************************************************** * * createEntity - */ TEST(mongoNotifyContextRequest, createEntity) { HttpStatusCode ms; NotifyContextRequest req; NotifyContextResponse res; /* Prepare database */ prepareDatabase(); /* Forge the request */ ContextElementResponse cer; req.subscriptionId.set("51307b66f481db11bf860001"); req.originator.set("localhost"); cer.contextElement.entityId.fill("E10", "T10", "false"); ContextAttribute ca("A1", "TA1", "new_val"); cer.contextElement.contextAttributeVector.push_back(&ca); cer.statusCode.fill(SccOk); req.contextElementResponseVector.push_back(&cer); /* Prepare mock */ TimerMock* timerMock = new TimerMock(); ON_CALL(*timerMock, getCurrentTime()) .WillByDefault(Return(1360232700)); setTimer(timerMock); /* Invoke the function in mongoBackend library */ ms = mongoNotifyContext(&req, &res); /* Check response is as expected */ EXPECT_EQ(SccOk, ms); EXPECT_EQ(SccOk, res.responseCode.code); EXPECT_EQ("OK", res.responseCode.reasonPhrase); EXPECT_EQ(0, res.responseCode.details.size()); /* Check that every involved collection at MongoDB is as expected */ /* Note we are using EXPECT_STREQ() for some cases, as Mongo Driver returns const char*, not string * objects (see http://code.google.com/p/googletest/wiki/Primer#String_Comparison) */ DBClientBase* connection = getMongoConnection(); /* entities collection */ BSONObj ent; std::vector<BSONElement> attrs; ASSERT_EQ(6, connection->count(ENTITIES_COLL, BSONObj())); ent = connection->findOne(ENTITIES_COLL, BSON("_id.id" << "E1" << "_id.type" << "T1")); EXPECT_STREQ("E1", C_STR_FIELD(ent.getObjectField("_id"), "id")); EXPECT_STREQ("T1", C_STR_FIELD(ent.getObjectField("_id"), "type")); EXPECT_FALSE(ent.hasField("modDate")); attrs = ent.getField("attrs").Array(); ASSERT_EQ(2, attrs.size()); BSONObj a1 = getAttr(attrs, "A1", "TA1"); BSONObj a2 = getAttr(attrs, "A2", "TA2"); EXPECT_STREQ("A1", C_STR_FIELD(a1, "name")); EXPECT_STREQ("TA1",C_STR_FIELD(a1, "type")); EXPECT_STREQ("val1", C_STR_FIELD(a1, "value")); EXPECT_FALSE(a1.hasField("modDate")); EXPECT_STREQ("A2", C_STR_FIELD(a2, "name")); EXPECT_STREQ("TA2", C_STR_FIELD(a2, "type")); EXPECT_FALSE(a2.hasField("value")); EXPECT_FALSE(a2.hasField("modDate")); ent = connection->findOne(ENTITIES_COLL, BSON("_id.id" << "E2" << "_id.type" << "T2")); EXPECT_STREQ("E2", C_STR_FIELD(ent.getObjectField("_id"), "id")); EXPECT_STREQ("T2", C_STR_FIELD(ent.getObjectField("_id"), "type")); EXPECT_FALSE(ent.hasField("modDate")); attrs = ent.getField("attrs").Array(); ASSERT_EQ(2, attrs.size()); BSONObj a3 = getAttr(attrs, "A3", "TA3"); BSONObj a4 = getAttr(attrs, "A4", "TA4"); EXPECT_STREQ("A3", C_STR_FIELD(a3, "name")); EXPECT_STREQ("TA3", C_STR_FIELD(a3, "type")); EXPECT_STREQ("val3", C_STR_FIELD(a3, "value")); EXPECT_FALSE(a3.hasField("modDate")); EXPECT_STREQ("A4", C_STR_FIELD(a4, "name")); EXPECT_STREQ("TA4", C_STR_FIELD(a4, "type")); EXPECT_FALSE(a4.hasField("value")); EXPECT_FALSE(a4.hasField("modDate")); ent = connection->findOne(ENTITIES_COLL, BSON("_id.id" << "E3" << "_id.type" << "T3")); EXPECT_STREQ("E3", C_STR_FIELD(ent.getObjectField("_id"), "id")); EXPECT_STREQ("T3", C_STR_FIELD(ent.getObjectField("_id"), "type")); EXPECT_FALSE(ent.hasField("modDate")); attrs = ent.getField("attrs").Array(); ASSERT_EQ(2, attrs.size()); BSONObj a5 = getAttr(attrs, "A5", "TA5"); BSONObj a6 = getAttr(attrs, "A6", "TA6"); EXPECT_STREQ("A5", C_STR_FIELD(a5, "name")); EXPECT_STREQ("TA5", C_STR_FIELD(a5, "type")); EXPECT_STREQ("val5", C_STR_FIELD(a5, "value")); EXPECT_FALSE(a5.hasField("modDate")); EXPECT_STREQ("A6", C_STR_FIELD(a6, "name")); EXPECT_STREQ("TA6", C_STR_FIELD(a6, "type")); EXPECT_FALSE(a6.hasField("value")); EXPECT_FALSE(a6.hasField("modDate")); ent = connection->findOne(ENTITIES_COLL, BSON("_id.id" << "E1" << "_id.type" << "T1bis")); EXPECT_STREQ("E1", C_STR_FIELD(ent.getObjectField("_id"), "id")); EXPECT_STREQ("T1bis", C_STR_FIELD(ent.getObjectField("_id"), "type")); EXPECT_FALSE(ent.hasField("modDate")); attrs = ent.getField("attrs").Array(); ASSERT_EQ(1, attrs.size()); a1 = getAttr(attrs, "A1", "TA1"); EXPECT_STREQ("A1", C_STR_FIELD(a1, "name")); EXPECT_STREQ("TA1",C_STR_FIELD(a1, "type")); EXPECT_STREQ("val1bis2", C_STR_FIELD(a1, "value")); EXPECT_FALSE(a1.hasField("modDate")); ent = connection->findOne(ENTITIES_COLL, BSON("_id.id" << "E10" << "_id.type" << "T10")); EXPECT_STREQ("E10", C_STR_FIELD(ent.getObjectField("_id"), "id")); EXPECT_STREQ("T10", C_STR_FIELD(ent.getObjectField("_id"), "type")); EXPECT_TRUE(ent.hasField("creDate")); EXPECT_TRUE(ent.hasField("modDate")); attrs = ent.getField("attrs").Array(); ASSERT_EQ(1, attrs.size()); a1 = getAttr(attrs, "A1", "TA1"); EXPECT_STREQ("A1", C_STR_FIELD(a1, "name")); EXPECT_STREQ("TA1",C_STR_FIELD(a1, "type")); EXPECT_STREQ("new_val", C_STR_FIELD(a1, "value")); EXPECT_TRUE(a1.hasField("creDate")); EXPECT_TRUE(a1.hasField("modDate")); /* Note "_id.type: {$exists: false}" is a way for querying for entities without type */ ent = connection->findOne(ENTITIES_COLL, BSON("_id.id" << "E1" << "_id.type" << BSON("$exists" << false))); EXPECT_STREQ("E1", C_STR_FIELD(ent.getObjectField("_id"), "id")); EXPECT_FALSE(ent.getObjectField("_id").hasField("type")); EXPECT_FALSE(ent.hasField("modDate")); attrs = ent.getField("attrs").Array(); ASSERT_EQ(2, attrs.size()); a1 = getAttr(attrs, "A1", "TA1"); a2 = getAttr(attrs, "A2", "TA2"); EXPECT_STREQ("A1", C_STR_FIELD(a1, "name")); EXPECT_STREQ("TA1",C_STR_FIELD(a1, "type")); EXPECT_STREQ("val1-nt", C_STR_FIELD(a1, "value")); EXPECT_FALSE(a1.hasField("modDate")); EXPECT_STREQ("A2", C_STR_FIELD(a2, "name")); EXPECT_STREQ("TA2", C_STR_FIELD(a2, "type")); EXPECT_FALSE(a2.hasField("value")); EXPECT_FALSE(a2.hasField("modDate")); /* Release connection */ mongoDisconnect(); /* Release mock */ delete timerMock; }
/* **************************************************************************** * * SubscriptionCache::insert - */ void SubscriptionCache::insert(const std::string& tenant, BSONObj bobj) { BSONElement idField = bobj.getField("_id"); if (idField.eoo() == true) { LM_E(("Database Error (error retrieving _id field in doc: '%s')", bobj.toString().c_str())); return; } // // 01. Extract values from database object 'bobj' // std::string subId = idField.OID().toString(); int64_t expiration = bobj.hasField(CSUB_EXPIRATION)? bobj.getField(CSUB_EXPIRATION).Long() : 0; std::string reference = bobj.getField(CSUB_REFERENCE).String(); int64_t throttling = bobj.hasField(CSUB_THROTTLING)? bobj.getField(CSUB_THROTTLING).Long() : -1; std::vector<BSONElement> eVec = bobj.getField(CSUB_ENTITIES).Array(); std::vector<BSONElement> attrVec = bobj.getField(CSUB_ATTRS).Array(); std::vector<BSONElement> condVec = bobj.getField(CSUB_CONDITIONS).Array(); std::string formatString = bobj.hasField(CSUB_FORMAT)? bobj.getField(CSUB_FORMAT).String() : "XML"; std::string servicePath = bobj.hasField(CSUB_SERVICE_PATH)? bobj.getField(CSUB_SERVICE_PATH).String() : "/"; Format format = stringToFormat(formatString); int lastNotification = bobj.hasField(CSUB_LASTNOTIFICATION)? bobj.getField(CSUB_LASTNOTIFICATION).Int() : 0; std::vector<EntityInfo*> eiV; std::vector<std::string> attrV; Restriction restriction; NotifyConditionVector notifyConditionVector; // // 02. Push Entity-data names to EntityInfo Vector (eiV) // for (unsigned int ix = 0; ix < eVec.size(); ++ix) { BSONObj entity = eVec[ix].embeddedObject(); if (!entity.hasField(CSUB_ENTITY_ID)) { LM_W(("Runtime Error (got a subscription without id)")); continue; } std::string id = entity.getStringField(ENT_ENTITY_ID); if (!entity.hasField(CSUB_ENTITY_ISPATTERN)) { continue; } std::string isPattern = entity.getStringField(CSUB_ENTITY_ISPATTERN); if (isPattern != "true") { continue; } std::string type = ""; if (entity.hasField(CSUB_ENTITY_TYPE)) { type = entity.getStringField(CSUB_ENTITY_TYPE); } EntityInfo* eiP = new EntityInfo(id, type); eiV.push_back(eiP); } if (eiV.size() == 0) { return; } // // 03. Push attribute names to Attribute Vector (attrV) // for (unsigned int ix = 0; ix < attrVec.size(); ++ix) { std::string attributeName = attrVec[ix].String(); attrV.push_back(attributeName); } // // 04. FIXME P4: Restriction not implemented // // // 05. Fill in notifyConditionVector from condVec // for (unsigned int ix = 0; ix < condVec.size(); ++ix) { BSONObj condition = condVec[ix].embeddedObject(); std::string condType; std::vector<BSONElement> valueVec; condType = condition.getStringField(CSUB_CONDITIONS_TYPE); if (condType != "ONCHANGE") { continue; } NotifyCondition* ncP = new NotifyCondition(); ncP->type = condType; valueVec = condition.getField(CSUB_CONDITIONS_VALUE).Array(); for (unsigned int vIx = 0; vIx < valueVec.size(); ++vIx) { std::string condValue; condValue = valueVec[vIx].String(); ncP->condValueList.push_back(condValue); } notifyConditionVector.push_back(ncP); } if (notifyConditionVector.size() == 0) { for (unsigned int ix = 0; ix < eiV.size(); ++ix) { delete(eiV[ix]); } eiV.clear(); restriction.release(); return; } // // 06. Create Subscription and add it to the subscription-cache // Subscription* subP = new Subscription(tenant, servicePath, subId, eiV, attrV, throttling, expiration, restriction, notifyConditionVector, reference, lastNotification, format); subCache->insert(subP); notifyConditionVector.release(); // Subscription constructor makes a copy }
// PD_TRACE_DECLARE_FUNCTION ( SDB_RTNQUERY, "rtnQuery" ) INT32 rtnQuery ( const CHAR *pCollectionName, const BSONObj &selector, const BSONObj &matcher, const BSONObj &orderBy, const BSONObj &hint, SINT32 flags, pmdEDUCB *cb, SINT64 numToSkip, SINT64 numToReturn, SDB_DMSCB *dmsCB, SDB_RTNCB *rtnCB, SINT64 &contextID, rtnContextBase **ppContext, BOOLEAN enablePrefetch ) { INT32 rc = SDB_OK ; PD_TRACE_ENTRY ( SDB_RTNQUERY ) ; dmsStorageUnitID suID = DMS_INVALID_CS ; contextID = -1 ; SDB_ASSERT ( pCollectionName, "collection name can't be NULL" ) ; SDB_ASSERT ( cb, "educb can't be NULL" ) ; SDB_ASSERT ( dmsCB, "dmsCB can't be NULL" ) ; SDB_ASSERT ( rtnCB, "rtnCB can't be NULL" ) ; dmsStorageUnit *su = NULL ; dmsMBContext *mbContext = NULL ; rtnContextData *dataContext = NULL ; const CHAR *pCollectionShortName = NULL ; rtnAccessPlanManager *apm = NULL ; optAccessPlan *plan = NULL ; BSONObj hintTmp = hint ; BSONObj blockObj ; BSONObj *pBlockObj = NULL ; const CHAR *indexName = NULL ; const CHAR *scanType = NULL ; INT32 indexLID = DMS_INVALID_EXTENT ; INT32 direction = 0 ; if ( FLG_QUERY_EXPLAIN & flags ) { rc = rtnExplain( pCollectionName, selector, matcher, orderBy, hint, flags, numToSkip, numToReturn, cb, dmsCB, rtnCB, contextID, ppContext ) ; if ( SDB_OK != rc ) { PD_LOG( PDERROR, "failed to explain query:%d", rc ) ; goto error ; } else { goto done ; } } rc = rtnResolveCollectionNameAndLock ( pCollectionName, dmsCB, &su, &pCollectionShortName, suID ) ; PD_RC_CHECK( rc, PDERROR, "Failed to resolve collection name %s", pCollectionName ) ; rc = su->data()->getMBContext( &mbContext, pCollectionShortName, -1 ) ; PD_RC_CHECK( rc, PDERROR, "Failed to get dms mb context, rc: %d", rc ) ; rc = rtnCB->contextNew ( ( flags & FLG_QUERY_PARALLED ) ? RTN_CONTEXT_PARADATA : RTN_CONTEXT_DATA, (rtnContext**)&dataContext, contextID, cb ) ; PD_RC_CHECK( rc, PDERROR, "Failed to create new data context" ) ; if ( Object == hint.getField( FIELD_NAME_META ).type() ) { BSONObjBuilder build ; rc = _rtnParseQueryMeta( hint.getField( FIELD_NAME_META ).embeddedObject(), scanType, indexName, indexLID, direction, blockObj ) ; PD_RC_CHECK( rc, PDERROR, "Failed to parase query meta[%s], rc: %d", hint.toString().c_str(), rc ) ; pBlockObj = &blockObj ; if ( indexName ) { build.append( "", indexName ) ; } else { build.appendNull( "" ) ; } hintTmp = build.obj () ; } apm = su->getAPM() ; SDB_ASSERT ( apm, "apm shouldn't be NULL" ) ; rc = apm->getPlan ( matcher, orderBy, // orderBy hintTmp, // hint pCollectionShortName, &plan ) ; if ( rc ) { PD_LOG ( PDERROR, "Failed to get access plan for %s, context %lld, " "rc: %d", pCollectionName, contextID, rc ) ; goto error ; } else if ( ( flags & FLG_QUERY_FORCE_HINT ) && !hintTmp.isEmpty() && plan->isHintFailed() ) { PD_LOG( PDERROR, "Query used force hint[%s] failed", hintTmp.toString().c_str() ) ; rc = SDB_RTN_INVALID_HINT ; goto error ; } if ( pBlockObj ) { if ( !indexName && TBSCAN != plan->getScanType() ) { PD_LOG( PDERROR, "Scan type[%d] must be TBSCAN", plan->getScanType() ) ; rc = SDB_SYS ; goto error ; } else if ( indexName && ( IXSCAN != plan->getScanType() || indexLID != plan->getIndexLID() ) ) { PD_LOG( PDERROR, "Scan type[%d] error or indexLID[%d] is the " "same with [%d]", plan->getScanType(), plan->getIndexLID(), indexLID ) ; rc = SDB_IXM_NOTEXIST ; goto error ; } } if ( flags & FLG_QUERY_STRINGOUT ) { dataContext->getSelector().setStringOutput( TRUE ) ; } rc = dataContext->open( su, mbContext, plan, cb, selector, plan->sortRequired() ? -1 : numToReturn, plan->sortRequired() ? 0 : numToSkip, pBlockObj, direction ) ; PD_RC_CHECK( rc, PDERROR, "Open data context failed, rc: %d", rc ) ; suID = DMS_INVALID_CS ; plan = NULL ; mbContext = NULL ; if ( cb->getMonConfigCB()->timestampON ) { dataContext->getMonCB()->recordStartTimestamp() ; } if ( dataContext->getPlan()->sortRequired() ) { rc = rtnSort ( (rtnContext**)&dataContext, orderBy, cb, numToSkip, numToReturn, rtnCB, contextID ) ; PD_RC_CHECK( rc, PDERROR, "Failed to sort, rc: %d", rc ) ; } if ( ppContext ) { *ppContext = dataContext ; } if ( enablePrefetch ) { dataContext->enablePrefetch ( cb ) ; } done : PD_TRACE_EXITRC ( SDB_RTNQUERY, rc ) ; return rc ; error : if ( su && mbContext ) { su->data()->releaseMBContext( mbContext ) ; } if ( plan ) { plan->release() ; } if ( DMS_INVALID_CS != suID ) { dmsCB->suUnlock( suID ) ; } if ( -1 != contextID ) { rtnCB->contextDelete ( contextID, cb ) ; contextID = -1 ; } goto done ; }
bool wrappedRun(OperationContext* txn, const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& anObjBuilder) { const std::string coll = jsobj.firstElement().valuestrsafe(); if (coll.empty()) { errmsg = "no collection name specified"; return false; } const std::string toDeleteNs = dbname + '.' + coll; if (!serverGlobalParams.quiet) { LOG(0) << "CMD: dropIndexes " << toDeleteNs << endl; } Client::Context ctx(txn, toDeleteNs); Database* db = ctx.db(); Collection* collection = db->getCollection( txn, toDeleteNs ); if ( ! collection ) { errmsg = "ns not found"; return false; } stopIndexBuilds(txn, db, jsobj); IndexCatalog* indexCatalog = collection->getIndexCatalog(); anObjBuilder.appendNumber("nIndexesWas", indexCatalog->numIndexesTotal(txn) ); BSONElement f = jsobj.getField("index"); if ( f.type() == String ) { string indexToDelete = f.valuestr(); if ( indexToDelete == "*" ) { Status s = indexCatalog->dropAllIndexes(txn, false); if ( !s.isOK() ) { appendCommandStatus( anObjBuilder, s ); return false; } anObjBuilder.append("msg", "non-_id indexes dropped for collection"); return true; } IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByName( txn, indexToDelete ); if ( desc == NULL ) { errmsg = str::stream() << "index not found with name [" << indexToDelete << "]"; return false; } if ( desc->isIdIndex() ) { errmsg = "cannot drop _id index"; return false; } Status s = indexCatalog->dropIndex(txn, desc); if ( !s.isOK() ) { appendCommandStatus( anObjBuilder, s ); return false; } return true; } if ( f.type() == Object ) { IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByKeyPattern( txn, f.embeddedObject() ); if ( desc == NULL ) { errmsg = "can't find index with key:"; errmsg += f.embeddedObject().toString(); return false; } if ( desc->isIdIndex() ) { errmsg = "cannot drop _id index"; return false; } Status s = indexCatalog->dropIndex(txn, desc); if ( !s.isOK() ) { appendCommandStatus( anObjBuilder, s ); return false; } return true; } errmsg = "invalid index name spec"; return false; }
Status ModifierCurrentDate::init(const BSONElement& modExpr, const Options& opts, bool* positional) { _updatePath.parse(modExpr.fieldName()); Status status = fieldchecker::isUpdatable(_updatePath); if (!status.isOK()) { return status; } // If a $-positional operator was used, get the index in which it occurred // and ensure only one occurrence. size_t foundCount; bool foundDollar = fieldchecker::isPositional(_updatePath, &_pathReplacementPosition, &foundCount); if (positional) *positional = foundDollar; if (foundDollar && foundCount > 1) { return Status(ErrorCodes::BadValue, str::stream() << "Too many positional (i.e. '$') elements found in path '" << _updatePath.dottedField() << "'"); } // Validate and store the type to produce switch (modExpr.type()) { case Bool: _typeIsDate = true; break; case Object: { const BSONObj argObj = modExpr.embeddedObject(); const BSONElement typeElem = argObj.getField(kType); bool badInput = typeElem.eoo() || !(typeElem.type() == String); if (!badInput) { std::string typeVal = typeElem.String(); badInput = !(typeElem.String() == kDate || typeElem.String() == kTimestamp); if (!badInput) _typeIsDate = (typeVal == kDate); if (!badInput) { // Check to make sure only the $type field was given as an arg BSONObjIterator i(argObj); const bool onlyHasTypeField = ((i.next().fieldNameStringData() == kType) && i.next().eoo()); if (!onlyHasTypeField) { return Status(ErrorCodes::BadValue, str::stream() << "The only valid field of the option is '$type': " "{$currentDate: {field : {$type: 'date/timestamp'}}}; " << "arg: " << argObj); } } } if (badInput) { return Status(ErrorCodes::BadValue, "The '$type' string field is required " "to be 'date' or 'timestamp': " "{$currentDate: {field : {$type: 'date'}}}"); } break; } default: return Status(ErrorCodes::BadValue, str::stream() << typeName(modExpr.type()) << " is not valid type for $currentDate." " Please use a boolean ('true')" " or a $type expression ({$type: 'timestamp/date'})."); } return Status::OK(); }
/* **************************************************************************** * * mongoAttributesForEntityType - */ HttpStatusCode mongoAttributesForEntityType ( std::string entityType, EntityTypeAttributesResponse* responseP, const std::string& tenant, const std::vector<std::string>& servicePathV, std::map<std::string, std::string>& uriParams ) { unsigned int offset = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str()); unsigned int limit = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str()); std::string detailsString = uriParams[URI_PARAM_PAGINATION_DETAILS]; bool details = (strcasecmp("on", detailsString.c_str()) == 0)? true : false; // Setting the name of the entity type for the response responseP->entityType.type = entityType; LM_T(LmtMongo, ("Query Types Attribute for <%s>", entityType.c_str())); LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false")); reqSemTake(__FUNCTION__, "query types attributes request"); DBClientBase* connection = getMongoConnection(); /* Compose query based on this aggregation command: * * FIXME P9: taking into account that type is no longer used as part of the attribute "key", not sure if the * aggregation query below is fully correct * * db.runCommand({aggregate: "entities", * pipeline: [ {$match: { "_id.type": "TYPE" } }, * {$project: {_id: 1, "attrs.name": 1, "attrs.type": 1} }, * {$unwind: "$attrs"}, * {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrs"}} }, * {$unwind: "$attrs"}, * {$group: {_id: "$attrs" }}, * {$sort: {_id.name: 1, _id.type: 1} } * ] * }) * */ BSONObj result; BSONObj cmd = BSON("aggregate" << COL_ENTITIES << "pipeline" << BSON_ARRAY( BSON("$match" << BSON(C_ID_ENTITY << entityType)) << BSON("$project" << BSON("_id" << 1 << C_ATTR_NAME << 1 << C_ATTR_TYPE << 1)) << BSON("$unwind" << S_ATTRS) << BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRS))) << BSON("$unwind" << S_ATTRS) << BSON("$group" << BSON("_id" << S_ATTRS)) << BSON("$sort" << BSON(C_ID_NAME << 1 << C_ID_TYPE << 1)) ) ); LM_T(LmtMongo, ("runCommand() in '%s' database: '%s'", composeDatabaseName(tenant).c_str(), cmd.toString().c_str())); mongoSemTake(__FUNCTION__, "aggregation command"); try { connection->runCommand(composeDatabaseName(tenant).c_str(), cmd, result); mongoSemGive(__FUNCTION__, "aggregation command"); LM_I(("Database Operation Successful (%s)", cmd.toString().c_str())); } catch (const DBException& e) { mongoSemGive(__FUNCTION__, "aggregation command"); std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() + " - command: " + cmd.toString() + " - exception: " + e.what(); LM_E(("Database Error (%s)", err.c_str())); responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request"); return SccOk; } catch (...) { mongoSemGive(__FUNCTION__, "aggregation command"); std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() + " - command: " + cmd.toString() + " - exception: " + "generic"; LM_E(("Database Error (%s)", err.c_str())); responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request"); return SccOk; } /* Processing result to build response*/ LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str())); std::vector<BSONElement> resultsArray = result.getField("result").Array(); /* See comment above in the other method regarding this strategy to implement pagination */ for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix) { BSONElement idField = resultsArray[ix].embeddedObject().getField("_id"); // // BSONElement::eoo returns true if 'not found', i.e. the field "_id" doesn't exist in 'sub' // // Now, if 'resultsArray[ix].embeddedObject().getField("_id")' is not found, if we continue, // calling embeddedObject() on it, then we get an exception and the broker crashes. // if (idField.eoo() == true) { LM_E(("Database Error (error retrieving _id field in doc: %s)", resultsArray[ix].embeddedObject().toString().c_str())); continue; } BSONObj resultItem = idField.embeddedObject(); ContextAttribute* ca = new ContextAttribute(resultItem.getStringField(ENT_ATTRS_NAME), resultItem.getStringField(ENT_ATTRS_TYPE)); responseP->entityType.contextAttributeVector.push_back(ca); } char detailsMsg[256]; if (responseP->entityType.contextAttributeVector.size() > 0) { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size()); responseP->statusCode.fill(SccOk, detailsMsg); } else { responseP->statusCode.fill(SccOk); } } else { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Number of attributes: %d. Offset is %d", (int) resultsArray.size(), offset); responseP->statusCode.fill(SccContextElementNotFound, detailsMsg); } else { responseP->statusCode.fill(SccContextElementNotFound); } } reqSemGive(__FUNCTION__, "query types request"); return SccOk; }
Status ProjectionExec::transform(const BSONObj& in, BSONObjBuilder* bob, const MatchDetails* details) const { const ArrayOpType& arrayOpType = _arrayOpType; BSONObjIterator it(in); while (it.more()) { BSONElement elt = it.next(); // Case 1: _id if (mongoutils::str::equals("_id", elt.fieldName())) { if (_includeID) { bob->append(elt); } continue; } // Case 2: no array projection for this field. Matchers::const_iterator matcher = _matchers.find(elt.fieldName()); if (_matchers.end() == matcher) { Status s = append(bob, elt, details, arrayOpType); if (!s.isOK()) { return s; } continue; } // Case 3: field has array projection with $elemMatch specified. if (ARRAY_OP_ELEM_MATCH != arrayOpType) { return Status(ErrorCodes::BadValue, "Matchers are only supported for $elemMatch"); } MatchDetails arrayDetails; arrayDetails.requestElemMatchKey(); if (matcher->second->matchesBSON(in, &arrayDetails)) { FieldMap::const_iterator fieldIt = _fields.find(elt.fieldName()); if (_fields.end() == fieldIt) { return Status(ErrorCodes::BadValue, "$elemMatch specified, but projection field not found."); } BSONArrayBuilder arrBuilder; BSONObjBuilder subBob; if (in.getField(elt.fieldName()).eoo()) { return Status(ErrorCodes::InternalError, "$elemMatch called on document element with eoo"); } if (in.getField(elt.fieldName()).Obj().getField(arrayDetails.elemMatchKey()).eoo()) { return Status(ErrorCodes::InternalError, "$elemMatch called on array element with eoo"); } arrBuilder.append( in.getField(elt.fieldName()).Obj().getField(arrayDetails.elemMatchKey())); subBob.appendArray(matcher->first, arrBuilder.arr()); Status status = append(bob, subBob.done().firstElement(), details, arrayOpType); if (!status.isOK()) { return status; } } } return Status::OK(); }
/* **************************************************************************** * * mongoEntityTypes - */ HttpStatusCode mongoEntityTypes ( EntityTypesResponse* responseP, const std::string& tenant, const std::vector<std::string>& servicePathV, std::map<std::string, std::string>& uriParams ) { unsigned int offset = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str()); unsigned int limit = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str()); std::string detailsString = uriParams[URI_PARAM_PAGINATION_DETAILS]; bool details = (strcasecmp("on", detailsString.c_str()) == 0)? true : false; LM_T(LmtMongo, ("Query Entity Types")); LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false")); reqSemTake(__FUNCTION__, "query types request"); DBClientBase* connection = getMongoConnection(); /* Compose query based on this aggregation command: * * FIXME P9: taking into account that type is no longer used as part of the attribute "key", not sure if the * aggregation query below is fully correct * * db.runCommand({aggregate: "entities", * pipeline: [ {$project: {_id: 1, "attrs.name": 1, "attrs.type": 1} }, * {$unwind: "$attrs"}, * {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrs"}} }, * {$sort: {_id: 1} } * ] * }) * * FIXME P6: in the future, we can interpret the collapse parameter at this layer. If collapse=true so we don't need attributes, the * following command can be used: * * db.runCommand({aggregate: "entities", pipeline: [ {$group: {_id: "$_id.type"} }]}) * */ BSONObj result; BSONObj cmd = BSON("aggregate" << COL_ENTITIES << "pipeline" << BSON_ARRAY( BSON("$project" << BSON("_id" << 1 << C_ATTR_NAME << 1 << C_ATTR_TYPE << 1)) << BSON("$unwind" << S_ATTRS) << BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRS))) << BSON("$sort" << BSON("_id" << 1)) ) ); LM_T(LmtMongo, ("runCommand() in '%s' database: '%s'", composeDatabaseName(tenant).c_str(), cmd.toString().c_str())); mongoSemTake(__FUNCTION__, "aggregation command"); try { connection->runCommand(composeDatabaseName(tenant).c_str(), cmd, result); mongoSemGive(__FUNCTION__, "aggregation command"); LM_I(("Database Operation Successful (%s)", cmd.toString().c_str())); } catch (const DBException& e) { mongoSemGive(__FUNCTION__, "aggregation command"); std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() + " - command: " + cmd.toString() + " - exception: " + e.what(); LM_E(("Database Error (%s)", err.c_str())); responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request"); return SccOk; } catch (...) { mongoSemGive(__FUNCTION__, "aggregation command"); std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() + " - command: " + cmd.toString() + " - exception: " + "generic"; LM_E(("Database Error (%s)", err.c_str())); responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request"); return SccOk; } /* Processing result to build response*/ LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str())); std::vector<BSONElement> resultsArray = result.getField("result").Array(); /* Another strategy to implement pagination is to use the $skip and $limit operators in the * aggregation framework. However, doing so, we don't know the total number of results, which can * be needed in the case of details=on (using that approach, we need to do two queries: one to get * the count and other to get the actual results with $skip and $limit, in the same "transaction" to * avoid incoherence between both if some entity type is created or deleted in the process). * * However, considering that the number of types will be small compared with the number of entities, * the current approach seems to be ok */ for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix) { BSONObj resultItem = resultsArray[ix].embeddedObject(); TypeEntity* type = new TypeEntity(resultItem.getStringField("_id")); std::vector<BSONElement> attrsArray = resultItem.getField("attrs").Array(); for (unsigned int jx = 0; jx < attrsArray.size(); ++jx) { BSONObj jAttr = attrsArray[jx].embeddedObject(); ContextAttribute* ca = new ContextAttribute(jAttr.getStringField(ENT_ATTRS_NAME), jAttr.getStringField(ENT_ATTRS_TYPE)); type->contextAttributeVector.push_back(ca); } responseP->typeEntityVector.push_back(type); } char detailsMsg[256]; if (responseP->typeEntityVector.size() > 0) { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size()); responseP->statusCode.fill(SccOk, detailsMsg); } else { responseP->statusCode.fill(SccOk); } } else { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Number of types: %d. Offset is %d", (int) resultsArray.size(), offset); responseP->statusCode.fill(SccContextElementNotFound, detailsMsg); } else { responseP->statusCode.fill(SccContextElementNotFound); } } reqSemGive(__FUNCTION__, "query types request"); return SccOk; }
void go(const string& db, const string& coll, const Query& query, const boost::filesystem::path& outdir, const string& outFilename) { // Can only provide outFilename if db and coll are provided fassert(17368, outFilename.empty() || (!coll.empty() && !db.empty())); boost::filesystem::create_directories( outdir ); map <string, BSONObj> collectionOptions; multimap <string, BSONObj> indexes; vector <string> collections; // Save indexes for database string ins = db + ".system.indexes"; auto_ptr<DBClientCursor> cursor = conn( true ).query( ins.c_str() , Query() , 0 , 0 , 0 , QueryOption_SlaveOk | QueryOption_NoCursorTimeout ); while ( cursor->more() ) { BSONObj obj = cursor->nextSafe(); const string name = obj.getField( "ns" ).valuestr(); indexes.insert( pair<string, BSONObj> (name, obj.getOwned()) ); } string sns = db + ".system.namespaces"; cursor = conn( true ).query( sns.c_str() , Query() , 0 , 0 , 0 , QueryOption_SlaveOk | QueryOption_NoCursorTimeout ); while ( cursor->more() ) { BSONObj obj = cursor->nextSafe(); const string name = obj.getField( "name" ).valuestr(); if (obj.hasField("options")) { collectionOptions[name] = obj.getField("options").embeddedObject().getOwned(); } // skip namespaces with $ in them only if we don't specify a collection to dump if (coll == "" && name.find(".$") != string::npos) { if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { toolInfoLog() << "\tskipping collection: " << name << std::endl; } continue; } const string filename = name.substr( db.size() + 1 ); //if a particular collections is specified, and it's not this one, skip it if (coll != "" && db + "." + coll != name && coll != name) { continue; } // raise error before writing collection with non-permitted filename chars in the name size_t hasBadChars = name.find_first_of("/\0"); if (hasBadChars != string::npos) { toolError() << "Cannot dump " << name << ". Collection has '/' or null in the collection name." << std::endl; continue; } if (nsToCollectionSubstring(name) == "system.indexes") { // Create system.indexes.bson for compatibility with pre 2.2 mongorestore const string filename = name.substr( db.size() + 1 ); writeCollectionFile( name.c_str() , query, outdir / ( filename + ".bson" ) ); // Don't dump indexes as *.metadata.json continue; } if (nsToCollectionSubstring(name) == "system.users" && !mongoDumpGlobalParams.dumpUsersAndRoles) { continue; } collections.push_back(name); } for (vector<string>::iterator it = collections.begin(); it != collections.end(); ++it) { string name = *it; const string filename = outFilename != "" ? outFilename : name.substr( db.size() + 1 ); writeCollectionFile( name , query, outdir / ( filename + ".bson" ) ); writeMetadataFile( name, outdir / (filename + ".metadata.json"), collectionOptions, indexes); } }
bool ReplSource::handleDuplicateDbName( const BSONObj &op, const char *ns, const char *db ) { if ( dbHolder()._isLoaded( ns, dbpath ) ) { // Database is already present. return true; } BSONElement ts = op.getField( "ts" ); if ( ( ts.type() == Date || ts.type() == Timestamp ) && ___databaseIgnorer.ignoreAt( db, ts.date() ) ) { // Database is ignored due to a previous indication that it is // missing from master after optime "ts". return false; } if ( Database::duplicateUncasedName( false, db, dbpath ).empty() ) { // No duplicate database names are present. return true; } OpTime lastTime; bool dbOk = false; { dbtemprelease release; // We always log an operation after executing it (never before), so // a database list will always be valid as of an oplog entry generated // before it was retrieved. BSONObj last = oplogReader.findOne( this->ns().c_str(), Query().sort( BSON( "$natural" << -1 ) ) ); if ( !last.isEmpty() ) { BSONElement ts = last.getField( "ts" ); massert( 14032, "Invalid 'ts' in remote log", ts.type() == Date || ts.type() == Timestamp ); lastTime = OpTime( ts.date() ); } BSONObj info; bool ok = oplogReader.conn()->runCommand( "admin", BSON( "listDatabases" << 1 ), info ); massert( 14033, "Unable to get database list", ok ); BSONObjIterator i( info.getField( "databases" ).embeddedObject() ); while( i.more() ) { BSONElement e = i.next(); const char * name = e.embeddedObject().getField( "name" ).valuestr(); if ( strcasecmp( name, db ) != 0 ) continue; if ( strcmp( name, db ) == 0 ) { // The db exists on master, still need to check that no conflicts exist there. dbOk = true; continue; } // The master has a db name that conflicts with the requested name. dbOk = false; break; } } if ( !dbOk ) { ___databaseIgnorer.doIgnoreUntilAfter( db, lastTime ); incompleteCloneDbs.erase(db); addDbNextPass.erase(db); return false; } // Check for duplicates again, since we released the lock above. set< string > duplicates; Database::duplicateUncasedName( false, db, dbpath, &duplicates ); // The database is present on the master and no conflicting databases // are present on the master. Drop any local conflicts. for( set< string >::const_iterator i = duplicates.begin(); i != duplicates.end(); ++i ) { ___databaseIgnorer.doIgnoreUntilAfter( *i, lastTime ); incompleteCloneDbs.erase(*i); addDbNextPass.erase(*i); Client::Context ctx(*i); dropDatabase(*i); } massert( 14034, "Duplicate database names present after attempting to delete duplicates", Database::duplicateUncasedName( false, db, dbpath ).empty() ); return true; }
// static StatusWith<unique_ptr<CanonicalQuery>> PlanCacheCommand::canonicalize(OperationContext* opCtx, const string& ns, const BSONObj& cmdObj) { // query - required BSONElement queryElt = cmdObj.getField("query"); if (queryElt.eoo()) { return Status(ErrorCodes::BadValue, "required field query missing"); } if (!queryElt.isABSONObj()) { return Status(ErrorCodes::BadValue, "required field query must be an object"); } if (queryElt.eoo()) { return Status(ErrorCodes::BadValue, "required field query missing"); } BSONObj queryObj = queryElt.Obj(); // sort - optional BSONElement sortElt = cmdObj.getField("sort"); BSONObj sortObj; if (!sortElt.eoo()) { if (!sortElt.isABSONObj()) { return Status(ErrorCodes::BadValue, "optional field sort must be an object"); } sortObj = sortElt.Obj(); } // projection - optional BSONElement projElt = cmdObj.getField("projection"); BSONObj projObj; if (!projElt.eoo()) { if (!projElt.isABSONObj()) { return Status(ErrorCodes::BadValue, "optional field projection must be an object"); } projObj = projElt.Obj(); } // collation - optional BSONObj collationObj; if (auto collationElt = cmdObj["collation"]) { if (!collationElt.isABSONObj()) { return Status(ErrorCodes::BadValue, "optional field collation must be an object"); } collationObj = collationElt.Obj(); if (collationObj.isEmpty()) { return Status(ErrorCodes::BadValue, "optional field collation cannot be an empty object"); } } // Create canonical query const NamespaceString nss(ns); auto qr = stdx::make_unique<QueryRequest>(std::move(nss)); qr->setFilter(queryObj); qr->setSort(sortObj); qr->setProj(projObj); qr->setCollation(collationObj); const ExtensionsCallbackReal extensionsCallback(opCtx, &nss); const boost::intrusive_ptr<ExpressionContext> expCtx; auto statusWithCQ = CanonicalQuery::canonicalize(opCtx, std::move(qr), expCtx, extensionsCallback, MatchExpressionParser::kAllowAllSpecialFeatures); if (!statusWithCQ.isOK()) { return statusWithCQ.getStatus(); } return std::move(statusWithCQ.getValue()); }
bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& anObjBuilder, bool /*fromRepl*/) { BSONElement e = jsobj.firstElement(); const string toDeleteNs = dbname + '.' + e.valuestr(); if (!serverGlobalParams.quiet) { MONGO_TLOG(0) << "CMD: dropIndexes " << toDeleteNs << endl; } Lock::DBWrite dbXLock(dbname); Client::Context ctx(toDeleteNs); Collection* collection = cc().database()->getCollection( toDeleteNs ); if ( ! collection ) { errmsg = "ns not found"; return false; } stopIndexBuilds(cc().database(), jsobj); IndexCatalog* indexCatalog = collection->getIndexCatalog(); anObjBuilder.appendNumber("nIndexesWas", indexCatalog->numIndexesTotal() ); BSONElement f = jsobj.getField("index"); if ( f.type() == String ) { string indexToDelete = f.valuestr(); if ( indexToDelete == "*" ) { Status s = indexCatalog->dropAllIndexes( false ); if ( !s.isOK() ) { appendCommandStatus( anObjBuilder, s ); return false; } anObjBuilder.append("msg", "non-_id indexes dropped for collection"); return true; } IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByName( indexToDelete ); if ( desc == NULL ) { errmsg = str::stream() << "index not found with name [" << indexToDelete << "]"; return false; } if ( desc->isIdIndex() ) { errmsg = "cannot drop _id index"; return false; } Status s = indexCatalog->dropIndex( desc ); if ( !s.isOK() ) { appendCommandStatus( anObjBuilder, s ); return false; } return true; } if ( f.type() == Object ) { IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByKeyPattern( f.embeddedObject() ); if ( desc == NULL ) { errmsg = "can't find index with key:"; errmsg += f.embeddedObject().toString(); return false; } if ( desc->isIdIndex() ) { errmsg = "cannot drop _id index"; return false; } Status s = indexCatalog->dropIndex( desc ); if ( !s.isOK() ) { appendCommandStatus( anObjBuilder, s ); return false; } return true; } errmsg = "invalid index name spec"; return false; }
StatusWith<BalancerSettingsType> BalancerSettingsType::fromBSON(const BSONObj& obj) { BalancerSettingsType settings; { bool stopped; Status status = bsonExtractBooleanFieldWithDefault(obj, kStopped, false, &stopped); if (!status.isOK()) return status; if (stopped) { settings._mode = kOff; } else { std::string modeStr; status = bsonExtractStringFieldWithDefault(obj, kMode, kBalancerModes[kFull], &modeStr); if (!status.isOK()) return status; auto it = std::find(std::begin(kBalancerModes), std::end(kBalancerModes), modeStr); if (it == std::end(kBalancerModes)) { return Status(ErrorCodes::BadValue, "Invalid balancer mode"); } settings._mode = static_cast<BalancerMode>(it - std::begin(kBalancerModes)); } } { BSONElement activeWindowElem; Status status = bsonExtractTypedField(obj, kActiveWindow, Object, &activeWindowElem); if (status.isOK()) { const BSONObj balancingWindowObj = activeWindowElem.Obj(); if (balancingWindowObj.isEmpty()) { return Status(ErrorCodes::BadValue, "activeWindow not specified"); } // Check if both 'start' and 'stop' are present const std::string start = balancingWindowObj.getField("start").str(); const std::string stop = balancingWindowObj.getField("stop").str(); if (start.empty() || stop.empty()) { return Status(ErrorCodes::BadValue, str::stream() << "must specify both start and stop of balancing window: " << balancingWindowObj); } // Check that both 'start' and 'stop' are valid time-of-day boost::posix_time::ptime startTime; boost::posix_time::ptime stopTime; if (!toPointInTime(start, &startTime) || !toPointInTime(stop, &stopTime)) { return Status(ErrorCodes::BadValue, str::stream() << kActiveWindow << " format is " << " { start: \"hh:mm\" , stop: \"hh:mm\" }"); } // Check that start and stop designate different time points if (startTime == stopTime) { return Status(ErrorCodes::BadValue, str::stream() << "start and stop times must be different"); } settings._activeWindowStart = startTime; settings._activeWindowStop = stopTime; } else if (status != ErrorCodes::NoSuchKey) { return status; } } { auto secondaryThrottleStatus = MigrationSecondaryThrottleOptions::createFromBalancerConfig(obj); if (!secondaryThrottleStatus.isOK()) { return secondaryThrottleStatus.getStatus(); } settings._secondaryThrottle = std::move(secondaryThrottleStatus.getValue()); } { bool waitForDelete; Status status = bsonExtractBooleanFieldWithDefault(obj, kWaitForDelete, false, &waitForDelete); if (!status.isOK()) return status; settings._waitForDelete = waitForDelete; } return settings; }
INT32 _omTaskStrategyInfo::fromBSON( const BSONObj &obj ) { INT32 rc = SDB_OK ; BSONElement beField ; BSONObj ipsObj ; beField = obj.getField( OM_REST_FIELD_RULE_ID ) ; if ( !beField.isNumber() ) { PD_LOG( PDERROR, "Field[%s] must be number", beField.toString( TRUE, TRUE ).c_str() ) ; rc = SDB_INVALIDARG ; goto error ; } setID( beField.numberLong() ) ; beField = obj.getField( OM_REST_FIELD_TASK_ID ) ; if ( !beField.isNumber() ) { PD_LOG( PDERROR, "Field[%s] must be number", beField.toString( TRUE, TRUE ).c_str() ) ; rc = SDB_INVALIDARG ; goto error ; } setTaskID( beField.numberLong() ) ; beField = obj.getField( OM_REST_FIELD_TASK_NAME ) ; if ( String != beField.type() ) { PD_LOG( PDERROR, "Field[%s] must be string", beField.toString( TRUE, TRUE ).c_str() ) ; rc = SDB_INVALIDARG ; goto error ; } setTaskName( beField.str() ) ; beField = obj.getField( OM_REST_FIELD_NICE ) ; if ( !beField.isNumber() ) { PD_LOG( PDERROR, "Field[%s] must be number", beField.toString( TRUE, TRUE ).c_str() ) ; rc = SDB_INVALIDARG ; goto error ; } setNice( beField.numberInt() ) ; beField = obj.getField( OM_REST_FIELD_USER_NAME ) ; if ( String != beField.type() ) { PD_LOG( PDERROR, "Field[%s] must be string", beField.toString( TRUE, TRUE ).c_str() ) ; rc = SDB_INVALIDARG ; goto error ; } setUserName( beField.str() ) ; beField = obj.getField( OM_REST_FIELD_IPS ) ; if ( Array != beField.type() ) { PD_LOG( PDERROR, "Field[%s] must be string array", beField.toString( TRUE, TRUE ).c_str() ) ; rc = SDB_INVALIDARG ; goto error ; } else { clearIPSet() ; string tmpStr ; BSONElement e ; BSONObjIterator itr( beField.embeddedObject() ) ; while( itr.more() ) { e = itr.next() ; if ( String != beField.type() ) { PD_LOG( PDERROR, "Field[%s] must be string array", beField.toString( TRUE, TRUE ).c_str() ) ; rc = SDB_INVALIDARG ; goto error ; } tmpStr = e.str() ; if ( !tmpStr.empty() ) { addIP( tmpStr ) ; } } } done: return rc ; error: goto done ; }
/* **************************************************************************** * * mongoUpdateContextAvailabilitySubscription - */ HttpStatusCode mongoUpdateContextAvailabilitySubscription(UpdateContextAvailabilitySubscriptionRequest* requestP, UpdateContextAvailabilitySubscriptionResponse* responseP, Format inFormat, const std::string& tenant) { LM_T(LmtMongo, ("Update Context Subscription")); reqSemTake(__FUNCTION__, "ngsi9 update subscription request"); DBClientConnection* connection = getMongoConnection(); /* Look for document */ BSONObj sub; try { OID id = OID(requestP->subscriptionId.get()); mongoSemTake(__FUNCTION__, "findOne from SubscribeContextAvailabilityCollection"); sub = connection->findOne(getSubscribeContextAvailabilityCollectionName(tenant).c_str(), BSON("_id" << id)); mongoSemGive(__FUNCTION__, "findOne from SubscribeContextAvailabilityCollection"); } catch( const AssertionException &e ) { /* This happens when OID format is wrong */ // FIXME: this checking should be done at parsing stage, without progressing to // mongoBackend. By the moment we can live this here, but we should remove in the future // (old issue #95) mongoSemGive(__FUNCTION__, "findOne from SubscribeContextAvailabilityCollection (mongo assertion exception)"); reqSemGive(__FUNCTION__, "ngsi9 update subscription request (mongo assertion exception)"); responseP->errorCode.fill(SccContextElementNotFound); return SccOk; } catch( const DBException &e ) { mongoSemGive(__FUNCTION__, "findOne from SubscribeContextAvailabilityCollection (mongo db exception)"); reqSemGive(__FUNCTION__, "ngsi9 update subscription request (mongo db exception)"); responseP->errorCode.fill(SccReceiverInternalError, std::string("collection: ") + getSubscribeContextAvailabilityCollectionName(tenant).c_str() + " - findOne() _id: " + requestP->subscriptionId.get() + " - exception: " + e.what()); return SccOk; } catch(...) { mongoSemGive(__FUNCTION__, "findOne from SubscribeContextAvailabilityCollection (mongo generic exception)"); reqSemGive(__FUNCTION__, "ngsi9 update subscription request (mongo generic exception)"); responseP->errorCode.fill(SccReceiverInternalError, std::string("collection: ") + getSubscribeContextAvailabilityCollectionName(tenant).c_str() + " - findOne() _id: " + requestP->subscriptionId.get() + " - exception: " + "generic"); return SccOk; } if (sub.isEmpty()) { responseP->errorCode.fill(SccContextElementNotFound); reqSemGive(__FUNCTION__, "ngsi9 update subscription request (no subscriptions found)"); return SccOk; } /* We start with an empty BSONObjBuilder and process requestP for all the fields that can * be updated. I don't like too much this strategy (I would have preferred to start with * a copy of the original document, then modify as neded, but this doesn't seem to be easy * using the API provide by the Mongo C++ driver) * * FIXME: a better implementation strategy could be doing an findAndModify() query to do the * update, so detecting if the document was not found, instead of using findOne() + update() * with $set operation. One operations to MongoDb. vs two operations. */ BSONObjBuilder newSub; /* Entities (mandatory) */ BSONArrayBuilder entities; for (unsigned int ix = 0; ix < requestP->entityIdVector.size(); ++ix) { EntityId* en = requestP->entityIdVector.get(ix); if (en->type == "") { entities.append(BSON(CASUB_ENTITY_ID << en->id << CASUB_ENTITY_ISPATTERN << en->isPattern)); } else { entities.append(BSON(CASUB_ENTITY_ID << en->id << CASUB_ENTITY_TYPE << en->type << CASUB_ENTITY_ISPATTERN << en->isPattern)); } } newSub.append(CASUB_ENTITIES, entities.arr()); /* Attributes (always taken into account) */ BSONArrayBuilder attrs; for (unsigned int ix = 0; ix < requestP->attributeList.size(); ++ix) { attrs.append(requestP->attributeList.get(ix)); } newSub.append(CASUB_ATTRS, attrs.arr()); /* Duration (optional) */ if (requestP->duration.isEmpty()) { newSub.append(CASUB_EXPIRATION, sub.getField(CASUB_EXPIRATION).numberLong()); } else { long long expiration = getCurrentTime() + requestP->duration.parse(); newSub.append(CASUB_EXPIRATION, expiration); LM_T(LmtMongo, ("New subscription expiration: %l", expiration)); } /* Reference is not updatable, so it is appended directly */ newSub.append(CASUB_REFERENCE, STR_FIELD(sub, CASUB_REFERENCE)); int count = sub.hasField(CASUB_COUNT) ? sub.getIntField(CASUB_COUNT) : 0; /* The hasField check is needed due to lastNotification/count could not be present in the original doc */ if (sub.hasField(CASUB_LASTNOTIFICATION)) { newSub.append(CASUB_LASTNOTIFICATION, sub.getIntField(CASUB_LASTNOTIFICATION)); } if (sub.hasField(CASUB_COUNT)) { newSub.append(CASUB_COUNT, count); } /* Adding format to use in notifications */ newSub.append(CASUB_FORMAT, std::string(formatToString(inFormat))); /* Update document in MongoDB */ BSONObj update = newSub.obj(); LM_T(LmtMongo, ("update() in '%s' collection _id '%s': %s}", getSubscribeContextAvailabilityCollectionName(tenant).c_str(), requestP->subscriptionId.get().c_str(), update.toString().c_str())); try { mongoSemTake(__FUNCTION__, "update in SubscribeContextAvailabilityCollection"); connection->update(getSubscribeContextAvailabilityCollectionName(tenant).c_str(), BSON("_id" << OID(requestP->subscriptionId.get())), update); mongoSemGive(__FUNCTION__, "update in SubscribeContextAvailabilityCollection"); } catch( const DBException &e ) { mongoSemGive(__FUNCTION__, "update in SubscribeContextAvailabilityCollection (mongo db exception)"); reqSemGive(__FUNCTION__, "ngsi9 update subscription request (mongo db exception)"); responseP->errorCode.fill(SccReceiverInternalError, std::string("collection: ") + getSubscribeContextAvailabilityCollectionName(tenant).c_str() + " - update() _id: " + requestP->subscriptionId.get().c_str() + " - update() doc: " + update.toString() + " - exception: " + e.what()); return SccOk; } catch(...) { mongoSemGive(__FUNCTION__, "update in SubscribeContextAvailabilityCollection (mongo generic exception)"); reqSemGive(__FUNCTION__, "ngsi9 update subscription request (mongo generic exception)"); responseP->errorCode.fill(SccReceiverInternalError, std::string("collection: ") + getSubscribeContextAvailabilityCollectionName(tenant).c_str() + " - update() _id: " + requestP->subscriptionId.get().c_str() + " - update() doc: " + update.toString() + " - exception: " + "generic"); return SccOk; } /* Send notifications for matching context registrations */ processAvailabilitySubscription(requestP->entityIdVector, requestP->attributeList, requestP->subscriptionId.get(), STR_FIELD(sub, CASUB_REFERENCE), inFormat, tenant); /* Duration is an optional parameter, it is only added in the case they * was used for update */ if (!requestP->duration.isEmpty()) { responseP->duration = requestP->duration; } responseP->subscriptionId = requestP->subscriptionId; reqSemGive(__FUNCTION__, "ngsi9 update subscription request"); return SccOk; }
// PD_TRACE_DECLARE_FUNCTION ( SDB_RTNCOORDREMOVELOB_EXECUTE, "rtnCoordRemoveLob::execute" ) INT32 rtnCoordRemoveLob::execute( CHAR *pReceiveBuffer, SINT32 packSize, CHAR **ppResultBuffer, pmdEDUCB *cb, MsgOpReply &replyHeader, BSONObj** ppErrorObj ) { INT32 rc = SDB_OK ; PD_TRACE_ENTRY( SDB_RTNCOORDREMOVELOB_EXECUTE ) ; const MsgOpLob *header = NULL ; const MsgHeader *baseHeader = ( const MsgHeader * )pReceiveBuffer ; BSONObj obj ; BSONElement ele ; const CHAR *fullName = NULL ; _rtnCoordLobStream stream ; replyHeader.header.messageLength = sizeof( MsgOpReply ) ; replyHeader.header.opCode = MSG_BS_LOB_CLOSE_RES ; replyHeader.header.requestID = baseHeader->requestID ; replyHeader.header.routeID.value = 0 ; replyHeader.header.TID = baseHeader->TID ; replyHeader.contextID = -1 ; replyHeader.flags = SDB_OK ; replyHeader.numReturned = 0 ; replyHeader.startFrom = 0 ; rc = msgExtractRemoveLobRequest( pReceiveBuffer, &header, obj ) ; if ( SDB_OK != rc ) { PD_LOG( PDERROR, "failed to extract remove msg:%d", rc ) ; goto error ; } ele = obj.getField( FIELD_NAME_COLLECTION ) ; if ( String != ele.type() ) { PD_LOG( PDERROR, "invalid type of field \"collection\":%s", obj.toString( FALSE, TRUE ).c_str() ) ; rc = SDB_SYS ; goto error ; } fullName = ele.valuestr() ; ele = obj.getField( FIELD_NAME_LOB_OID ) ; if ( jstOID != ele.type() ) { PD_LOG( PDERROR, "invalid type of field \"oid\":%s", obj.toString( FALSE, TRUE ).c_str() ) ; rc = SDB_SYS ; goto error ; } rc = stream.open( fullName, ele.__oid(), SDB_LOB_MODE_REMOVE, cb ) ; if ( SDB_OK != rc ) { PD_LOG( PDERROR, "failed to remove lob:%s, rc:%d", ele.__oid().str().c_str(), rc ) ; goto error ; } else { } rc = stream.truncate( 0, cb ) ; if ( SDB_OK != rc ) { PD_LOG( PDERROR, "faield to truncate lob:%d", rc ) ; goto error ; } done: { INT32 rcTmp = SDB_OK ; rcTmp = stream.close( cb ) ; if ( SDB_OK != rcTmp ) { PD_LOG( PDERROR, "failed to remove lob:%d", rcTmp ) ; rc = rc == SDB_OK ? rcTmp : rc ; replyHeader.flags = rc ; } } PD_TRACE_EXITRC( SDB_RTNCOORDREMOVELOB_EXECUTE, rc ) ; return rc ; error: replyHeader.flags = rc ; goto done ; }