namespace mongo {
const BSONObj reverseIDObj = BSON( "_id" << -1 );
BSONObj userReplQuery = fromjson("{\"user\":\"repl\"}");
/* Generally replAuthenticate will only be called within system threads to fully authenticate
* connections to other nodes in the cluster that will be used as part of internal operations.
* If a user-initiated action results in needing to call replAuthenticate, you can call it
* with skipAuthCheck set to false. Only do this if you are certain that the proper auth
* checks have already run to ensure that the user is authorized to do everything that this
* connection will be used for!
*/
bool replAuthenticate(DBClientBase *conn, bool skipAuthCheck) {
if( noauth ) {
return true;
}
if (!skipAuthCheck && !cc().getAuthorizationManager()->hasInternalAuthorization()) {
log() << "replauthenticate: requires internal authorization, failing" << endl;
return false;
}
string u;
string p;
if (internalSecurity.pwd.length() > 0) {
u = internalSecurity.user;
p = internalSecurity.pwd;
}
else {
BSONObj user;
{
StringData ns("local.system.users");
LOCK_REASON(lockReason, "repl: authenticating with local db");
Client::ReadContext ctx(ns, lockReason);
if (!Collection::findOne(ns, userReplQuery, user) ||
// try the first user in local
!Collection::findOne(ns, BSONObj(), user)) {
log() << "replauthenticate: no user in local.system.users to use for authentication\n";
return false;
}
}
u = user.getStringField("user");
p = user.getStringField("pwd");
massert( 10392 , "bad user object? [1]", !u.empty());
massert( 10393 , "bad user object? [2]", !p.empty());
}
string err;
if( !conn->auth("local", u.c_str(), p.c_str(), err, false) ) {
log() << "replauthenticate: can't authenticate to master server, user:"******"local.me", BSONObj(), me) ||
!me.hasField("host") || me["host"].String() != myname) {
// cleaning out local.me requires write
// lock. This is a rare operation, so it should
// be ok
if (!Lock::isWriteLocked("local")) {
throw RetryWithWriteLock();
}
// clean out local.me
deleteObjects("local.me", BSONObj(), false, false);
// repopulate
BSONObjBuilder b;
b.appendOID( "_id" , 0 , true );
b.append( "host", myname );
me = b.obj();
updateObjects("local.me", me, BSONObj(), true, false);
}
transaction.commit(0);
}
bool replHandshake(DBClientConnection *conn) {
BSONObj me;
LOCK_REASON(lockReason, "repl: handshake");
try {
Client::ReadContext ctx("local", lockReason);
getMe(me);
}
catch (RetryWithWriteLock &e) {
Client::WriteContext ctx("local", lockReason);
getMe(me);
}
BSONObjBuilder cmd;
cmd.appendAs( me["_id"] , "handshake" );
if (theReplSet) {
cmd.append("member", theReplSet->selfId());
}
BSONObj res;
bool ok = conn->runCommand( "admin" , cmd.obj() , res );
// ignoring for now on purpose for older versions
LOG(ok ? 1 : 0) << "replHandshake res not: " << ok << " res: " << res << endl;
return true;
}
//number of readers created;
// this happens when the source source changes, a reconfig/network-error or the cursor dies
static Counter64 readersCreatedStats;
static ServerStatusMetricField<Counter64> displayReadersCreated(
"repl.network.readersCreated",
&readersCreatedStats );
OplogReader::OplogReader( bool doHandshake ) :
_doHandshake( doHandshake ) {
_tailingQueryOptions = QueryOption_SlaveOk;
_tailingQueryOptions |= QueryOption_CursorTailable | QueryOption_OplogReplay;
/* TODO: slaveOk maybe shouldn't use? */
_tailingQueryOptions |= QueryOption_AwaitData;
readersCreatedStats.increment();
}
bool OplogReader::commonConnect(const string& hostName, const double default_timeout) {
if( conn() == 0 ) {
_conn = shared_ptr<DBClientConnection>(new DBClientConnection(false,
0,
default_timeout /* tcp timeout */));
string errmsg;
if ( !_conn->connect(hostName.c_str(), errmsg) ||
(!noauth && !replAuthenticate(_conn.get(), true)) ) {
resetConnection();
log() << "repl: " << errmsg << endl;
return false;
}
}
return true;
}
bool OplogReader::connect(const std::string& hostName, const double default_timeout) {
if (conn() != 0) {
return true;
}
if ( ! commonConnect(hostName, default_timeout) ) {
return false;
}
if ( _doHandshake && ! replHandshake(_conn.get() ) ) {
return false;
}
return true;
}
bool OplogReader::connect(const BSONObj& rid, const int from, const string& to) {
if (conn() != 0) {
return true;
}
if (commonConnect(to, default_so_timeout)) {
log() << "handshake between " << from << " and " << to << endl;
return passthroughHandshake(rid, from);
}
return false;
}
bool OplogReader::passthroughHandshake(const BSONObj& rid, const int nextOnChainId) {
BSONObjBuilder cmd;
cmd.appendAs(rid["_id"], "handshake");
if (theReplSet) {
const Member* chainedMember = theReplSet->findById(nextOnChainId);
if (chainedMember != NULL) {
cmd.append("config", chainedMember->config().asBson());
}
}
cmd.append("member", nextOnChainId);
BSONObj res;
return conn()->runCommand("admin", cmd.obj(), res);
}
void OplogReader::tailingQuery(const char *ns, Query& query, const BSONObj* fields ) {
verify( !haveCursor() );
LOG(2) << "repl: " << ns << ".find(" << query.toString() << ')' << endl;
cursor.reset( _conn->query( ns, query, 0, 0, fields, _tailingQueryOptions ).release() );
}
void OplogReader::tailingQueryGTE(const char *ns, GTID gtid, const BSONObj* fields ) {
BSONObjBuilder q;
addGTIDToBSON("$gte", gtid, q);
BSONObjBuilder query;
query.append("_id", q.done());
tailingQuery(ns, Query(query.done()).hint(BSON("_id" << 1)), fields);
}
shared_ptr<DBClientCursor> OplogReader::getRollbackCursor(GTID lastGTID) {
shared_ptr<DBClientCursor> retCursor;
BSONObjBuilder q;
addGTIDToBSON("$lte", lastGTID, q);
BSONObjBuilder query;
query.append("_id", q.done());
retCursor.reset(
_conn->query(rsoplog, Query(query.done()).sort(reverseIDObj), 0, 0, NULL, QueryOption_SlaveOk).release()
);
return retCursor;
}
bool OplogReader::propogateSlaveLocation(GTID lastGTID){
BSONObjBuilder cmd;
cmd.append("updateSlave", 1);
addGTIDToBSON("gtid", lastGTID, cmd);
BSONObj ret;
return _conn->runCommand(
"local",
cmd.done(),
ret
);
}
shared_ptr<DBClientCursor> OplogReader::getOplogRefsCursor(OID &oid) {
shared_ptr<DBClientCursor> retCursor;
// this maps to {_id : {$gt : { oid : oid , seq : 0 }}}
retCursor.reset(_conn->query(rsOplogRefs, QUERY("_id" << BSON("$gt" << BSON("oid" << oid << "seq" << 0)) ).hint(BSON("_id" << 1))).release());
return retCursor;
}
}
namespace mongo {
Query& Query::where(const string &jscode, BSONObj scope) {
/* use where() before sort() and hint() and explain(), else this will assert. */
assert( !obj.hasField("query") );
BSONObjBuilder b;
b.appendElements(obj);
b.appendWhere(jscode, scope);
obj = b.obj();
return *this;
}
Query& Query::sort(const BSONObj& s) {
BSONObjBuilder b;
if( obj.hasElement("query") )
b.appendElements(obj);
else
b.append("query", obj);
b.append("orderby", s);
obj = b.obj();
return *this;
}
Query& Query::hint(BSONObj keyPattern) {
BSONObjBuilder b;
if( obj.hasElement("query") )
b.appendElements(obj);
else
b.append("query", obj);
b.append("$hint", keyPattern);
obj = b.obj();
return *this;
}
Query& Query::explain() {
BSONObjBuilder b;
if( obj.hasElement("query") )
b.appendElements(obj);
else
b.append("query", obj);
b.append("$explain", true);
obj = b.obj();
return *this;
}
bool Query::isComplex() const{
return obj.hasElement( "query" );
}
BSONObj Query::getFilter() const {
if ( ! isComplex() )
return obj;
return obj.getObjectField( "query" );
}
BSONObj Query::getSort() const {
if ( ! isComplex() )
return BSONObj();
return obj.getObjectField( "orderby" );
}
BSONObj Query::getHint() const {
if ( ! isComplex() )
return BSONObj();
return obj.getObjectField( "$hint" );
}
bool Query::isExplain() const {
return isComplex() && obj.getBoolField( "$explain" );
}
string Query::toString() const{
return obj.toString();
}
/* --- dbclientcommands --- */
inline bool DBClientWithCommands::isOk(const BSONObj& o) {
return o.getIntField("ok") == 1;
}
inline bool DBClientWithCommands::runCommand(const string &dbname, const BSONObj& cmd, BSONObj &info) {
string ns = dbname + ".$cmd";
info = findOne(ns, cmd);
return isOk(info);
}
/* note - we build a bson obj here -- for something that is super common like getlasterror you
should have that object prebuilt as that would be faster.
*/
bool DBClientWithCommands::simpleCommand(const string &dbname, BSONObj *info, const string &command) {
BSONObj o;
if ( info == 0 )
info = &o;
BSONObjBuilder b;
b.append(command, 1);
return runCommand(dbname, b.done(), *info);
}
unsigned long long DBClientWithCommands::count(const string &_ns, BSONObj query) {
NamespaceString ns(_ns);
BSONObj cmd = BSON( "count" << ns.coll << "query" << query );
BSONObj res;
if( !runCommand(ns.db.c_str(), cmd, res) )
uasserted(string("count fails:") + res.toString());
return res.getIntField("n");
}
BSONObj getlasterrorcmdobj = fromjson("{getlasterror:1}");
string DBClientWithCommands::getLastError() {
BSONObj info;
runCommand("admin", getlasterrorcmdobj, info);
BSONElement e = info["err"];
if( e.eoo() ) return "";
if( e.type() == Object ) return e.toString();
return e.str();
}
BSONObj getpreverrorcmdobj = fromjson("{getpreverror:1}");
BSONObj DBClientWithCommands::getPrevError() {
BSONObj info;
runCommand("admin", getpreverrorcmdobj, info);
return info;
}
BSONObj getnoncecmdobj = fromjson("{getnonce:1}");
string DBClientWithCommands::createPasswordDigest( const string & username , const string & clearTextPassword ){
md5digest d;
{
md5_state_t st;
md5_init(&st);
md5_append(&st, (const md5_byte_t *) username.data(), username.length());
md5_append(&st, (const md5_byte_t *) ":mongo:", 7 );
md5_append(&st, (const md5_byte_t *) clearTextPassword.data(), clearTextPassword.length());
md5_finish(&st, d);
}
return digestToString( d );
}
bool DBClientWithCommands::auth(const string &dbname, const string &username, const string &password_text, string& errmsg, bool digestPassword) {
//cout << "TEMP AUTH " << toString() << dbname << ' ' << username << ' ' << password_text << ' ' << digestPassword << endl;
string password = password_text;
if( digestPassword )
password = createPasswordDigest( username , password_text );
BSONObj info;
string nonce;
if( !runCommand(dbname, getnoncecmdobj, info) ) {
errmsg = "getnonce fails - connection problem?";
return false;
}
{
BSONElement e = info.getField("nonce");
assert( e.type() == String );
nonce = e.valuestr();
}
BSONObj authCmd;
BSONObjBuilder b;
{
b << "authenticate" << 1 << "nonce" << nonce << "user" << username;
md5digest d;
{
md5_state_t st;
md5_init(&st);
md5_append(&st, (const md5_byte_t *) nonce.c_str(), nonce.size() );
md5_append(&st, (const md5_byte_t *) username.data(), username.length());
md5_append(&st, (const md5_byte_t *) password.c_str(), password.size() );
md5_finish(&st, d);
}
b << "key" << digestToString( d );
authCmd = b.done();
}
if( runCommand(dbname, authCmd, info) )
return true;
errmsg = info.toString();
return false;
}
BSONObj ismastercmdobj = fromjson("{\"ismaster\":1}");
bool DBClientWithCommands::isMaster(bool& isMaster, BSONObj *info) {
BSONObj o;
if ( info == 0 ) info = &o;
bool ok = runCommand("admin", ismastercmdobj, *info);
isMaster = (info->getIntField("ismaster") == 1);
return ok;
}
bool DBClientWithCommands::createCollection(const string &ns, unsigned size, bool capped, int max, BSONObj *info) {
BSONObj o;
if ( info == 0 ) info = &o;
BSONObjBuilder b;
b.append("create", ns);
if ( size ) b.append("size", size);
if ( capped ) b.append("capped", true);
if ( max ) b.append("max", max);
string db = nsToClient(ns.c_str());
return runCommand(db.c_str(), b.done(), *info);
}
bool DBClientWithCommands::copyDatabase(const string &fromdb, const string &todb, const string &fromhost, BSONObj *info) {
BSONObj o;
if ( info == 0 ) info = &o;
BSONObjBuilder b;
b.append("copydb", 1);
b.append("fromhost", fromhost);
b.append("fromdb", fromdb);
b.append("todb", todb);
return runCommand("admin", b.done(), *info);
}
bool DBClientWithCommands::setDbProfilingLevel(const string &dbname, ProfilingLevel level, BSONObj *info ) {
BSONObj o;
if ( info == 0 ) info = &o;
if ( level ) {
// Create system.profile collection. If it already exists this does nothing.
// TODO: move this into the db instead of here so that all
// drivers don't have to do this.
string ns = dbname + ".system.profile";
createCollection(ns.c_str(), 1024 * 1024, true, 0, info);
}
BSONObjBuilder b;
b.append("profile", (int) level);
return runCommand(dbname, b.done(), *info);
}
BSONObj getprofilingcmdobj = fromjson("{\"profile\":-1}");
bool DBClientWithCommands::getDbProfilingLevel(const string &dbname, ProfilingLevel& level, BSONObj *info) {
BSONObj o;
if ( info == 0 ) info = &o;
if ( runCommand(dbname, getprofilingcmdobj, *info) ) {
level = (ProfilingLevel) info->getIntField("was");
return true;
}
return false;
}
bool DBClientWithCommands::eval(const string &dbname, const string &jscode, BSONObj& info, BSONElement& retValue, BSONObj *args) {
BSONObjBuilder b;
b.appendCode("$eval", jscode.c_str());
if ( args )
b.appendArray("args", *args);
bool ok = runCommand(dbname, b.done(), info);
if ( ok )
retValue = info.getField("retval");
return ok;
}
bool DBClientWithCommands::eval(const string &dbname, const string &jscode) {
BSONObj info;
BSONElement retValue;
return eval(dbname, jscode, info, retValue);
}
void testSort() {
DBClientConnection c;
string err;
if ( !c.connect("localhost", err) ) {
out() << "can't connect to server " << err << endl;
return;
}
cout << "findOne returns:" << endl;
cout << c.findOne("test.foo", QUERY( "x" << 3 ) ).toString() << endl;
cout << c.findOne("test.foo", QUERY( "x" << 3 ).sort("name") ).toString() << endl;
}
/* TODO: unit tests should run this? */
void testDbEval() {
DBClientConnection c;
string err;
if ( !c.connect("localhost", err) ) {
out() << "can't connect to server " << err << endl;
return;
}
if( !c.auth("dwight", "u", "p", err) ) {
out() << "can't authenticate " << err << endl;
return;
}
BSONObj info;
BSONElement retValue;
BSONObjBuilder b;
b.append("0", 99);
BSONObj args = b.done();
bool ok = c.eval("dwight", "function() { return args[0]; }", info, retValue, &args);
out() << "eval ok=" << ok << endl;
out() << "retvalue=" << retValue.toString() << endl;
out() << "info=" << info.toString() << endl;
out() << endl;
int x = 3;
assert( c.eval("dwight", "function() { return 3; }", x) );
out() << "***\n";
BSONObj foo = fromjson("{\"x\":7}");
out() << foo.toString() << endl;
int res=0;
ok = c.eval("dwight", "function(parm1) { return parm1.x; }", foo, res);
out() << ok << " retval:" << res << endl;
}
void testPaired();
int test2() {
testSort();
return 0;
}
/* --- dbclientconnection --- */
bool DBClientConnection::auth(const string &dbname, const string &username, const string &password_text, string& errmsg, bool digestPassword) {
string password = password_text;
if( digestPassword )
password = createPasswordDigest( username , password_text );
if( autoReconnect ) {
/* note we remember the auth info before we attempt to auth -- if the connection is broken, we will
then have it for the next autoreconnect attempt.
*/
pair<string,string> p = pair<string,string>(username, password);
authCache[dbname] = p;
}
return DBClientBase::auth(dbname, username, password.c_str(), errmsg, false);
}
BSONObj DBClientBase::findOne(const string &ns, Query query, BSONObj *fieldsToReturn, int queryOptions) {
auto_ptr<DBClientCursor> c =
this->query(ns, query, 1, 0, fieldsToReturn, queryOptions);
massert( "DBClientBase::findOne: transport error", c.get() );
if ( !c->more() )
return BSONObj();
return c->next().copy();
}
bool DBClientConnection::connect(const string &_serverAddress, string& errmsg) {
serverAddress = _serverAddress;
string ip;
int port;
size_t idx = serverAddress.find( ":" );
if ( idx != string::npos ) {
port = strtol( serverAddress.substr( idx + 1 ).c_str(), 0, 10 );
ip = serverAddress.substr( 0 , idx );
ip = hostbyname(ip.c_str());
} else {
port = DBPort;
ip = hostbyname( serverAddress.c_str() );
}
massert( "Unable to parse hostname", !ip.empty() );
// we keep around SockAddr for connection life -- maybe MessagingPort
// requires that?
server = auto_ptr<SockAddr>(new SockAddr(ip.c_str(), port));
p = auto_ptr<MessagingPort>(new MessagingPort());
if ( !p->connect(*server) ) {
stringstream ss;
ss << "couldn't connect to server " << serverAddress << " " << ip << ":" << port;
errmsg = ss.str();
failed = true;
return false;
}
return true;
}
void DBClientConnection::_checkConnection() {
if ( !failed )
return;
if ( lastReconnectTry && time(0)-lastReconnectTry < 2 )
return;
if ( !autoReconnect )
return;
lastReconnectTry = time(0);
log() << "trying reconnect to " << serverAddress << endl;
string errmsg;
string tmp = serverAddress;
failed = false;
if ( !connect(tmp.c_str(), errmsg) ) {
log() << "reconnect " << serverAddress << " failed " << errmsg << endl;
return;
}
log() << "reconnect " << serverAddress << " ok" << endl;
for( map< string, pair<string,string> >::iterator i = authCache.begin(); i != authCache.end(); i++ ) {
const char *dbname = i->first.c_str();
const char *username = i->second.first.c_str();
const char *password = i->second.second.c_str();
if( !DBClientBase::auth(dbname, username, password, errmsg, false) )
log() << "reconnect: auth failed db:" << dbname << " user:"******"_id" ) )
flags |= 1;
b.append( flags );
obj.obj.appendSelfToBufBuilder( b );
toSend.setData( dbDelete , b.buf() , b.len() );
say( toSend );
}
void DBClientBase::update( const string & ns , Query query , BSONObj obj , bool upsert ) {
BufBuilder b;
b.append( (int)0 ); // reserverd
b.append( ns );
b.append( (int)upsert );
query.obj.appendSelfToBufBuilder( b );
obj.appendSelfToBufBuilder( b );
Message toSend;
toSend.setData( dbUpdate , b.buf() , b.len() );
say( toSend );
}
bool DBClientBase::ensureIndex( const string &ns , BSONObj keys , const string & name ) {
BSONObjBuilder toSave;
toSave.append( "ns" , ns );
toSave.append( "key" , keys );
string cacheKey(ns);
cacheKey += "--";
if ( name != "" ) {
toSave.append( "name" , name );
cacheKey += name;
}
else {
stringstream ss;
bool first = 1;
for ( BSONObjIterator i(keys); i.more(); ) {
BSONElement f = i.next();
if ( f.eoo() )
break;
if ( first )
first = 0;
else
ss << "_";
ss << f.fieldName() << "_";
if ( f.type() == NumberInt )
ss << (int)(f.number() );
else if ( f.type() == NumberDouble )
ss << f.number();
}
toSave.append( "name" , ss.str() );
cacheKey += ss.str();
}
if ( _seenIndexes.count( cacheKey ) )
return 0;
_seenIndexes.insert( cacheKey );
insert( Namespace( ns.c_str() ).getSisterNS( "system.indexes" ).c_str() , toSave.obj() );
return 1;
}
void DBClientBase::resetIndexCache() {
_seenIndexes.clear();
}
/* -- DBClientCursor ---------------------------------------------- */
void assembleRequest( const string &ns, BSONObj query, int nToReturn, int nToSkip, BSONObj *fieldsToReturn, int queryOptions, Message &toSend ) {
CHECK_OBJECT( query , "assembleRequest query" );
// see query.h for the protocol we are using here.
BufBuilder b;
int opts = queryOptions;
assert( (opts&Option_ALLMASK) == opts );
b.append(opts);
b.append(ns.c_str());
b.append(nToSkip);
b.append(nToReturn);
query.appendSelfToBufBuilder(b);
if ( fieldsToReturn )
fieldsToReturn->appendSelfToBufBuilder(b);
toSend.setData(dbQuery, b.buf(), b.len());
}
void DBClientConnection::say( Message &toSend ) {
checkConnection();
try {
port().say( toSend );
} catch( SocketException & ) {
failed = true;
throw;
}
}
void DBClientConnection::sayPiggyBack( Message &toSend ) {
port().piggyBack( toSend );
}
bool DBClientConnection::call( Message &toSend, Message &response, bool assertOk ) {
/* todo: this is very ugly messagingport::call returns an error code AND can throw
an exception. we should make it return void and just throw an exception anytime
it fails
*/
try {
if ( !port().call(toSend, response) ) {
failed = true;
if ( assertOk )
massert("dbclient error communicating with server", false);
return false;
}
}
catch( SocketException & ) {
failed = true;
throw;
}
return true;
}
void DBClientConnection::checkResponse( const char *data, int nReturned ) {
/* check for errors. the only one we really care about at
this stage is "not master" */
if ( clientPaired && nReturned ) {
BSONObj o(data);
BSONElement e = o.firstElement();
if ( strcmp(e.fieldName(), "$err") == 0 &&
e.type() == String && strncmp(e.valuestr(), "not master", 10) == 0 ) {
clientPaired->isntMaster();
}
}
}
bool DBClientCursor::init() {
Message toSend;
if ( !cursorId ) {
assembleRequest( ns, query, nToReturn, nToSkip, fieldsToReturn, opts, toSend );
} else {
BufBuilder b;
b.append( opts );
b.append( ns.c_str() );
b.append( nToReturn );
b.append( cursorId );
toSend.setData( dbGetMore, b.buf(), b.len() );
}
if ( !connector->call( toSend, *m, false ) )
return false;
dataReceived();
return true;
}
void DBClientCursor::requestMore() {
assert( cursorId && pos == nReturned );
BufBuilder b;
b.append(opts);
b.append(ns.c_str());
b.append(nToReturn);
b.append(cursorId);
Message toSend;
toSend.setData(dbGetMore, b.buf(), b.len());
auto_ptr<Message> response(new Message());
connector->call( toSend, *response );
m = response;
dataReceived();
}
void DBClientCursor::dataReceived() {
QueryResult *qr = (QueryResult *) m->data;
if ( qr->resultFlags() & QueryResult::ResultFlag_CursorNotFound ) {
// cursor id no longer valid at the server.
assert( qr->cursorId == 0 );
cursorId = 0; // 0 indicates no longer valid (dead)
}
if ( cursorId == 0 || ! ( opts & Option_CursorTailable ) ) {
// only set initially: we don't want to kill it on end of data
// if it's a tailable cursor
cursorId = qr->cursorId;
}
nReturned = qr->nReturned;
pos = 0;
data = qr->data();
connector->checkResponse( data, nReturned );
/* this assert would fire the way we currently work:
assert( nReturned || cursorId == 0 );
*/
}
bool DBClientCursor::more() {
if ( pos < nReturned )
return true;
if ( cursorId == 0 )
return false;
requestMore();
return pos < nReturned;
}
BSONObj DBClientCursor::next() {
assert( more() );
pos++;
BSONObj o(data);
data += o.objsize();
return o;
}
DBClientCursor::~DBClientCursor() {
if ( cursorId && ownCursor_ ) {
BufBuilder b;
b.append( (int)0 ); // reserved
b.append( (int)1 ); // number
b.append( cursorId );
Message m;
m.setData( dbKillCursors , b.buf() , b.len() );
connector->sayPiggyBack( m );
}
}
/* ------------------------------------------------------ */
// "./db testclient" to invoke
void testClient3() {
out() << "testClient()" << endl;
// DBClientConnection c(true);
DBClientPaired c;
string err;
if ( !c.connect("10.211.55.2", "1.2.3.4") ) {
// if( !c.connect("10.211.55.2", err) ) {
out() << "testClient: connect() failed" << endl;
}
else {
// temp:
out() << "test query returns: " << c.findOne("foo.bar", fromjson("{}")).toString() << endl;
}
again:
out() << "query foo.bar..." << endl;
auto_ptr<DBClientCursor> cursor =
c.query("foo.bar", BSONObj(), 0, 0, 0, Option_CursorTailable);
DBClientCursor *cc = cursor.get();
if ( cc == 0 ) {
out() << "query() returned 0, sleeping 10 secs" << endl;
sleepsecs(10);
goto again;
}
while ( 1 ) {
bool m;
try {
m = cc->more();
} catch (AssertionException&) {
out() << "more() asserted, sleeping 10 sec" << endl;
goto again;
}
out() << "more: " << m << " dead:" << cc->isDead() << endl;
if ( !m ) {
if ( cc->isDead() )
out() << "cursor dead, stopping" << endl;
else {
out() << "Sleeping 10 seconds" << endl;
sleepsecs(10);
continue;
}
break;
}
out() << cc->next().toString() << endl;
}
}
/* --- class dbclientpaired --- */
string DBClientPaired::toString() {
stringstream ss;
ss << "state: " << master << '\n';
ss << "left: " << left.toStringLong() << '\n';
ss << "right: " << right.toStringLong() << '\n';
return ss.str();
}
#pragma warning(disable: 4355)
DBClientPaired::DBClientPaired() :
left(true, this), right(true, this)
{
master = NotSetL;
}
#pragma warning(default: 4355)
/* find which server, the left or right, is currently master mode */
void DBClientPaired::_checkMaster() {
for ( int retry = 0; retry < 2; retry++ ) {
int x = master;
for ( int pass = 0; pass < 2; pass++ ) {
DBClientConnection& c = x == 0 ? left : right;
try {
bool im;
BSONObj o;
c.isMaster(im, &o);
if ( retry )
log() << "checkmaster: " << c.toString() << ' ' << o.toString() << '\n';
if ( im ) {
master = (State) (x + 2);
return;
}
}
catch (AssertionException&) {
if ( retry )
log() << "checkmaster: caught exception " << c.toString() << '\n';
}
x = x^1;
}
sleepsecs(1);
}
uassert("checkmaster: no master found", false);
}
inline DBClientConnection& DBClientPaired::checkMaster() {
if ( master > NotSetR ) {
// a master is selected. let's just make sure connection didn't die
DBClientConnection& c = master == Left ? left : right;
if ( !c.isFailed() )
return c;
// after a failure, on the next checkMaster, start with the other
// server -- presumably it took over. (not critical which we check first,
// just will make the failover slightly faster if we guess right)
master = master == Left ? NotSetR : NotSetL;
}
_checkMaster();
assert( master > NotSetR );
return master == Left ? left : right;
}
bool DBClientPaired::connect(const string &serverHostname1, const string &serverHostname2) {
string errmsg;
bool l = left.connect(serverHostname1, errmsg);
bool r = right.connect(serverHostname2, errmsg);
master = l ? NotSetL : NotSetR;
if ( !l && !r ) // it would be ok to fall through, but checkMaster will then try an immediate reconnect which is slow
return false;
try {
checkMaster();
}
catch (AssertionException&) {
return false;
}
return true;
}
bool DBClientPaired::auth(const string &dbname, const string &username, const string &pwd, string& errmsg) {
DBClientConnection& m = checkMaster();
if( !m.auth(dbname, username, pwd, errmsg) )
return false;
/* we try to authentiate with the other half of the pair -- even if down, that way the authInfo is cached. */
string e;
try {
if( &m == &left )
right.auth(dbname, username, pwd, e);
else
left.auth(dbname, username, pwd, e);
}
catch( AssertionException&) {
}
return true;
}
auto_ptr<DBClientCursor> DBClientPaired::query(const string &a, Query b, int c, int d,
BSONObj *e, int f)
{
return checkMaster().query(a,b,c,d,e,f);
}
BSONObj DBClientPaired::findOne(const string &a, Query b, BSONObj *c, int d) {
return checkMaster().findOne(a,b,c,d);
}
void testPaired() {
DBClientPaired p;
log() << "connect returns " << p.connect("localhost:27017", "localhost:27018") << endl;
//DBClientConnection p(true);
string errmsg;
// log() << "connect " << p.connect("localhost", errmsg) << endl;
log() << "auth " << p.auth("dwight", "u", "p", errmsg) << endl;
while( 1 ) {
sleepsecs(3);
try {
log() << "findone returns " << p.findOne("dwight.foo", BSONObj()).toString() << endl;
sleepsecs(3);
BSONObj info;
bool im;
log() << "ismaster returns " << p.isMaster(im,&info) << " info: " << info.toString() << endl;
}
catch(...) {
cout << "caught exception" << endl;
}
}
}
} // namespace mongo
mutablebson::Document doc(fromjson("{a: [0, 1, 0]}"));
addIndexedPath("a");
auto result = root.apply(getApplyParams(doc.root()));
ASSERT_TRUE(result.indexesAffected);
ASSERT_FALSE(result.noop);
ASSERT_EQUALS(fromjson("{a: [2, 1, 2]}"), doc);
ASSERT_TRUE(doc.isInPlaceModeEnabled());
ASSERT_EQUALS(fromjson("{$set: {a: [2, 1, 2]}}"), getLogDoc());
ASSERT_EQUALS("{a.0, a.2}", getModifiedPaths());
}
DEATH_TEST_F(UpdateArrayNodeTest,
ArrayElementsMustNotBeDeserialized,
"Invariant failure childElement.hasValue()") {
auto update = fromjson("{$set: {'a.$[i].b': 0}}");
auto arrayFilter = fromjson("{'i.c': 0}");
boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest());
std::map<StringData, std::unique_ptr<ExpressionWithPlaceholder>> arrayFilters;
auto parsedFilter = assertGet(MatchExpressionParser::parse(arrayFilter, expCtx));
arrayFilters["i"] = assertGet(ExpressionWithPlaceholder::make(std::move(parsedFilter)));
std::set<std::string> foundIdentifiers;
UpdateObjectNode root;
ASSERT_OK(UpdateObjectNode::parseAndMerge(&root,
modifiertable::ModifierType::MOD_SET,
update["$set"]["a.$[i].b"],
expCtx,
arrayFilters,
foundIdentifiers));
mutablebson::Document doc(fromjson("{a: [{c: 0}, {c: 0}, {c: 1}]}"));
void run() {
for( int i = 0; i < 10000; ++i )
fromjson( sample );
}
Json() : o_( fromjson( sample ) ) {}
ReplSource tmp(c->current());
if ( tmp.syncedTo.isNull() ) {
DBDirectClient c;
if ( c.exists( "local.oplog.$main" ) ) {
BSONObj op = c.findOne( "local.oplog.$main", QUERY( "op" << NE << "n" ).sort( BSON( "$natural" << -1 ) ) );
if ( !op.isEmpty() ) {
tmp.syncedTo = op[ "ts" ].date();
}
}
}
addSourceToList(v, tmp, old);
c->advance();
}
}
BSONObj opTimeQuery = fromjson("{\"getoptime\":1}");
bool ReplSource::throttledForceResyncDead( const char *requester ) {
if ( time( 0 ) - lastForcedResync > 600 ) {
forceResyncDead( requester );
lastForcedResync = time( 0 );
return true;
}
return false;
}
void ReplSource::forceResyncDead( const char *requester ) {
if ( !replAllDead )
return;
SourceVector sources;
ReplSource::loadAll(sources);
static void insert( const char *s ) {
insert( fromjson( s ) );
}
virtual BSONObj query() const { return fromjson( "{$or:[{a:'u'},{a:'y'}]}" ); }
virtual BSONObj query() const {
return fromjson( "{$or:[{a:'u'},{a:{$gte:'zz'}},{}]}" );
}
INT32 migWorker::_getBsonFromQueue( pmdEDUCB *eduCB, BSONObj &obj )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB__MIGWORKER__GETBSON );
INT32 tempRc = SDB_OK ;
UINT32 offset = 0 ;
UINT32 size = 0 ;
UINT32 line = 0 ;
UINT32 column = 0 ;
UINT32 startBlock = 0 ;
UINT32 endBlock = 0 ;
//CHAR *pJsonBuffer = NULL ;
_master->popFromQueue ( eduCB,
offset, size,
line, column ) ;
if ( 0 == offset && 0 == size &&
0 == line && 0 == column )
{
rc = SDB_MIG_END_OF_QUEUE ;
goto done ;
}
if ( MIG_PARSER_JSON == _master->_fileType )
{
tempRc = fromjson ( _master->getBuffer() + offset, obj ) ;
}
else if ( MIG_PARSER_CSV == _master->_fileType )
{
rc = _csvParser.csv2bson( _master->getBuffer() + offset,
size, &obj ) ;
if ( rc )
{
rc = SDB_UTIL_PARSE_JSON_INVALID ;
PD_LOG ( PDERROR, "Failed to convert Bson, rc=%d", rc ) ;
goto error ;
}
}
else
{
rc = SDB_MIG_UNKNOW_FILE_TYPE ;
PD_LOG ( PDERROR, "unknow file type" ) ;
goto error ;
}
if ( tempRc )
{
//PD_LOG ( PDERROR, "Failed to json convert bson, json: %s , rc=%d",
// _pJsonBuffer, tempRc ) ;
_master->sendMsgToClient ( "Error: error "
"in json format, line %u, column %u",
line, column ) ;
}
rc = _master->getBlockFromPointer ( offset, size,
startBlock, endBlock ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to get block from pointer, rc=%d", rc ) ;
goto error ;
}
for ( UINT32 i = startBlock; i <= endBlock; ++i )
{
_master->bucketDec( i ) ;
}
done:
if ( tempRc )
{
rc = tempRc ;
}
PD_TRACE_EXITRC ( SDB__MIGWORKER__GETBSON, rc );
return rc ;
error:
goto done ;
}
virtual BSONObj query() const { return fromjson( "{a:{$in:['u','y']}}" ); }
bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
log(1) << " authenticate: " << cmdObj << endl;
string user = cmdObj.getStringField("user");
string key = cmdObj.getStringField("key");
string received_nonce = cmdObj.getStringField("nonce");
if( user.empty() || key.empty() || received_nonce.empty() ) {
log() << "field missing/wrong type in received authenticate command "
<< dbname
<< endl;
errmsg = "auth fails";
sleepmillis(10);
return false;
}
stringstream digestBuilder;
{
bool reject = false;
nonce *ln = lastNonce.release();
if ( ln == 0 ) {
reject = true;
} else {
digestBuilder << hex << *ln;
reject = digestBuilder.str() != received_nonce;
}
if ( reject ) {
log() << "auth: bad nonce received or getnonce not called. could be a driver bug or a security attack. db:" << cc().database()->name << endl;
errmsg = "auth fails";
sleepmillis(30);
return false;
}
}
static BSONObj userPattern = fromjson("{\"user\":1}");
string systemUsers = dbname + ".system.users";
OCCASIONALLY Helpers::ensureIndex(systemUsers.c_str(), userPattern, false, "user_1");
BSONObj userObj;
{
BSONObjBuilder b;
b << "user" << user;
BSONObj query = b.done();
if( !Helpers::findOne(systemUsers.c_str(), query, userObj) ) {
log() << "auth: couldn't find user " << user << ", " << systemUsers << endl;
errmsg = "auth fails";
return false;
}
}
md5digest d;
{
string pwd = userObj.getStringField("pwd");
digestBuilder << user << pwd;
string done = digestBuilder.str();
md5_state_t st;
md5_init(&st);
md5_append(&st, (const md5_byte_t *) done.c_str(), done.size());
md5_finish(&st, d);
}
string computed = digestToString( d );
if ( key != computed ){
log() << "auth: key mismatch " << user << ", ns:" << dbname << endl;
errmsg = "auth fails";
return false;
}
AuthenticationInfo *ai = cc().getAuthenticationInfo();
if ( userObj[ "readOnly" ].isBoolean() && userObj[ "readOnly" ].boolean() ) {
ai->authorizeReadOnly( cc().database()->name.c_str() );
} else {
ai->authorize( cc().database()->name.c_str() );
}
return true;
}
return auditOptions.destination != "";
}
Status initialize() {
if (!_auditEnabledOnCommandLine()) {
// Write audit events into the void for debug builds, so we get
// coverage on the code that generates audit log objects.
DEV {
log() << "Initializing dev null audit..." << std::endl;
_setGlobalAuditLog(new VoidAuditLog(fromjson(auditOptions.filter)));
}
return Status::OK();
}
log() << "Initializing audit..." << std::endl;
const BSONObj filter = fromjson(auditOptions.filter);
if (auditOptions.destination == "console")
_setGlobalAuditLog(new ConsoleAuditLog(filter));
else if (auditOptions.destination == "syslog")
_setGlobalAuditLog(new SyslogAuditLog(filter));
// "file" destination
else if (auditOptions.format == "BSON")
_setGlobalAuditLog(new BSONAuditLog(auditOptions.path, filter));
else
_setGlobalAuditLog(new JSONAuditLog(auditOptions.path, filter));
return Status::OK();
}
MONGO_INITIALIZER_WITH_PREREQUISITES(AuditInit, ("SetGlobalEnvironment"))
(InitializerContext *context)
{
BSONObj pipelineFromJsonArray(const string& array) {
return fromjson("{pipeline: " + array + "}");
}
void cloudCmdLineParamIs(string cmd) {
string errmsg;
BSONObjBuilder res;
BSONObj o = fromjson(cmd);
cmdCloud.run("", o, 0, errmsg, res, false);
}
virtual BSONObj query() const {
return fromjson( "{$or:[{a:'x'},{a:{$gt:'u',$lt:'u'}},{a:{$gte:'y'}}]}" );
}
namespace repl {
// used in replAuthenticate
static const BSONObj userReplQuery = fromjson("{\"user\":\"repl\"}");
SyncSourceFeedback::SyncSourceFeedback() : _positionChanged(false),
_handshakeNeeded(false),
_shutdownSignaled(false) {}
SyncSourceFeedback::~SyncSourceFeedback() {}
void SyncSourceFeedback::_resetConnection() {
LOG(1) << "resetting connection in sync source feedback";
_connection.reset();
}
bool SyncSourceFeedback::replAuthenticate() {
if (!getGlobalAuthorizationManager()->isAuthEnabled())
return true;
if (!isInternalAuthSet())
return false;
return authenticateInternalUser(_connection.get());
}
void SyncSourceFeedback::ensureMe(OperationContext* txn) {
string myname = getHostName();
{
Lock::DBLock dlk(txn->lockState(), "local", MODE_X);
WriteUnitOfWork wunit(txn);
Client::Context ctx(txn, "local");
// local.me is an identifier for a server for getLastError w:2+
if (!Helpers::getSingleton(txn, "local.me", _me) ||
!_me.hasField("host") ||
_me["host"].String() != myname) {
// clean out local.me
Helpers::emptyCollection(txn, "local.me");
// repopulate
BSONObjBuilder b;
b.appendOID("_id", 0, true);
b.append("host", myname);
_me = b.obj();
Helpers::putSingleton(txn, "local.me", _me);
}
wunit.commit();
// _me is used outside of a read lock, so we must copy it out of the mmap
_me = _me.getOwned();
}
}
bool SyncSourceFeedback::replHandshake(OperationContext* txn) {
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
if (replCoord->getCurrentMemberState().primary()) {
// primary has no one to handshake to
return true;
}
// construct a vector of handshake obj for us as well as all chained members
std::vector<BSONObj> handshakeObjs;
replCoord->prepareReplSetUpdatePositionCommandHandshakes(txn, &handshakeObjs);
LOG(1) << "handshaking upstream updater";
for (std::vector<BSONObj>::iterator it = handshakeObjs.begin();
it != handshakeObjs.end();
++it) {
BSONObj res;
try {
LOG(2) << "Sending to " << _connection.get()->toString() << " the replication "
"handshake: " << *it;
if (!_connection->runCommand("admin", *it, res)) {
std::string errMsg = res["errmsg"].valuestrsafe();
massert(17447, "upstream updater is not supported by the member from which we"
" are syncing, please update all nodes to 2.6 or later.",
errMsg.find("no such cmd") == std::string::npos);
log() << "replSet error while handshaking the upstream updater: "
<< errMsg;
// sleep half a second if we are not in our sync source's config
// TODO(dannenberg) after 2.8, remove the string comparison
if (res["code"].numberInt() == ErrorCodes::NodeNotFound ||
errMsg.find("could not be found in replica set config while attempting "
"to associate it with") != std::string::npos) {
// black list sync target for 10 seconds and find a new one
replCoord->blacklistSyncSource(_syncTarget,
Date_t(curTimeMillis64() + 10*1000));
BackgroundSync::get()->clearSyncTarget();
}
_resetConnection();
return false;
}
}
catch (const DBException& e) {
log() << "SyncSourceFeedback error sending handshake: " << e.what() << endl;
_resetConnection();
return false;
}
}
return true;
}
bool SyncSourceFeedback::_connect(OperationContext* txn, const HostAndPort& host) {
if (hasConnection()) {
return true;
}
log() << "replset setting syncSourceFeedback to " << host.toString() << rsLog;
_connection.reset(new DBClientConnection(false, 0, OplogReader::tcp_timeout));
string errmsg;
try {
if (!_connection->connect(host, errmsg) ||
(getGlobalAuthorizationManager()->isAuthEnabled() && !replAuthenticate())) {
_resetConnection();
log() << "repl: " << errmsg << endl;
return false;
}
}
catch (const DBException& e) {
log() << "Error connecting to " << host.toString() << ": " << e.what();
_resetConnection();
return false;
}
return hasConnection();
}
void SyncSourceFeedback::forwardSlaveHandshake() {
boost::unique_lock<boost::mutex> lock(_mtx);
_handshakeNeeded = true;
_cond.notify_all();
}
void SyncSourceFeedback::forwardSlaveProgress() {
boost::unique_lock<boost::mutex> lock(_mtx);
_positionChanged = true;
_cond.notify_all();
}
Status SyncSourceFeedback::updateUpstream(OperationContext* txn) {
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
if (replCoord->getCurrentMemberState().primary()) {
// primary has no one to update to
return Status::OK();
}
BSONObjBuilder cmd;
{
boost::unique_lock<boost::mutex> lock(_mtx);
if (_handshakeNeeded) {
// Don't send updates if there are nodes that haven't yet been handshaked
return Status(ErrorCodes::NodeNotFound,
"Need to send handshake before updating position upstream");
}
replCoord->prepareReplSetUpdatePositionCommand(txn, &cmd);
}
BSONObj res;
LOG(2) << "Sending slave oplog progress to upstream updater: " << cmd.done();
try {
_connection->runCommand("admin", cmd.obj(), res);
}
catch (const DBException& e) {
log() << "SyncSourceFeedback error sending update: " << e.what() << endl;
_resetConnection();
return e.toStatus();
}
Status status = Command::getStatusFromCommandResult(res);
if (!status.isOK()) {
log() << "SyncSourceFeedback error sending update, response: " << res.toString() <<endl;
_resetConnection();
}
return status;
}
void SyncSourceFeedback::shutdown() {
boost::unique_lock<boost::mutex> lock(_mtx);
_shutdownSignaled = true;
_cond.notify_all();
}
void SyncSourceFeedback::run() {
Client::initThread("SyncSourceFeedbackThread");
OperationContextImpl txn;
bool positionChanged = false;
bool handshakeNeeded = false;
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
while (!inShutdown()) { // TODO(spencer): Remove once legacy repl coordinator is gone.
{
boost::unique_lock<boost::mutex> lock(_mtx);
while (!_positionChanged && !_handshakeNeeded && !_shutdownSignaled) {
_cond.wait(lock);
}
if (_shutdownSignaled) {
break;
}
positionChanged = _positionChanged;
handshakeNeeded = _handshakeNeeded;
_positionChanged = false;
_handshakeNeeded = false;
if (handshakeNeeded) {
positionChanged = true; // Always update position after sending a handshake
}
}
MemberState state = replCoord->getCurrentMemberState();
if (state.primary() || state.startup()) {
_resetConnection();
continue;
}
const HostAndPort target = BackgroundSync::get()->getSyncTarget();
if (_syncTarget != target) {
_resetConnection();
_syncTarget = target;
}
if (!hasConnection()) {
// fix connection if need be
if (target.empty()) {
sleepmillis(500);
continue;
}
if (!_connect(&txn, target)) {
sleepmillis(500);
continue;
}
handshakeNeeded = true;
}
if (handshakeNeeded) {
if (!replHandshake(&txn)) {
boost::unique_lock<boost::mutex> lock(_mtx);
_handshakeNeeded = true;
continue;
}
}
if (positionChanged) {
Status status = updateUpstream(&txn);
if (!status.isOK()) {
boost::unique_lock<boost::mutex> lock(_mtx);
_positionChanged = true;
if (status == ErrorCodes::NodeNotFound) {
_handshakeNeeded = true;
}
}
}
}
cc().shutdown();
}
} // namespace repl
namespace mongo {
// used in replAuthenticate
static const BSONObj userReplQuery = fromjson("{\"user\":\"repl\"}");
void SyncSourceFeedback::associateMember(const BSONObj& id, Member* member) {
invariant(member);
const OID rid = id["_id"].OID();
boost::unique_lock<boost::mutex> lock(_mtx);
_handshakeNeeded = true;
_members[rid] = member;
_cond.notify_all();
}
bool SyncSourceFeedback::replAuthenticate() {
if (!getGlobalAuthorizationManager()->isAuthEnabled())
return true;
if (!isInternalAuthSet())
return false;
return authenticateInternalUser(_connection.get());
}
void SyncSourceFeedback::ensureMe() {
string myname = getHostName();
{
Client::WriteContext ctx("local");
// local.me is an identifier for a server for getLastError w:2+
if (!Helpers::getSingleton("local.me", _me) ||
!_me.hasField("host") ||
_me["host"].String() != myname) {
// clean out local.me
Helpers::emptyCollection("local.me");
// repopulate
BSONObjBuilder b;
b.appendOID("_id", 0, true);
b.append("host", myname);
_me = b.obj();
Helpers::putSingleton("local.me", _me);
}
// _me is used outside of a read lock, so we must copy it out of the mmap
_me = _me.getOwned();
}
}
bool SyncSourceFeedback::replHandshake() {
// handshake for us
BSONObjBuilder cmd;
cmd.append("replSetUpdatePosition", 1);
BSONObjBuilder sub (cmd.subobjStart("handshake"));
sub.appendAs(_me["_id"], "handshake");
sub.append("member", theReplSet->selfId());
sub.append("config", theReplSet->myConfig().asBson());
sub.doneFast();
LOG(1) << "detecting upstream updater";
BSONObj res;
try {
if (!_connection->runCommand("admin", cmd.obj(), res)) {
if (res["errmsg"].str().find("no such cmd") != std::string::npos) {
LOG(1) << "upstream updater is not supported by the member from which we"
" are syncing, using oplogreader-based updating instead";
_supportsUpdater = false;
}
resetConnection();
return false;
}
else {
LOG(1) << "upstream updater is supported";
_supportsUpdater = true;
}
}
catch (const DBException& e) {
log() << "SyncSourceFeedback error sending handshake: " << e.what() << endl;
resetConnection();
return false;
}
// handshakes for those connected to us
{
for (OIDMemberMap::iterator itr = _members.begin();
itr != _members.end(); ++itr) {
BSONObjBuilder slaveCmd;
slaveCmd.append("replSetUpdatePosition", 1);
// outer handshake indicates this is a handshake command
// inner is needed as part of the structure to be passed to gotHandshake
BSONObjBuilder slaveSub (slaveCmd.subobjStart("handshake"));
slaveSub.append("handshake", itr->first);
slaveSub.append("member", itr->second->id());
slaveSub.append("config", itr->second->config().asBson());
slaveSub.doneFast();
BSONObj slaveRes;
try {
if (!_connection->runCommand("admin", slaveCmd.obj(), slaveRes)) {
resetConnection();
return false;
}
}
catch (const DBException& e) {
log() << "SyncSourceFeedback error sending chained handshakes: "
<< e.what() << endl;
resetConnection();
return false;
}
}
}
return true;
}
bool SyncSourceFeedback::_connect(const std::string& hostName) {
if (hasConnection()) {
return true;
}
log() << "replset setting syncSourceFeedback to " << hostName << rsLog;
_connection.reset(new DBClientConnection(false, 0, OplogReader::tcp_timeout));
string errmsg;
if (!_connection->connect(hostName.c_str(), errmsg) ||
(getGlobalAuthorizationManager()->isAuthEnabled() && !replAuthenticate())) {
resetConnection();
log() << "repl: " << errmsg << endl;
return false;
}
if (!replHandshake()) {
if (!supportsUpdater()) {
return connectOplogReader(hostName);
}
return false;
}
return true;
}
bool SyncSourceFeedback::connect(const Member* target) {
boost::unique_lock<boost::mutex> lock(_mtx);
boost::unique_lock<boost::mutex> connlock(_connmtx);
resetConnection();
resetOplogReaderConnection();
_syncTarget = target;
if (_connect(target->fullName())) {
if (!supportsUpdater()) {
return true;
}
}
return false;
}
void SyncSourceFeedback::forwardSlaveHandshake() {
boost::unique_lock<boost::mutex> lock(_mtx);
_handshakeNeeded = true;
_cond.notify_all();
}
void SyncSourceFeedback::percolate(const mongo::OID& rid, const OpTime& ot) {
// Update our own record of where this node is, and then register an upstream
// message about this.
// Note that we must keep the map up to date even if we are not actively reporting
// upstream via the new command, since our sync source might later change to a node
// that does support the command.
updateMap(rid, ot);
if (!supportsUpdater()) {
// this is only necessary if our sync source does not support
// the new syncSourceFeedback command
theReplSet->ghost->send(boost::bind(&GhostSync::percolate,
theReplSet->ghost,
rid,
ot));
}
}
void SyncSourceFeedback::updateMap(const mongo::OID& rid, const OpTime& ot) {
boost::unique_lock<boost::mutex> lock(_mtx);
// only update if ot is newer than what we have already
if (ot > _slaveMap[rid]) {
_slaveMap[rid] = ot;
_positionChanged = true;
LOG(2) << "now last is " << _slaveMap[rid].toString() << endl;
_cond.notify_all();
}
}
bool SyncSourceFeedback::updateUpstream() {
if (theReplSet->isPrimary()) {
// primary has no one to update to
return true;
}
BSONObjBuilder cmd;
cmd.append("replSetUpdatePosition", 1);
// create an array containing objects each member connected to us and for ourself
BSONArrayBuilder array (cmd.subarrayStart("optimes"));
OID myID = _me["_id"].OID();
{
for (map<mongo::OID, OpTime>::const_iterator itr = _slaveMap.begin();
itr != _slaveMap.end(); ++itr) {
BSONObjBuilder entry(array.subobjStart());
entry.append("_id", itr->first);
entry.append("optime", itr->second);
if (itr->first == myID) {
entry.append("config", theReplSet->myConfig().asBson());
}
else {
entry.append("config", _members[itr->first]->config().asBson());
}
entry.doneFast();
}
}
array.done();
BSONObj res;
bool ok;
try {
ok = _connection->runCommand("admin", cmd.obj(), res);
}
catch (const DBException& e) {
log() << "SyncSourceFeedback error sending update: " << e.what() << endl;
resetConnection();
return false;
}
if (!ok) {
log() << "SyncSourceFeedback error sending update, response: " << res.toString() <<endl;
resetConnection();
return false;
}
return true;
}
void SyncSourceFeedback::run() {
Client::initThread("SyncSourceFeedbackThread");
bool sleepNeeded = false;
while (true) {
if (sleepNeeded) {
sleepmillis(500);
sleepNeeded = false;
}
{
boost::unique_lock<boost::mutex> lock(_mtx);
while (!_positionChanged && !_handshakeNeeded) {
_cond.wait(lock);
}
if (theReplSet->isPrimary()) {
_positionChanged = false;
_handshakeNeeded = false;
continue;
}
const Member* target = replset::BackgroundSync::get()->getSyncTarget();
boost::unique_lock<boost::mutex> connlock(_connmtx);
if (_syncTarget != target) {
resetConnection();
_syncTarget = target;
}
if (!hasConnection()) {
// fix connection if need be
if (!target) {
sleepNeeded = true;
continue;
}
if (!_connect(target->fullName())) {
sleepNeeded = true;
continue;
}
else if (!supportsUpdater()) {
_handshakeNeeded = false;
_positionChanged = false;
continue;
}
}
if (_handshakeNeeded) {
if (!replHandshake()) {
_handshakeNeeded = true;
continue;
}
else {
_handshakeNeeded = false;
}
}
if (_positionChanged) {
if (!updateUpstream()) {
_positionChanged = true;
continue;
}
else {
_positionChanged = false;
}
}
}
}
}
}
Base() : _context( ns() ) {
addIndex( fromjson( "{\"a\":1}" ) );
}
void run(){
Scope * s = globalScriptEngine->newScope();
{ // date
BSONObj o;
{
BSONObjBuilder b;
b.appendDate( "d" , 123456789 );
o = b.obj();
}
s->setObject( "x" , o );
s->invoke( "return x.d.getTime() != 12;" , BSONObj() );
ASSERT_EQUALS( true, s->getBoolean( "return" ) );
s->invoke( "z = x.d.getTime();" , BSONObj() );
ASSERT_EQUALS( 123456789 , s->getNumber( "z" ) );
s->invoke( "z = { z : x.d }" , BSONObj() );
BSONObj out = s->getObject( "z" );
ASSERT( out["z"].type() == Date );
}
{ // regex
BSONObj o;
{
BSONObjBuilder b;
b.appendRegex( "r" , "^a" , "i" );
o = b.obj();
}
s->setObject( "x" , o );
s->invoke( "z = x.r.test( 'b' );" , BSONObj() );
ASSERT_EQUALS( false , s->getBoolean( "z" ) );
s->invoke( "z = x.r.test( 'a' );" , BSONObj() );
ASSERT_EQUALS( true , s->getBoolean( "z" ) );
s->invoke( "z = x.r.test( 'ba' );" , BSONObj() );
ASSERT_EQUALS( false , s->getBoolean( "z" ) );
s->invoke( "z = { a : x.r };" , BSONObj() );
BSONObj out = s->getObject("z");
ASSERT_EQUALS( (string)"^a" , out["a"].regex() );
ASSERT_EQUALS( (string)"i" , out["a"].regexFlags() );
}
// array
{
BSONObj o = fromjson( "{r:[1,2,3]}" );
s->setObject( "x", o, false );
BSONObj out = s->getObject( "x" );
ASSERT_EQUALS( Array, out.firstElement().type() );
s->setObject( "x", o, true );
out = s->getObject( "x" );
ASSERT_EQUALS( Array, out.firstElement().type() );
}
delete s;
}
void run() {
insert( "{\"a\":\"b\"}" );
BSONObj cmd = fromjson( "{\"query\":{}}" );
string err;
ASSERT_EQUALS( 1, runCount( ns(), cmd, err ) );
}
TEST( LeafMatchExpressionTest, Comp1 ) {
BSONObj temp = BSON( "x" << 5 );
{
ComparisonMatchExpression e;
e.init( "x", ComparisonMatchExpression::LTE, temp["x"] );
ASSERT_TRUE( e.matches( fromjson( "{ x : 5 }" ) ) );
ASSERT_TRUE( e.matches( fromjson( "{ x : 4 }" ) ) );
ASSERT_FALSE( e.matches( fromjson( "{ x : 6 }" ) ) );
ASSERT_FALSE( e.matches( fromjson( "{ x : 'eliot' }" ) ) );
}
{
ComparisonMatchExpression e;
e.init( "x", ComparisonMatchExpression::LT, temp["x"] );
ASSERT_FALSE( e.matches( fromjson( "{ x : 5 }" ) ) );
ASSERT_TRUE( e.matches( fromjson( "{ x : 4 }" ) ) );
ASSERT_FALSE( e.matches( fromjson( "{ x : 6 }" ) ) );
ASSERT_FALSE( e.matches( fromjson( "{ x : 'eliot' }" ) ) );
}
{
ComparisonMatchExpression e;
e.init( "x", ComparisonMatchExpression::GTE, temp["x"] );
ASSERT_TRUE( e.matches( fromjson( "{ x : 5 }" ) ) );
ASSERT_FALSE( e.matches( fromjson( "{ x : 4 }" ) ) );
ASSERT_TRUE( e.matches( fromjson( "{ x : 6 }" ) ) );
ASSERT_FALSE( e.matches( fromjson( "{ x : 'eliot' }" ) ) );
}
{
ComparisonMatchExpression e;
e.init( "x", ComparisonMatchExpression::GT, temp["x"] );
ASSERT_FALSE( e.matches( fromjson( "{ x : 5 }" ) ) );
ASSERT_FALSE( e.matches( fromjson( "{ x : 4 }" ) ) );
ASSERT_TRUE( e.matches( fromjson( "{ x : 6 }" ) ) );
ASSERT_FALSE( e.matches( fromjson( "{ x : 'eliot' }" ) ) );
}
}
void run() {
for( int i = 0; i < 10000; ++i )
fromjson( shopwikiSample );
}
namespace repl {
// used in replAuthenticate
static const BSONObj userReplQuery = fromjson("{\"user\":\"repl\"}");
SyncSourceFeedback::SyncSourceFeedback() : _syncTarget(NULL),
_positionChanged(false),
_handshakeNeeded(false),
_shutdownSignaled(false) {}
SyncSourceFeedback::~SyncSourceFeedback() {}
bool SyncSourceFeedback::replAuthenticate() {
if (!getGlobalAuthorizationManager()->isAuthEnabled())
return true;
if (!isInternalAuthSet())
return false;
return authenticateInternalUser(_connection.get());
}
void SyncSourceFeedback::ensureMe(OperationContext* txn) {
string myname = getHostName();
{
Client::WriteContext ctx(txn, "local");
// local.me is an identifier for a server for getLastError w:2+
if (!Helpers::getSingleton(txn, "local.me", _me) ||
!_me.hasField("host") ||
_me["host"].String() != myname) {
// clean out local.me
Helpers::emptyCollection(txn, "local.me");
// repopulate
BSONObjBuilder b;
b.appendOID("_id", 0, true);
b.append("host", myname);
_me = b.obj();
Helpers::putSingleton(txn, "local.me", _me);
}
ctx.commit();
// _me is used outside of a read lock, so we must copy it out of the mmap
_me = _me.getOwned();
}
}
bool SyncSourceFeedback::replHandshake(OperationContext* txn) {
// construct a vector of handshake obj for us as well as all chained members
std::vector<BSONObj> handshakeObjs;
getGlobalReplicationCoordinator()->prepareReplSetUpdatePositionCommandHandshakes(
txn,
&handshakeObjs);
LOG(1) << "handshaking upstream updater";
for (std::vector<BSONObj>::iterator it = handshakeObjs.begin();
it != handshakeObjs.end();
++it) {
BSONObj res;
try {
LOG(2) << "Sending to " << _connection.get()->toString() << " the replication "
"handshake: " << *it;
if (!_connection->runCommand("admin", *it, res)) {
massert(17447, "upstream updater is not supported by the member from which we"
" are syncing, please update all nodes to 2.6 or later.",
res["errmsg"].str().find("no such cmd") == std::string::npos);
log() << "replSet error while handshaking the upstream updater: "
<< res["errmsg"].valuestrsafe();
_resetConnection();
return false;
}
}
catch (const DBException& e) {
log() << "SyncSourceFeedback error sending handshake: " << e.what() << endl;
_resetConnection();
return false;
}
}
return true;
}
bool SyncSourceFeedback::_connect(OperationContext* txn, const std::string& hostName) {
if (hasConnection()) {
return true;
}
log() << "replset setting syncSourceFeedback to " << hostName << rsLog;
_connection.reset(new DBClientConnection(false, 0, OplogReader::tcp_timeout));
string errmsg;
try {
if (!_connection->connect(hostName.c_str(), errmsg) ||
(getGlobalAuthorizationManager()->isAuthEnabled() && !replAuthenticate())) {
_resetConnection();
log() << "repl: " << errmsg << endl;
return false;
}
}
catch (const DBException& e) {
log() << "Error connecting to " << hostName << ": " << e.what();
_resetConnection();
return false;
}
replHandshake(txn);
return hasConnection();
}
void SyncSourceFeedback::forwardSlaveHandshake() {
boost::unique_lock<boost::mutex> lock(_mtx);
_handshakeNeeded = true;
_cond.notify_all();
}
void SyncSourceFeedback::forwardSlaveProgress() {
boost::unique_lock<boost::mutex> lock(_mtx);
_positionChanged = true;
_cond.notify_all();
}
bool SyncSourceFeedback::updateUpstream(OperationContext* txn) {
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
if (replCoord->getCurrentMemberState().primary()) {
// primary has no one to update to
return true;
}
BSONObjBuilder cmd;
{
boost::unique_lock<boost::mutex> lock(_mtx);
if (_handshakeNeeded) {
// Don't send updates if there are nodes that haven't yet been handshaked
return false;
}
replCoord->prepareReplSetUpdatePositionCommand(txn, &cmd);
}
BSONObj res;
LOG(2) << "Sending slave oplog progress to upstream updater: " << cmd.done();
bool ok;
try {
ok = _connection->runCommand("admin", cmd.obj(), res);
}
catch (const DBException& e) {
log() << "SyncSourceFeedback error sending update: " << e.what() << endl;
_resetConnection();
return false;
}
if (!ok) {
log() << "SyncSourceFeedback error sending update, response: " << res.toString() <<endl;
_resetConnection();
return false;
}
return true;
}
void SyncSourceFeedback::shutdown() {
boost::unique_lock<boost::mutex> lock(_mtx);
_shutdownSignaled = true;
_cond.notify_all();
}
void SyncSourceFeedback::run() {
Client::initThread("SyncSourceFeedbackThread");
OperationContextImpl txn;
bool positionChanged = false;
bool handshakeNeeded = false;
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
while (!inShutdown()) { // TODO(spencer): Remove once legacy repl coordinator is gone.
{
boost::unique_lock<boost::mutex> lock(_mtx);
while (!_positionChanged && !_handshakeNeeded && !_shutdownSignaled) {
_cond.wait(lock);
}
if (_shutdownSignaled) {
break;
}
positionChanged = _positionChanged;
handshakeNeeded = _handshakeNeeded;
_positionChanged = false;
_handshakeNeeded = false;
}
MemberState state = replCoord->getCurrentMemberState();
if (state.primary() || state.fatal() || state.startup()) {
continue;
}
const Member* target = BackgroundSync::get()->getSyncTarget();
if (_syncTarget != target) {
_resetConnection();
_syncTarget = target;
}
if (!hasConnection()) {
// fix connection if need be
if (!target) {
sleepmillis(500);
continue;
}
if (!_connect(&txn, target->fullName())) {
sleepmillis(500);
continue;
}
}
if (handshakeNeeded) {
if (!replHandshake(&txn)) {
boost::unique_lock<boost::mutex> lock(_mtx);
_handshakeNeeded = true;
continue;
}
}
if (positionChanged) {
if (!updateUpstream(&txn)) {
boost::unique_lock<boost::mutex> lock(_mtx);
_positionChanged = true;
}
}
}
cc().shutdown();
}
} // namespace repl
ShopwikiJson() : o_( fromjson( shopwikiSample ) ) {}
void run() {
AutoGetCollectionForRead ctx(&_txn, nss.ns());
Collection* collection = ctx.getCollection();
ASSERT(collection);
// Query can be answered by either index on "a" or index on "b".
auto statusWithCQ = CanonicalQuery::canonicalize(nss, fromjson("{a: {$gte: 8}, b: 1}"));
ASSERT_OK(statusWithCQ.getStatus());
const std::unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue());
// We shouldn't have anything in the plan cache for this shape yet.
PlanCache* cache = collection->infoCache()->getPlanCache();
ASSERT(cache);
CachedSolution* rawCachedSolution;
ASSERT_NOT_OK(cache->get(*cq, &rawCachedSolution));
// Get planner params.
QueryPlannerParams plannerParams;
fillOutPlannerParams(&_txn, collection, cq.get(), &plannerParams);
// Set up queued data stage to take a long time before returning EOF. Should be long
// enough to trigger a replan.
const size_t decisionWorks = 10;
const size_t mockWorks =
1U + static_cast<size_t>(internalQueryCacheEvictionRatio * decisionWorks);
auto mockChild = stdx::make_unique<QueuedDataStage>(&_txn, &_ws);
for (size_t i = 0; i < mockWorks; i++) {
mockChild->pushBack(PlanStage::NEED_TIME);
}
CachedPlanStage cachedPlanStage(
&_txn, collection, &_ws, cq.get(), plannerParams, decisionWorks, mockChild.release());
// This should succeed after triggering a replan.
PlanYieldPolicy yieldPolicy(nullptr, PlanExecutor::YIELD_MANUAL);
ASSERT_OK(cachedPlanStage.pickBestPlan(&yieldPolicy));
// Make sure that we get 2 legit results back.
size_t numResults = 0;
PlanStage::StageState state = PlanStage::NEED_TIME;
while (state != PlanStage::IS_EOF) {
WorkingSetID id = WorkingSet::INVALID_ID;
state = cachedPlanStage.work(&id);
ASSERT_NE(state, PlanStage::FAILURE);
ASSERT_NE(state, PlanStage::DEAD);
if (state == PlanStage::ADVANCED) {
WorkingSetMember* member = _ws.get(id);
ASSERT(cq->root()->matchesBSON(member->obj.value()));
numResults++;
}
}
ASSERT_EQ(numResults, 2U);
// This time we expect to find something in the plan cache. Replans after hitting the
// works threshold result in a cache entry.
ASSERT_OK(cache->get(*cq, &rawCachedSolution));
const std::unique_ptr<CachedSolution> cachedSolution(rawCachedSolution);
}
StackChecker::check( desc() );
}
}
#endif
_shutdown = true;
if ( inShutdown() )
return false;
{
scoped_lock bl(clientsMutex);
clients.erase(this);
}
return false;
}
BSONObj CachedBSONObj::_tooBig = fromjson("{\"$msg\":\"query not recording (too large)\"}");
Client::Context::Context(const std::string& ns , Database * db) :
_client( currentClient.get() ),
_oldContext( _client->_context ),
_path(storageGlobalParams.dbpath), // is this right? could be a different db?
// may need a dassert for this
_justCreated(false),
_doVersion( true ),
_ns( ns ),
_db(db)
{
verify( db == 0 || db->isOk() );
_client->_context = this;
}
Client::Context::Context(const string& ns, const std::string& path, bool doVersion) :
o = k.next();
if ( l.eoo() )
return r.eoo() ? 0 : -1;
if ( r.eoo() )
return 1;
int x = l.woCompare( r, considerFieldName );
if ( ordered && o.number() < 0 )
x = -x;
if ( x != 0 )
return x;
}
return -1;
}
BSONObj staticNull = fromjson( "{'':null}" );
/* well ordered compare */
int BSONObj::woSortOrder(const BSONObj& other, const BSONObj& sortKey ) const{
if ( isEmpty() )
return other.isEmpty() ? 0 : -1;
if ( other.isEmpty() )
return 1;
uassert( "woSortOrder needs a non-empty sortKey" , ! sortKey.isEmpty() );
BSONObjIterator i(sortKey);
while ( 1 ){
BSONElement f = i.next();
if ( f.eoo() )
return 0;
virtual void prep() {
insert( ns(), fromjson( "{a:1}" ) );
}
TEST(QueryRequestTest, ValidateSortOrder) {
// Valid sorts
ASSERT(QueryRequest::isValidSortOrder(fromjson("{}")));
ASSERT(QueryRequest::isValidSortOrder(fromjson("{a: 1}")));
ASSERT(QueryRequest::isValidSortOrder(fromjson("{a: -1}")));
ASSERT(QueryRequest::isValidSortOrder(fromjson("{a: {$meta: \"textScore\"}}")));
// Invalid sorts
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: 100}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: 0}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: -100}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: Infinity}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: -Infinity}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: true}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: false}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: null}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: {}}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: {b: 1}}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: []}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: [1, 2, 3]}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: \"\"}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: \"bb\"}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: {$meta: 1}}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: {$meta: \"image\"}}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{a: {$world: \"textScore\"}}")));
ASSERT_FALSE(
QueryRequest::isValidSortOrder(fromjson("{a: {$meta: \"textScore\","
" b: 1}}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{'': 1}")));
ASSERT_FALSE(QueryRequest::isValidSortOrder(fromjson("{'': -1}")));
}
