virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
BSONObj fromToken = cmdObj.getObjectField("finishCloneCollection");
if ( fromToken.isEmpty() ) {
errmsg = "missing finishCloneCollection finishToken spec";
return false;
}
string fromhost = fromToken.getStringField( "fromhost" );
if ( fromhost.empty() ) {
errmsg = "missing fromhost spec";
return false;
}
string collection = fromToken.getStringField("collection");
if ( collection.empty() ) {
errmsg = "missing collection spec";
return false;
}
BSONObj query = fromToken.getObjectField("query");
if ( query.isEmpty() ) {
query = BSONObj();
}
long long cursorId = 0;
BSONElement cursorIdToken = fromToken.getField( "cursorId" );
if ( cursorIdToken.type() == Date ) {
cursorId = cursorIdToken._numberLong();
}
setClient( collection.c_str() );
log() << "finishCloneCollection. db:" << ns << " collection:" << collection << " from: " << fromhost << " query: " << query << endl;
Cloner c;
return c.finishCloneCollection( fromhost.c_str(), collection.c_str(), query, cursorId, errmsg );
}
virtual bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
string fromhost = cmdObj.getStringField("fromhost");
if ( fromhost.empty() ) {
/* copy from self */
stringstream ss;
ss << "localhost:" << cmdLine.port;
fromhost = ss.str();
}
string fromdb = cmdObj.getStringField("fromdb");
string todb = cmdObj.getStringField("todb");
if ( fromhost.empty() || todb.empty() || fromdb.empty() ) {
errmsg = "parms missing - {copydb: 1, fromhost: <hostname>, fromdb: <db>, todb: <db>}";
return false;
}
Cloner c;
string username = cmdObj.getStringField( "username" );
string nonce = cmdObj.getStringField( "nonce" );
string key = cmdObj.getStringField( "key" );
if ( !username.empty() && !nonce.empty() && !key.empty() ) {
uassert( 13008, "must call copydbgetnonce first", authConn_.get() );
BSONObj ret;
{
dbtemprelease t;
if ( !authConn_->runCommand( fromdb, BSON( "authenticate" << 1 << "user" << username << "nonce" << nonce << "key" << key ), ret ) ) {
errmsg = "unable to login " + string( ret );
return false;
}
}
c.setConnection( authConn_.release() );
}
Client::Context ctx(todb);
bool res = c.go(fromhost.c_str(), errmsg, fromdb, /*logForReplication=*/!fromRepl, /*slaveok*/false, /*replauth*/false, /*snapshot*/true);
return res;
}
virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
string fromhost = cmdObj.getStringField("from");
if ( fromhost.empty() ) {
errmsg = "missing from spec";
return false;
}
string collection = cmdObj.getStringField("cloneCollection");
if ( collection.empty() ) {
errmsg = "missing cloneCollection spec";
return false;
}
BSONObj query = cmdObj.getObjectField("query");
if ( query.isEmpty() )
query = BSONObj();
BSONElement copyIndexesSpec = cmdObj.getField("copyindexes");
bool copyIndexes = copyIndexesSpec.isBoolean() ? copyIndexesSpec.boolean() : true;
// Will not be used if doesn't exist.
int logSizeMb = cmdObj.getIntField( "logSizeMb" );
/* replication note: we must logOp() not the command, but the cloned data -- if the slave
were to clone it would get a different point-in-time and not match.
*/
setClient( collection.c_str() );
log() << "cloneCollection. db:" << ns << " collection:" << collection << " from: " << fromhost << " query: " << query << " logSizeMb: " << logSizeMb << ( copyIndexes ? "" : ", not copying indexes" ) << endl;
Cloner c;
long long cursorId;
if ( !c.startCloneCollection( fromhost.c_str(), collection.c_str(), query, errmsg, !fromRepl, copyIndexes, logSizeMb, cursorId ) )
return false;
return c.finishCloneCollection( fromhost.c_str(), collection.c_str(), query, cursorId, errmsg);
}
virtual bool run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj,
int,
string& errmsg,
BSONObjBuilder& result) {
boost::optional<DisableDocumentValidation> maybeDisableValidation;
if (shouldBypassDocumentValidationForCommand(cmdObj))
maybeDisableValidation.emplace(txn);
string fromhost = cmdObj.getStringField("from");
if (fromhost.empty()) {
errmsg = "missing 'from' parameter";
return false;
}
{
HostAndPort h(fromhost);
if (repl::isSelf(h)) {
errmsg = "can't cloneCollection from self";
return false;
}
}
string collection = parseNs(dbname, cmdObj);
Status allowedWriteStatus = userAllowedWriteNS(dbname, collection);
if (!allowedWriteStatus.isOK()) {
return appendCommandStatus(result, allowedWriteStatus);
}
BSONObj query = cmdObj.getObjectField("query");
if (query.isEmpty())
query = BSONObj();
BSONElement copyIndexesSpec = cmdObj.getField("copyindexes");
bool copyIndexes = copyIndexesSpec.isBoolean() ? copyIndexesSpec.boolean() : true;
log() << "cloneCollection. db:" << dbname << " collection:" << collection
<< " from: " << fromhost << " query: " << query << " "
<< (copyIndexes ? "" : ", not copying indexes") << endl;
Cloner cloner;
unique_ptr<DBClientConnection> myconn;
myconn.reset(new DBClientConnection());
if (!myconn->connect(HostAndPort(fromhost), errmsg))
return false;
cloner.setConnection(myconn.release());
return cloner.copyCollection(
txn, collection, query, errmsg, true, true /* interruptable */, copyIndexes);
}
void testCloner() {
testing("Cloner");
GameManager* gm = new GameManager();
Item* i = new Item("Testing Item");
gm->insertObject(i);
i->setName("Test name")->setDescription("Some going to be cloned item");
Cloner cl = Cloner(i);
ObjectPointer other = cl.getShallowClone();
Item* j = other.safeCast<Item>();
assertEqual<string>(i->getName(), j->getName(), "Name is different");
assertEqual<string>(i->getDescription(), j->getDescription(), "Description is different");
}
void RollbackSourceImpl::copyCollectionFromRemote(OperationContext* opCtx,
const NamespaceString& nss) const {
std::string errmsg;
std::unique_ptr<DBClientConnection> tmpConn(new DBClientConnection());
uassert(15908,
errmsg,
tmpConn->connect(_source, StringData(), errmsg) && replAuthenticate(tmpConn.get()));
// cloner owns _conn in unique_ptr
Cloner cloner;
cloner.setConnection(tmpConn.release());
uassert(15909,
str::stream() << "replSet rollback error resyncing collection " << nss.ns() << ' '
<< errmsg,
cloner.copyCollection(opCtx, nss.ns(), BSONObj(), errmsg, true));
}
virtual bool run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj,
int,
string& errmsg,
BSONObjBuilder& result,
bool fromRepl) {
string fromhost = cmdObj.getStringField("from");
if ( fromhost.empty() ) {
errmsg = "missing 'from' parameter";
return false;
}
{
HostAndPort h(fromhost);
if (repl::isSelf(h)) {
errmsg = "can't cloneCollection from self";
return false;
}
}
string collection = parseNs(dbname, cmdObj);
if ( collection.empty() ) {
errmsg = "bad 'cloneCollection' value";
return false;
}
BSONObj query = cmdObj.getObjectField("query");
if ( query.isEmpty() )
query = BSONObj();
BSONElement copyIndexesSpec = cmdObj.getField("copyindexes");
bool copyIndexes = copyIndexesSpec.isBoolean() ? copyIndexesSpec.boolean() : true;
log() << "cloneCollection. db:" << dbname << " collection:" << collection << " from: " << fromhost
<< " query: " << query << " " << ( copyIndexes ? "" : ", not copying indexes" ) << endl;
Cloner cloner;
auto_ptr<DBClientConnection> myconn;
myconn.reset( new DBClientConnection() );
if ( ! myconn->connect( fromhost , errmsg ) )
return false;
cloner.setConnection( myconn.release() );
return cloner.copyCollection(txn, collection, query, errmsg, true, false, copyIndexes);
}
virtual bool run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj,
int,
string& errmsg,
BSONObjBuilder& result) {
boost::optional<DisableDocumentValidation> maybeDisableValidation;
if (shouldBypassDocumentValidationforCommand(cmdObj))
maybeDisableValidation.emplace(txn);
string from = cmdObj.getStringField("clone");
if ( from.empty() )
return false;
CloneOptions opts;
opts.fromDB = dbname;
opts.slaveOk = cmdObj["slaveOk"].trueValue();
// See if there's any collections we should ignore
if( cmdObj["collsToIgnore"].type() == Array ){
BSONObjIterator it( cmdObj["collsToIgnore"].Obj() );
while( it.more() ){
BSONElement e = it.next();
if( e.type() == String ){
opts.collsToIgnore.insert( e.String() );
}
}
}
set<string> clonedColls;
ScopedTransaction transaction(txn, MODE_IX);
Lock::DBLock dbXLock(txn->lockState(), dbname, MODE_X);
Cloner cloner;
bool rval = cloner.go(txn, dbname, from, opts, &clonedColls, errmsg);
BSONArrayBuilder barr;
barr.append( clonedColls );
result.append( "clonedColls", barr.arr() );
return rval;
}
void RollbackSourceImpl::copyCollectionFromRemote(OperationContext* opCtx,
const NamespaceString& nss) const {
std::string errmsg;
auto tmpConn = stdx::make_unique<DBClientConnection>();
uassert(15908,
errmsg,
tmpConn->connect(_source, StringData(), errmsg) && replAuthenticate(tmpConn.get()));
// cloner owns _conn in unique_ptr
Cloner cloner;
cloner.setConnection(std::move(tmpConn));
uassert(15909,
str::stream() << "replSet rollback error resyncing collection " << nss.ns() << ' '
<< errmsg,
cloner.copyCollection(
opCtx, nss.ns(), BSONObj(), errmsg, true, CollectionOptions::parseForStorage));
}
virtual bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
string fromhost = cmdObj.getStringField("from");
if ( fromhost.empty() ) {
errmsg = "missing from spec";
return false;
}
string collection = cmdObj.getStringField("startCloneCollection");
if ( collection.empty() ) {
errmsg = "missing startCloneCollection spec";
return false;
}
BSONObj query = cmdObj.getObjectField("query");
if ( query.isEmpty() )
query = BSONObj();
BSONElement copyIndexesSpec = cmdObj.getField("copyindexes");
bool copyIndexes = copyIndexesSpec.isBoolean() ? copyIndexesSpec.boolean() : true;
// Will not be used if doesn't exist.
int logSizeMb = cmdObj.getIntField( "logSizeMb" );
/* replication note: we must logOp() not the command, but the cloned data -- if the slave
were to clone it would get a different point-in-time and not match.
*/
Client::Context ctx(collection);
log() << "startCloneCollection. db:" << dbname << " collection:" << collection << " from: " << fromhost << " query: " << query << endl;
Cloner c;
long long cursorId;
bool res = c.startCloneCollection( fromhost.c_str(), collection.c_str(), query, errmsg, !fromRepl, copyIndexes, logSizeMb, cursorId );
if ( res ) {
BSONObjBuilder b;
b << "fromhost" << fromhost;
b << "collection" << collection;
b << "query" << query;
b.appendDate( "cursorId", cursorId );
BSONObj token = b.done();
result << "finishToken" << token;
}
return res;
}
/* grab initial copy of a database from the master */
void ReplSource::resync(OperationContext* txn, const std::string& dbName) {
const std::string db(dbName); // need local copy of the name, we're dropping the original
resyncDrop(txn, db);
{
log() << "resync: cloning database " << db << " to get an initial copy" << endl;
ReplInfo r("resync: cloning a database");
string errmsg;
int errCode = 0;
CloneOptions cloneOptions;
cloneOptions.fromDB = db;
cloneOptions.logForRepl = false;
cloneOptions.slaveOk = true;
cloneOptions.useReplAuth = true;
cloneOptions.snapshot = true;
cloneOptions.mayYield = true;
cloneOptions.mayBeInterrupted = false;
Cloner cloner;
bool ok = cloner.go(txn, db, hostName.c_str(), cloneOptions, NULL, errmsg, &errCode);
if (!ok) {
if (errCode == DatabaseDifferCaseCode) {
resyncDrop(txn, db);
log() << "resync: database " << db
<< " not valid on the master due to a name conflict, dropping." << endl;
return;
} else {
log() << "resync of " << db << " from " << hostName << " failed " << errmsg << endl;
throw SyncException();
}
}
}
log() << "resync: done with initial clone for db: " << db << endl;
return;
}
bool ReplSetImpl::_syncDoInitialSync_clone(OperationContext* txn,
Cloner& cloner,
const char *master,
const list<string>& dbs,
bool dataPass) {
for( list<string>::const_iterator i = dbs.begin(); i != dbs.end(); i++ ) {
const string db = *i;
if( db == "local" )
continue;
if ( dataPass )
sethbmsg( str::stream() << "initial sync cloning db: " << db , 0);
else
sethbmsg( str::stream() << "initial sync cloning indexes for : " << db , 0);
string err;
int errCode;
CloneOptions options;
options.fromDB = db;
options.logForRepl = false;
options.slaveOk = true;
options.useReplAuth = true;
options.snapshot = false;
options.mayYield = true;
options.mayBeInterrupted = false;
options.syncData = dataPass;
options.syncIndexes = ! dataPass;
// Make database stable
Lock::DBWrite dbWrite(txn->lockState(), db);
WriteUnitOfWork wunit(txn->recoveryUnit());
if (!cloner.go(txn, db, master, options, NULL, err, &errCode)) {
sethbmsg(str::stream() << "initial sync: error while "
<< (dataPass ? "cloning " : "indexing ") << db
<< ". " << (err.empty() ? "" : err + ". ")
<< "sleeping 5 minutes" ,0);
return false;
}
wunit.commit();
}
return true;
}
bool ReplSetImpl::_syncDoInitialSync_clone(Cloner& cloner, const char *master,
const list<string>& dbs, bool dataPass) {
for( list<string>::const_iterator i = dbs.begin(); i != dbs.end(); i++ ) {
string db = *i;
if( db == "local" )
continue;
if ( dataPass )
sethbmsg( str::stream() << "initial sync cloning db: " << db , 0);
else
sethbmsg( str::stream() << "initial sync cloning indexes for : " << db , 0);
Client::WriteContext ctx(db);
OperationContextImpl txn;
string err;
int errCode;
CloneOptions options;
options.fromDB = db;
options.logForRepl = false;
options.slaveOk = true;
options.useReplAuth = true;
options.snapshot = false;
options.mayYield = true;
options.mayBeInterrupted = false;
options.syncData = dataPass;
options.syncIndexes = ! dataPass;
if (!cloner.go(&txn, ctx.ctx(), master, options, NULL, err, &errCode)) {
sethbmsg(str::stream() << "initial sync: error while "
<< (dataPass ? "cloning " : "indexing ") << db
<< ". " << (err.empty() ? "" : err + ". ")
<< "sleeping 5 minutes" ,0);
return false;
}
}
return true;
}
virtual bool run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj,
int,
string& errmsg,
BSONObjBuilder& result,
bool fromRepl) {
string fromhost = cmdObj.getStringField("fromhost");
bool fromSelf = fromhost.empty();
if ( fromSelf ) {
/* copy from self */
stringstream ss;
ss << "localhost:" << serverGlobalParams.port;
fromhost = ss.str();
}
CloneOptions cloneOptions;
cloneOptions.fromDB = cmdObj.getStringField("fromdb");
cloneOptions.logForRepl = !fromRepl;
cloneOptions.slaveOk = cmdObj["slaveOk"].trueValue();
cloneOptions.useReplAuth = false;
cloneOptions.snapshot = true;
cloneOptions.mayYield = true;
cloneOptions.mayBeInterrupted = false;
string todb = cmdObj.getStringField("todb");
if ( fromhost.empty() || todb.empty() || cloneOptions.fromDB.empty() ) {
errmsg = "params missing - {copydb: 1, fromhost: <connection string>, "
"fromdb: <db>, todb: <db>}";
return false;
}
if ( !NamespaceString::validDBName( todb ) ) {
errmsg = "invalid todb name: " + todb;
return false;
}
Cloner cloner;
// Get MONGODB-CR parameters
string username = cmdObj.getStringField( "username" );
string nonce = cmdObj.getStringField( "nonce" );
string key = cmdObj.getStringField( "key" );
if ( !username.empty() && !nonce.empty() && !key.empty() ) {
uassert( 13008, "must call copydbgetnonce first", authConn_.get() );
BSONObj ret;
{
if ( !authConn_->runCommand( cloneOptions.fromDB,
BSON( "authenticate" << 1 << "user" << username
<< "nonce" << nonce << "key" << key ), ret ) ) {
errmsg = "unable to login " + ret.toString();
return false;
}
}
cloner.setConnection( authConn_.release() );
}
else if (cmdObj.hasField(saslCommandConversationIdFieldName) &&
cmdObj.hasField(saslCommandPayloadFieldName)) {
uassert( 25487, "must call copydbsaslstart first", authConn_.get() );
BSONObj ret;
if ( !authConn_->runCommand( cloneOptions.fromDB,
BSON( "saslContinue" << 1 <<
cmdObj[saslCommandConversationIdFieldName] <<
cmdObj[saslCommandPayloadFieldName] ),
ret ) ) {
errmsg = "unable to login " + ret.toString();
return false;
}
if (!ret["done"].Bool()) {
result.appendElements( ret );
return true;
}
result.append("done", true);
cloner.setConnection( authConn_.release() );
}
else if (!fromSelf) {
// If fromSelf leave the cloner's conn empty, it will use a DBDirectClient instead.
ConnectionString cs = ConnectionString::parse(fromhost, errmsg);
if (!cs.isValid()) {
return false;
}
DBClientBase* conn = cs.connect(errmsg);
if (!conn) {
return false;
}
cloner.setConnection(conn);
}
if (fromSelf) {
// SERVER-4328 todo lock just the two db's not everything for the fromself case
Lock::GlobalWrite lk(txn->lockState());
return cloner.go(txn, todb, fromhost, cloneOptions, NULL, errmsg);
}
Lock::DBLock lk (txn->lockState(), todb, MODE_X);
return cloner.go(txn, todb, fromhost, cloneOptions, NULL, errmsg);
}
/* slaveOk - if true it is ok if the source of the data is !ismaster.
useReplAuth - use the credentials we normally use as a replication slave for the cloning
snapshot - use $snapshot mode for copying collections. note this should not be used when it isn't required, as it will be slower.
for example repairDatabase need not use it.
*/
bool cloneFrom(const char *masterHost, string& errmsg, const string& fromdb, bool logForReplication,
bool slaveOk, bool useReplAuth, bool snapshot)
{
Cloner c;
return c.go(masterHost, errmsg, fromdb, logForReplication, slaveOk, useReplAuth, snapshot);
}
virtual bool run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj,
int,
string& errmsg,
BSONObjBuilder& result,
bool fromRepl) {
string fromhost = cmdObj.getStringField("fromhost");
bool fromSelf = fromhost.empty();
if ( fromSelf ) {
/* copy from self */
stringstream ss;
ss << "localhost:" << serverGlobalParams.port;
fromhost = ss.str();
}
CloneOptions cloneOptions;
cloneOptions.fromDB = cmdObj.getStringField("fromdb");
cloneOptions.logForRepl = !fromRepl;
cloneOptions.slaveOk = cmdObj["slaveOk"].trueValue();
cloneOptions.useReplAuth = false;
cloneOptions.snapshot = true;
cloneOptions.mayYield = true;
cloneOptions.mayBeInterrupted = false;
string todb = cmdObj.getStringField("todb");
if ( fromhost.empty() || todb.empty() || cloneOptions.fromDB.empty() ) {
errmsg = "parms missing - {copydb: 1, fromhost: <connection string>, "
"fromdb: <db>, todb: <db>}";
return false;
}
Cloner cloner;
string username = cmdObj.getStringField( "username" );
string nonce = cmdObj.getStringField( "nonce" );
string key = cmdObj.getStringField( "key" );
if ( !username.empty() && !nonce.empty() && !key.empty() ) {
uassert( 13008, "must call copydbgetnonce first", authConn_.get() );
BSONObj ret;
{
if ( !authConn_->runCommand( cloneOptions.fromDB,
BSON( "authenticate" << 1 << "user" << username
<< "nonce" << nonce << "key" << key ), ret ) ) {
errmsg = "unable to login " + ret.toString();
return false;
}
}
cloner.setConnection( authConn_.release() );
}
else if (!fromSelf) {
// If fromSelf leave the cloner's conn empty, it will use a DBDirectClient instead.
ConnectionString cs = ConnectionString::parse(fromhost, errmsg);
if (!cs.isValid()) {
return false;
}
DBClientBase* conn = cs.connect(errmsg);
if (!conn) {
return false;
}
cloner.setConnection(conn);
}
// SERVER-4328 todo lock just the two db's not everything for the fromself case
scoped_ptr<Lock::ScopedLock> lk( fromSelf ?
static_cast<Lock::ScopedLock*>(new Lock::GlobalWrite(txn->lockState())) :
static_cast<Lock::ScopedLock*>(new Lock::DBWrite(txn->lockState(), todb)));
WriteUnitOfWork wunit(txn);
if (!cloner.go(txn, todb, fromhost, cloneOptions, NULL, errmsg )) {
return false;
}
wunit.commit();
return true;
}
