virtual bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
string source = cmdObj.getStringField( name.c_str() );
string target = cmdObj.getStringField( "to" );
if ( source.empty() || target.empty() ) {
errmsg = "invalid command syntax";
return false;
}
bool capped = false;
long long size = 0;
{
Client::Context ctx( source );
NamespaceDetails *nsd = nsdetails( source.c_str() );
uassert( 10026 , "source namespace does not exist", nsd );
capped = nsd->capped;
if ( capped )
for( DiskLoc i = nsd->firstExtent; !i.isNull(); i = i.ext()->xnext )
size += i.ext()->length;
}
Client::Context ctx( target );
if ( nsdetails( target.c_str() ) ){
uassert( 10027 , "target namespace exists", cmdObj["dropTarget"].trueValue() );
BSONObjBuilder bb( result.subobjStart( "dropTarget" ) );
dropCollection( target , errmsg , bb );
bb.done();
if ( errmsg.size() > 0 )
return false;
}
{
char from[256];
nsToDatabase( source.c_str(), from );
char to[256];
nsToDatabase( target.c_str(), to );
if ( strcmp( from, to ) == 0 ) {
renameNamespace( source.c_str(), target.c_str() );
return true;
}
}
BSONObjBuilder spec;
if ( capped ) {
spec.appendBool( "capped", true );
spec.append( "size", double( size ) );
}
if ( !userCreateNS( target.c_str(), spec.done(), errmsg, false ) )
return false;
auto_ptr< DBClientCursor > c;
DBDirectClient bridge;
{
c = bridge.query( source, BSONObj() );
}
while( 1 ) {
{
if ( !c->more() )
break;
}
BSONObj o = c->next();
theDataFileMgr.insertWithObjMod( target.c_str(), o );
}
char cl[256];
nsToDatabase( source.c_str(), cl );
string sourceIndexes = string( cl ) + ".system.indexes";
nsToDatabase( target.c_str(), cl );
string targetIndexes = string( cl ) + ".system.indexes";
{
c = bridge.query( sourceIndexes, QUERY( "ns" << source ) );
}
while( 1 ) {
{
if ( !c->more() )
break;
}
BSONObj o = c->next();
BSONObjBuilder b;
BSONObjIterator i( o );
while( i.moreWithEOO() ) {
BSONElement e = i.next();
if ( e.eoo() )
break;
if ( strcmp( e.fieldName(), "ns" ) == 0 ) {
b.append( "ns", target );
} else {
b.append( e );
}
}
BSONObj n = b.done();
theDataFileMgr.insertWithObjMod( targetIndexes.c_str(), n );
}
{
Client::Context ctx( source );
dropCollection( source, errmsg, result );
}
return true;
}
Status cloneCollectionAsCapped( OperationContext* txn,
Database* db,
const string& shortFrom,
const string& shortTo,
double size,
bool temp,
bool logForReplication ) {
string fromNs = db->name() + "." + shortFrom;
string toNs = db->name() + "." + shortTo;
Collection* fromCollection = db->getCollection( txn, fromNs );
if ( !fromCollection )
return Status( ErrorCodes::NamespaceNotFound,
str::stream() << "source collection " << fromNs << " does not exist" );
if ( db->getCollection( toNs ) )
return Status( ErrorCodes::NamespaceExists, "to collection already exists" );
// create new collection
{
Client::Context ctx( toNs );
BSONObjBuilder spec;
spec.appendBool( "capped", true );
spec.append( "size", size );
if ( temp )
spec.appendBool( "temp", true );
Status status = userCreateNS( txn, ctx.db(), toNs, spec.done(), logForReplication );
if ( !status.isOK() )
return status;
}
Collection* toCollection = db->getCollection( txn, toNs );
invariant( toCollection ); // we created above
// how much data to ignore because it won't fit anyway
// datasize and extentSize can't be compared exactly, so add some padding to 'size'
long long excessSize =
static_cast<long long>( fromCollection->dataSize() -
( toCollection->getRecordStore()->storageSize() * 2 ) );
scoped_ptr<Runner> runner( InternalPlanner::collectionScan(fromNs,
fromCollection,
InternalPlanner::FORWARD ) );
while ( true ) {
BSONObj obj;
Runner::RunnerState state = runner->getNext(&obj, NULL);
switch( state ) {
case Runner::RUNNER_EOF:
return Status::OK();
case Runner::RUNNER_DEAD:
db->dropCollection( txn, toNs );
return Status( ErrorCodes::InternalError, "runner turned dead while iterating" );
case Runner::RUNNER_ERROR:
return Status( ErrorCodes::InternalError, "runner error while iterating" );
case Runner::RUNNER_ADVANCED:
if ( excessSize > 0 ) {
excessSize -= ( 4 * obj.objsize() ); // 4x is for padding, power of 2, etc...
continue;
}
toCollection->insertDocument( txn, obj, true );
if ( logForReplication )
repl::logOp(txn, "i", toNs.c_str(), obj);
txn->recoveryUnit()->commitIfNeeded();
}
}
invariant( false ); // unreachable
}
virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
string source = cmdObj.getStringField( name.c_str() );
string target = cmdObj.getStringField( "to" );
if ( source.empty() || target.empty() ) {
errmsg = "invalid command syntax";
return false;
}
setClient( source.c_str() );
NamespaceDetails *nsd = nsdetails( source.c_str() );
uassert( "source namespace does not exist", nsd );
bool capped = nsd->capped;
long long size = 0;
if ( capped )
for( DiskLoc i = nsd->firstExtent; !i.isNull(); i = i.ext()->xnext )
size += i.ext()->length;
setClient( target.c_str() );
BSONObjBuilder spec;
if ( capped ) {
spec.appendBool( "capped", true );
spec.append( "size", double( size ) );
}
if ( !userCreateNS( target.c_str(), spec.done(), errmsg, true ) )
return false;
auto_ptr< DBClientCursor > c;
DBDirectClient bridge;
{
c = bridge.query( source, BSONObj() );
}
while( 1 ) {
{
if ( !c->more() )
break;
}
BSONObj o = c->next();
theDataFileMgr.insertAndLog( target.c_str(), o );
}
char cl[256];
nsToClient( source.c_str(), cl );
string sourceIndexes = string( cl ) + ".system.indexes";
nsToClient( target.c_str(), cl );
string targetIndexes = string( cl ) + ".system.indexes";
{
c = bridge.query( sourceIndexes, QUERY( "ns" << source ) );
}
while( 1 ) {
{
if ( !c->more() )
break;
}
BSONObj o = c->next();
BSONObjBuilder b;
BSONObjIterator i( o );
while( i.moreWithEOO() ) {
BSONElement e = i.next();
if ( e.eoo() )
break;
if ( strcmp( e.fieldName(), "ns" ) == 0 ) {
b.append( "ns", target );
} else {
b.append( e );
}
}
BSONObj n = b.done();
theDataFileMgr.insertAndLog( targetIndexes.c_str(), n );
}
{
if ( !bridge.dropCollection( source ) ) {
errmsg = "failed to drop old name collection";
return false;
}
}
return true;
}
void run() {
NamespaceString source("unittests.rollback_rename_droptarget_collection_src");
NamespaceString target("unittests.rollback_rename_droptarget_collection_dest");
const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext();
OperationContext& opCtx = *opCtxPtr;
dropDatabase(&opCtx, source);
dropDatabase(&opCtx, target);
Lock::GlobalWrite globalWriteLock(&opCtx);
OldClientContext ctx(&opCtx, source.ns());
BSONObj sourceDoc = BSON("_id"
<< "source");
BSONObj targetDoc = BSON("_id"
<< "target");
{
WriteUnitOfWork uow(&opCtx);
ASSERT(!collectionExists(&ctx, source.ns()));
ASSERT(!collectionExists(&ctx, target.ns()));
auto options = capped ? BSON("capped" << true << "size" << 1000) : BSONObj();
ASSERT_OK(userCreateNS(&opCtx,
ctx.db(),
source.ns(),
options,
CollectionOptions::parseForCommand,
defaultIndexes));
ASSERT_OK(userCreateNS(&opCtx,
ctx.db(),
target.ns(),
options,
CollectionOptions::parseForCommand,
defaultIndexes));
insertRecord(&opCtx, source, sourceDoc);
insertRecord(&opCtx, target, targetDoc);
uow.commit();
}
ASSERT(collectionExists(&ctx, source.ns()));
ASSERT(collectionExists(&ctx, target.ns()));
assertOnlyRecord(&opCtx, source, sourceDoc);
assertOnlyRecord(&opCtx, target, targetDoc);
// END OF SETUP / START OF TEST
{
WriteUnitOfWork uow(&opCtx);
BSONObjBuilder result;
ASSERT_OK(
dropCollection(&opCtx,
target,
result,
{},
DropCollectionSystemCollectionMode::kDisallowSystemCollectionDrops));
ASSERT_OK(renameCollection(&opCtx, source, target));
ASSERT(!collectionExists(&ctx, source.ns()));
ASSERT(collectionExists(&ctx, target.ns()));
assertOnlyRecord(&opCtx, target, sourceDoc);
if (!rollback) {
uow.commit();
}
}
if (rollback) {
ASSERT(collectionExists(&ctx, source.ns()));
ASSERT(collectionExists(&ctx, target.ns()));
assertOnlyRecord(&opCtx, source, sourceDoc);
assertOnlyRecord(&opCtx, target, targetDoc);
} else {
ASSERT(!collectionExists(&ctx, source.ns()));
ASSERT(collectionExists(&ctx, target.ns()));
assertOnlyRecord(&opCtx, target, sourceDoc);
}
}
Status cloneCollectionAsCapped(OperationContext* txn,
Database* db,
const std::string& shortFrom,
const std::string& shortTo,
double size,
bool temp) {
std::string fromNs = db->name() + "." + shortFrom;
std::string toNs = db->name() + "." + shortTo;
Collection* fromCollection = db->getCollection(fromNs);
if (!fromCollection)
return Status(ErrorCodes::NamespaceNotFound,
str::stream() << "source collection " << fromNs << " does not exist");
if (db->getCollection(toNs))
return Status(ErrorCodes::NamespaceExists, "to collection already exists");
// create new collection
{
const auto fromOptions = fromCollection->getCatalogEntry()
->getCollectionOptions(txn)
.toBSON();
OldClientContext ctx(txn, toNs);
BSONObjBuilder spec;
spec.appendBool("capped", true);
spec.append("size", size);
if (temp)
spec.appendBool("temp", true);
spec.appendElementsUnique(fromOptions);
WriteUnitOfWork wunit(txn);
Status status = userCreateNS(txn, ctx.db(), toNs, spec.done());
if (!status.isOK())
return status;
wunit.commit();
}
Collection* toCollection = db->getCollection(toNs);
invariant(toCollection); // we created above
// how much data to ignore because it won't fit anyway
// datasize and extentSize can't be compared exactly, so add some padding to 'size'
long long allocatedSpaceGuess =
std::max(static_cast<long long>(size * 2),
static_cast<long long>(toCollection->getRecordStore()->storageSize(txn) * 2));
long long excessSize = fromCollection->dataSize(txn) - allocatedSpaceGuess;
boost::scoped_ptr<PlanExecutor> exec(InternalPlanner::collectionScan(
txn,
fromNs,
fromCollection,
InternalPlanner::FORWARD));
DisableDocumentValidation validationDisabler(txn);
while (true) {
BSONObj obj;
PlanExecutor::ExecState state = exec->getNext(&obj, NULL);
switch(state) {
case PlanExecutor::IS_EOF:
return Status::OK();
case PlanExecutor::DEAD:
db->dropCollection(txn, toNs);
return Status(ErrorCodes::InternalError, "executor turned dead while iterating");
case PlanExecutor::FAILURE:
return Status(ErrorCodes::InternalError, "executor error while iterating");
case PlanExecutor::ADVANCED:
if (excessSize > 0) {
excessSize -= (4 * obj.objsize()); // 4x is for padding, power of 2, etc...
continue;
}
WriteUnitOfWork wunit(txn);
toCollection->insertDocument(txn, obj, true, txn->writesAreReplicated());
wunit.commit();
}
}
invariant(false); // unreachable
}
void createOplog() {
dblock lk;
const char * ns = "local.oplog.$main";
bool rs = !cmdLine._replSet.empty();
if( rs )
ns = rsoplog;
Client::Context ctx(ns);
NamespaceDetails * nsd = nsdetails( ns );
if ( nsd ) {
if ( cmdLine.oplogSize != 0 ) {
int o = (int)(nsd->storageSize() / ( 1024 * 1024 ) );
int n = (int)(cmdLine.oplogSize / ( 1024 * 1024 ) );
if ( n != o ) {
stringstream ss;
ss << "cmdline oplogsize (" << n << ") different than existing (" << o << ") see: http://dochub.mongodb.org/core/increase-oplog";
log() << ss.str() << endl;
throw UserException( 13257 , ss.str() );
}
}
if( rs ) return;
DBDirectClient c;
BSONObj lastOp = c.findOne( ns, Query().sort(reverseNaturalObj) );
if ( !lastOp.isEmpty() ) {
OpTime::setLast( lastOp[ "ts" ].date() );
}
return;
}
/* create an oplog collection, if it doesn't yet exist. */
BSONObjBuilder b;
double sz;
if ( cmdLine.oplogSize != 0 )
sz = (double)cmdLine.oplogSize;
else {
/* not specified. pick a default size */
sz = 50.0 * 1000 * 1000;
if ( sizeof(int *) >= 8 ) {
#if defined(__APPLE__)
// typically these are desktops (dev machines), so keep it smallish
sz = (256-64) * 1000 * 1000;
#else
sz = 990.0 * 1000 * 1000;
boost::intmax_t free = File::freeSpace(dbpath); //-1 if call not supported.
double fivePct = free * 0.05;
if ( fivePct > sz )
sz = fivePct;
#endif
}
}
log() << "******" << endl;
log() << "creating replication oplog of size: " << (int)( sz / ( 1024 * 1024 ) ) << "MB..." << endl;
b.append("size", sz);
b.appendBool("capped", 1);
b.appendBool("autoIndexId", false);
string err;
BSONObj o = b.done();
userCreateNS(ns, o, err, false);
if( !rs )
logOp( "n", "", BSONObj() );
/* sync here so we don't get any surprising lag later when we try to sync */
MemoryMappedFile::flushAll(true);
log() << "******" << endl;
}
virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
string source = cmdObj.getStringField( name.c_str() );
string target = cmdObj.getStringField( "to" );
uassert(15967,
"invalid collection name: " + target,
NamespaceString::validCollectionComponent(target.c_str()));
if ( source.empty() || target.empty() ) {
errmsg = "invalid command syntax";
return false;
}
string sourceDB = nsToDatabase(source);
string targetDB = nsToDatabase(target);
string databaseName = sourceDB;
databaseName += ".system.indexes";
int longestIndexNameLength = 0;
vector<BSONObj> oldIndSpec = Helpers::findAll(databaseName, BSON("ns" << source));
for (size_t i = 0; i < oldIndSpec.size(); ++i) {
int thisLength = oldIndSpec[i].getField("name").valuesize();
if (thisLength > longestIndexNameLength) {
longestIndexNameLength = thisLength;
}
}
unsigned int longestAllowed = maxNamespaceLen - longestIndexNameLength - 1;
if (target.size() > longestAllowed) {
StringBuilder sb;
sb << "collection name length of " << target.size()
<< " exceeds maximum length of " << longestAllowed
<< ", allowing for index names";
uasserted(16451, sb.str());
}
bool capped = false;
long long size = 0;
std::vector<BSONObj> indexesInProg;
{
Client::Context ctx( source );
NamespaceDetails *nsd = nsdetails( source );
uassert( 10026 , "source namespace does not exist", nsd );
indexesInProg = stopIndexBuilds(dbname, cmdObj);
capped = nsd->isCapped();
if ( capped )
for( DiskLoc i = nsd->firstExtent(); !i.isNull(); i = i.ext()->xnext )
size += i.ext()->length;
}
Client::Context ctx( target );
if ( nsdetails( target ) ) {
uassert( 10027 , "target namespace exists", cmdObj["dropTarget"].trueValue() );
Status s = cc().database()->dropCollection( target );
if ( !s.isOK() ) {
errmsg = s.toString();
return false;
}
}
// if we are renaming in the same database, just
// rename the namespace and we're done.
{
if ( sourceDB == targetDB ) {
Status s = ctx.db()->renameCollection( source, target, cmdObj["stayTemp"].trueValue() );
if ( !s.isOK() ) {
errmsg = s.toString();
return false;
}
return true;
}
}
// renaming across databases, so we must copy all
// the data and then remove the source collection.
BSONObjBuilder spec;
if ( capped ) {
spec.appendBool( "capped", true );
spec.append( "size", double( size ) );
}
if ( !userCreateNS( target.c_str(), spec.done(), errmsg, false ) )
return false;
auto_ptr< DBClientCursor > c;
DBDirectClient bridge;
{
c = bridge.query( source, BSONObj(), 0, 0, 0, fromRepl ? QueryOption_SlaveOk : 0 );
}
while( 1 ) {
{
if ( !c->more() )
break;
}
BSONObj o = c->next();
theDataFileMgr.insertWithObjMod( target.c_str(), o );
}
string sourceIndexes = nsToDatabase( source ) + ".system.indexes";
string targetIndexes = nsToDatabase( target ) + ".system.indexes";
{
c = bridge.query( sourceIndexes, QUERY( "ns" << source ), 0, 0, 0, fromRepl ? QueryOption_SlaveOk : 0 );
}
while( 1 ) {
{
if ( !c->more() )
break;
}
BSONObj o = c->next();
BSONObjBuilder b;
BSONObjIterator i( o );
while( i.moreWithEOO() ) {
BSONElement e = i.next();
if ( e.eoo() )
break;
if ( strcmp( e.fieldName(), "ns" ) == 0 ) {
b.append( "ns", target );
}
else {
b.append( e );
}
}
BSONObj n = b.done();
theDataFileMgr.insertWithObjMod( targetIndexes.c_str(), n );
}
{
Client::Context ctx( source );
Status s = ctx.db()->dropCollection( source );
if ( !s.isOK() ) {
errmsg = s.toString();
return false;
}
IndexBuilder::restoreIndexes(targetIndexes, indexesInProg);
}
return true;
}
void create() {
dblock lk;
string err;
ASSERT(userCreateNS( _ns.c_str(), fromjson( spec() ), err, false ));
}
/* Prepare to build an index. Does not actually build it (except for a special _id case).
- We validate that the params are good
- That the index does not already exist
- Creates the source collection if it DNE
example of 'io':
{ ns : 'test.foo', name : 'z', key : { z : 1 } }
throws DBException
@param sourceNS - source NS we are indexing
@param sourceCollection - its details ptr
@return true if ok to continue. when false we stop/fail silently (index already exists)
*/
bool prepareToBuildIndex(const BSONObj& io, bool god, string& sourceNS, NamespaceDetails *&sourceCollection, BSONObj& fixedIndexObject ) {
sourceCollection = 0;
// logical name of the index. todo: get rid of the name, we don't need it!
const char *name = io.getStringField("name");
uassert(12523, "no index name specified", *name);
// the collection for which we are building an index
sourceNS = io.getStringField("ns");
uassert(10096, "invalid ns to index", sourceNS.find( '.' ) != string::npos);
uassert(10097, "bad table to index name on add index attempt",
cc().database()->name == nsToDatabase(sourceNS.c_str()));
BSONObj key = io.getObjectField("key");
uassert(12524, "index key pattern too large", key.objsize() <= 2048);
if( !validKeyPattern(key) ) {
string s = string("bad index key pattern ") + key.toString();
uasserted(10098 , s.c_str());
}
if ( sourceNS.empty() || key.isEmpty() ) {
log(2) << "bad add index attempt name:" << (name?name:"") << "\n ns:" <<
sourceNS << "\n idxobj:" << io.toString() << endl;
string s = "bad add index attempt " + sourceNS + " key:" + key.toString();
uasserted(12504, s);
}
sourceCollection = nsdetails(sourceNS.c_str());
if( sourceCollection == 0 ) {
// try to create it
string err;
if ( !userCreateNS(sourceNS.c_str(), BSONObj(), err, false) ) {
problem() << "ERROR: failed to create collection while adding its index. " << sourceNS << endl;
return false;
}
sourceCollection = nsdetails(sourceNS.c_str());
tlog() << "info: creating collection " << sourceNS << " on add index" << endl;
assert( sourceCollection );
}
if ( sourceCollection->findIndexByName(name) >= 0 ) {
// index already exists.
return false;
}
if( sourceCollection->findIndexByKeyPattern(key) >= 0 ) {
log(2) << "index already exists with diff name " << name << ' ' << key.toString() << endl;
return false;
}
if ( sourceCollection->nIndexes >= NamespaceDetails::NIndexesMax ) {
stringstream ss;
ss << "add index fails, too many indexes for " << sourceNS << " key:" << key.toString();
string s = ss.str();
log() << s << '\n';
uasserted(12505,s);
}
/* we can't build a new index for the ns if a build is already in progress in the background -
EVEN IF this is a foreground build.
*/
uassert(12588, "cannot add index with a background operation in progress",
!BackgroundOperation::inProgForNs(sourceNS.c_str()));
/* this is because we want key patterns like { _id : 1 } and { _id : <someobjid> } to
all be treated as the same pattern.
*/
if ( IndexDetails::isIdIndexPattern(key) ) {
if( !god ) {
ensureHaveIdIndex( sourceNS.c_str() );
return false;
}
}
else {
/* is buildIndexes:false set for this replica set member?
if so we don't build any indexes except _id
*/
if( theReplSet && !theReplSet->buildIndexes() )
return false;
}
string pluginName = IndexPlugin::findPluginName( key );
IndexPlugin * plugin = pluginName.size() ? IndexPlugin::get( pluginName ) : 0;
if ( plugin ) {
fixedIndexObject = plugin->adjustIndexSpec( io );
}
else if ( io["v"].eoo() ) {
// add "v" if it doesn't exist
// if it does - leave whatever value was there
// this is for testing and replication
BSONObjBuilder b( io.objsize() + 32 );
b.appendElements( io );
b.append( "v" , 0 );
fixedIndexObject = b.obj();
}
return true;
}
bool Cloner::go(const char *masterHost, string& errmsg, const string& fromdb, bool logForRepl, bool slaveOk, bool useReplAuth, bool snapshot) {
massert( "useReplAuth is not written to replication log", !useReplAuth || !logForRepl );
string todb = database->name;
stringstream a,b;
a << "localhost:" << cmdLine.port;
b << "127.0.0.1:" << cmdLine.port;
bool masterSameProcess = ( a.str() == masterHost || b.str() == masterHost );
if ( masterSameProcess ) {
if ( fromdb == todb && database->path == dbpath ) {
// guard against an "infinite" loop
/* if you are replicating, the local.sources config may be wrong if you get this */
errmsg = "can't clone from self (localhost).";
return false;
}
}
/* todo: we can put these releases inside dbclient or a dbclient specialization.
or just wait until we get rid of global lock anyway.
*/
string ns = fromdb + ".system.namespaces";
list<BSONObj> toClone;
{
dbtemprelease r;
auto_ptr<DBClientCursor> c;
{
if ( !masterSameProcess ) {
auto_ptr< DBClientConnection > c( new DBClientConnection() );
if ( !c->connect( masterHost, errmsg ) )
return false;
if( !replAuthenticate(c.get()) )
return false;
conn = c;
} else {
conn.reset( new DBDirectClient() );
}
c = conn->query( ns.c_str(), BSONObj(), 0, 0, 0, slaveOk ? Option_SlaveOk : 0 );
}
if ( c.get() == 0 ) {
errmsg = "query failed " + ns;
return false;
}
while ( c->more() ){
BSONObj collection = c->next();
log(2) << "\t cloner got " << collection << endl;
BSONElement e = collection.findElement("name");
if ( e.eoo() ) {
string s = "bad system.namespaces object " + collection.toString();
massert(s.c_str(), false);
}
assert( !e.eoo() );
assert( e.type() == String );
const char *from_name = e.valuestr();
if( strstr(from_name, ".system.") ) {
/* system.users is cloned -- but nothing else from system. */
if( strstr(from_name, ".system.users") == 0 ){
log(2) << "\t\t not cloning because system collection" << endl;
continue;
}
}
else if( strchr(from_name, '$') ) {
// don't clone index namespaces -- we take care of those separately below.
log(2) << "\t\t not cloning because has $ " << endl;
continue;
}
toClone.push_back( collection.getOwned() );
}
}
for ( list<BSONObj>::iterator i=toClone.begin(); i != toClone.end(); i++ ){
{
dbtemprelease r;
}
BSONObj collection = *i;
log(2) << " really will clone: " << collection << endl;
const char * from_name = collection["name"].valuestr();
BSONObj options = collection.getObjectField("options");
/* change name "<fromdb>.collection" -> <todb>.collection */
const char *p = strchr(from_name, '.');
assert(p);
string to_name = todb + p;
//if( !options.isEmpty() )
{
string err;
const char *toname = to_name.c_str();
userCreateNS(toname, options, err, logForRepl);
/* chunks are big enough that we should create the _id index up front, that should
be faster. perhaps we should do that for everything? Not doing that yet -- not sure
how we want to handle _id-less collections, and we might not want to create the index
there.
*/
if ( strstr(toname, "._chunks") )
ensureHaveIdIndex(toname);
}
log(1) << "\t\t cloning " << from_name << " -> " << to_name << endl;
Query q;
if( snapshot )
q.snapshot();
copy(from_name, to_name.c_str(), false, logForRepl, masterSameProcess, slaveOk, q);
}
// now build the indexes
string system_indexes_from = fromdb + ".system.indexes";
string system_indexes_to = todb + ".system.indexes";
/* [dm]: is the ID index sometimes not called "_id_"? There is other code in the system that looks for a "_id" prefix
rather than this exact value. we should standardize. OR, remove names - which is in the bugdb. Anyway, this
is dubious here at the moment.
*/
copy(system_indexes_from.c_str(), system_indexes_to.c_str(), true, logForRepl, masterSameProcess, slaveOk, BSON( "name" << NE << "_id_" ) );
return true;
}
Status cloneCollectionAsCapped( Database* db,
const string& shortFrom,
const string& shortTo,
double size,
bool temp,
bool logForReplication ) {
string fromNs = db->name() + "." + shortFrom;
string toNs = db->name() + "." + shortTo;
Collection* fromCollection = db->getCollection( fromNs );
if ( !fromCollection )
return Status( ErrorCodes::NamespaceNotFound,
str::stream() << "source collection " << fromNs << " does not exist" );
if ( db->getCollection( toNs ) )
return Status( ErrorCodes::NamespaceExists, "to collection already exists" );
// create new collection
{
Client::Context ctx( toNs );
BSONObjBuilder spec;
spec.appendBool( "capped", true );
spec.append( "size", size );
if ( temp )
spec.appendBool( "temp", true );
string errmsg;
if ( !userCreateNS( toNs.c_str(), spec.done(), errmsg, logForReplication ) )
return Status( ErrorCodes::InternalError, errmsg );
}
auto_ptr<Runner> runner;
{
const NamespaceDetails* details = fromCollection->details();
DiskLoc extent = details->firstExtent();
// datasize and extentSize can't be compared exactly, so add some padding to 'size'
long long excessSize =
static_cast<long long>( fromCollection->dataSize() - size * 2 );
// skip ahead some extents since not all the data fits,
// so we have to chop a bunch off
for( ;
excessSize > extent.ext()->length && extent != details->lastExtent();
extent = extent.ext()->xnext ) {
excessSize -= extent.ext()->length;
LOG( 2 ) << "cloneCollectionAsCapped skipping extent of size "
<< extent.ext()->length << endl;
LOG( 6 ) << "excessSize: " << excessSize << endl;
}
DiskLoc startLoc = extent.ext()->firstRecord;
runner.reset( InternalPlanner::collectionScan(fromNs,
InternalPlanner::FORWARD,
startLoc) );
}
Collection* toCollection = db->getCollection( toNs );
verify( toCollection );
while ( true ) {
BSONObj obj;
Runner::RunnerState state = runner->getNext(&obj, NULL);
switch( state ) {
case Runner::RUNNER_EOF:
return Status::OK();
case Runner::RUNNER_DEAD:
db->dropCollection( toNs );
return Status( ErrorCodes::InternalError, "runner turned dead while iterating" );
case Runner::RUNNER_ERROR:
return Status( ErrorCodes::InternalError, "runner error while iterating" );
case Runner::RUNNER_ADVANCED:
toCollection->insertDocument( obj, true );
if ( logForReplication )
logOp( "i", toNs.c_str(), obj );
getDur().commitIfNeeded();
}
}
verify( false ); // unreachable
}
virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
string source = cmdObj.getStringField( name.c_str() );
string target = cmdObj.getStringField( "to" );
if ( !NamespaceString::validCollectionComponent(target.c_str()) ) {
errmsg = "invalid collection name: " + target;
return false;
}
if ( source.empty() || target.empty() ) {
errmsg = "invalid command syntax";
return false;
}
if (!fromRepl) { // If it got through on the master, need to allow it here too
Status sourceStatus = userAllowedWriteNS(source);
if (!sourceStatus.isOK()) {
errmsg = "error with source namespace: " + sourceStatus.reason();
return false;
}
Status targetStatus = userAllowedWriteNS(target);
if (!targetStatus.isOK()) {
errmsg = "error with target namespace: " + targetStatus.reason();
return false;
}
}
string sourceDB = nsToDatabase(source);
string targetDB = nsToDatabase(target);
bool capped = false;
long long size = 0;
std::vector<BSONObj> indexesInProg;
{
Client::Context srcCtx( source );
Collection* sourceColl = srcCtx.db()->getCollection( source );
if ( !sourceColl ) {
errmsg = "source namespace does not exist";
return false;
}
// Ensure that collection name does not exceed maximum length.
// Ensure that index names do not push the length over the max.
// Iterator includes unfinished indexes.
IndexCatalog::IndexIterator sourceIndIt =
sourceColl->getIndexCatalog()->getIndexIterator( true );
int longestIndexNameLength = 0;
while ( sourceIndIt.more() ) {
int thisLength = sourceIndIt.next()->indexName().length();
if ( thisLength > longestIndexNameLength )
longestIndexNameLength = thisLength;
}
unsigned int longestAllowed =
min(int(Namespace::MaxNsColletionLen),
int(Namespace::MaxNsLen) - 2/*strlen(".$")*/ - longestIndexNameLength);
if (target.size() > longestAllowed) {
StringBuilder sb;
sb << "collection name length of " << target.size()
<< " exceeds maximum length of " << longestAllowed
<< ", allowing for index names";
errmsg = sb.str();
return false;
}
{
const NamespaceDetails *nsd = nsdetails( source );
indexesInProg = stopIndexBuilds( dbname, cmdObj );
capped = nsd->isCapped();
if ( capped )
for( DiskLoc i = nsd->firstExtent(); !i.isNull(); i = i.ext()->xnext )
size += i.ext()->length;
}
}
{
Client::Context ctx( target );
// Check if the target namespace exists and if dropTarget is true.
// If target exists and dropTarget is not true, return false.
if ( ctx.db()->getCollection( target ) ) {
if ( !cmdObj["dropTarget"].trueValue() ) {
errmsg = "target namespace exists";
return false;
}
Status s = cc().database()->dropCollection( target );
if ( !s.isOK() ) {
errmsg = s.toString();
restoreIndexBuildsOnSource( indexesInProg, source );
return false;
}
}
// If we are renaming in the same database, just
// rename the namespace and we're done.
if ( sourceDB == targetDB ) {
Status s = ctx.db()->renameCollection( source, target,
cmdObj["stayTemp"].trueValue() );
if ( !s.isOK() ) {
errmsg = s.toString();
restoreIndexBuildsOnSource( indexesInProg, source );
return false;
}
return true;
}
// Otherwise, we are enaming across databases, so we must copy all
// the data and then remove the source collection.
// Create the target collection.
Collection* targetColl = NULL;
if ( capped ) {
BSONObjBuilder spec;
spec.appendBool( "capped", true );
spec.append( "size", double( size ) );
spec.appendBool( "autoIndexId", false );
userCreateNS( target.c_str(), spec.obj(), errmsg, false );
targetColl = ctx.db()->getCollection( target );
}
else {
CollectionOptions options;
options.setNoIdIndex();
// No logOp necessary because the entire renameCollection command is one logOp.
targetColl = ctx.db()->createCollection( target, options );
}
if ( !targetColl ) {
errmsg = "Failed to create target collection.";
restoreIndexBuildsOnSource( indexesInProg, source );
return false;
}
}
// Copy over all the data from source collection to target collection.
bool insertSuccessful = true;
boost::scoped_ptr<CollectionIterator> sourceIt;
{
Client::Context srcCtx( source );
Collection* sourceColl = srcCtx.db()->getCollection( source );
sourceIt.reset( sourceColl->getIterator( DiskLoc(), false, CollectionScanParams::FORWARD ) );
}
Collection* targetColl = NULL;
while ( !sourceIt->isEOF() ) {
BSONObj o;
{
Client::Context srcCtx( source );
o = sourceIt->getNext().obj();
}
// Insert and check return status of insert.
{
Client::Context ctx( target );
if ( !targetColl )
targetColl = ctx.db()->getCollection( target );
// No logOp necessary because the entire renameCollection command is one logOp.
Status s = targetColl->insertDocument( o, true ).getStatus();
if ( !s.isOK() ) {
insertSuccessful = false;
errmsg = s.toString();
break;
}
}
}
// If inserts were unsuccessful, drop the target collection and return false.
if ( !insertSuccessful ) {
Client::Context ctx( target );
Status s = ctx.db()->dropCollection( target );
if ( !s.isOK() )
errmsg = s.toString();
restoreIndexBuildsOnSource( indexesInProg, source );
return false;
}
// Copy over the indexes to temp storage and then to the target..
vector<BSONObj> copiedIndexes;
bool indexSuccessful = true;
{
Client::Context srcCtx( source );
Collection* sourceColl = srcCtx.db()->getCollection( source );
IndexCatalog::IndexIterator sourceIndIt =
sourceColl->getIndexCatalog()->getIndexIterator( true );
while ( sourceIndIt.more() ) {
BSONObj currIndex = sourceIndIt.next()->infoObj();
// Process the source index.
BSONObjBuilder b;
BSONObjIterator i( currIndex );
while( i.moreWithEOO() ) {
BSONElement e = i.next();
if ( e.eoo() )
break;
else if ( strcmp( e.fieldName(), "ns" ) == 0 )
b.append( "ns", target );
else
b.append( e );
}
BSONObj newIndex = b.obj();
copiedIndexes.push_back( newIndex );
}
}
{
Client::Context ctx( target );
if ( !targetColl )
targetColl = ctx.db()->getCollection( target );
for ( vector<BSONObj>::iterator it = copiedIndexes.begin();
it != copiedIndexes.end(); ++it ) {
Status s = targetColl->getIndexCatalog()->createIndex( *it, true );
if ( !s.isOK() ) {
indexSuccessful = false;
errmsg = s.toString();
break;
}
}
// If indexes were unsuccessful, drop the target collection and return false.
if ( !indexSuccessful ) {
Status s = ctx.db()->dropCollection( target );
if ( !s.isOK() )
errmsg = s.toString();
restoreIndexBuildsOnSource( indexesInProg, source );
return false;
}
}
// Drop the source collection.
{
Client::Context srcCtx( source );
Status s = srcCtx.db()->dropCollection( source );
if ( !s.isOK() ) {
errmsg = s.toString();
restoreIndexBuildsOnSource( indexesInProg, source );
return false;
}
}
return true;
}
void run() {
NamespaceString nss("unittests.rollback_replace_collection");
const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext();
OperationContext& opCtx = *opCtxPtr;
dropDatabase(&opCtx, nss);
Lock::DBLock dbXLock(&opCtx, nss.db(), MODE_X);
OldClientContext ctx(&opCtx, nss.ns());
BSONObj oldDoc = BSON("_id"
<< "old");
BSONObj newDoc = BSON("_id"
<< "new");
{
WriteUnitOfWork uow(&opCtx);
ASSERT(!collectionExists(&ctx, nss.ns()));
ASSERT_OK(userCreateNS(&opCtx,
ctx.db(),
nss.ns(),
BSONObj(),
CollectionOptions::parseForCommand,
defaultIndexes));
insertRecord(&opCtx, nss, oldDoc);
uow.commit();
}
ASSERT(collectionExists(&ctx, nss.ns()));
assertOnlyRecord(&opCtx, nss, oldDoc);
// END OF SETUP / START OF TEST
{
WriteUnitOfWork uow(&opCtx);
BSONObjBuilder result;
ASSERT_OK(
dropCollection(&opCtx,
nss,
result,
{},
DropCollectionSystemCollectionMode::kDisallowSystemCollectionDrops));
ASSERT(!collectionExists(&ctx, nss.ns()));
ASSERT_OK(userCreateNS(&opCtx,
ctx.db(),
nss.ns(),
BSONObj(),
CollectionOptions::parseForCommand,
defaultIndexes));
ASSERT(collectionExists(&ctx, nss.ns()));
insertRecord(&opCtx, nss, newDoc);
assertOnlyRecord(&opCtx, nss, newDoc);
if (!rollback) {
uow.commit();
}
}
ASSERT(collectionExists(&ctx, nss.ns()));
if (rollback) {
assertOnlyRecord(&opCtx, nss, oldDoc);
} else {
assertOnlyRecord(&opCtx, nss, newDoc);
}
}
void create() {
Client::Context c(_cappedNs);
OperationContextImpl txn;
ASSERT( userCreateNS( &txn, c.db(), _cappedNs, fromjson( spec() ), false ).isOK() );
}
/* Prepare to build an index. Does not actually build it (except for a special _id case).
- We validate that the params are good
- That the index does not already exist
- Creates the source collection if it DNE
example of 'io':
{ ns : 'test.foo', name : 'z', key : { z : 1 } }
throws DBException
@return
true if ok to continue. when false we stop/fail silently (index already exists)
sourceNS - source NS we are indexing
sourceCollection - its details ptr
*/
bool prepareToBuildIndex(const BSONObj& io, bool god, string& sourceNS, NamespaceDetails *&sourceCollection) {
sourceCollection = 0;
// logical name of the index. todo: get rid of the name, we don't need it!
const char *name = io.getStringField("name");
uassert(12523, "no index name specified", *name);
// the collection for which we are building an index
sourceNS = io.getStringField("ns");
uassert(10096, "invalid ns to index", sourceNS.find( '.' ) != string::npos);
uassert(10097, "bad table to index name on add index attempt",
cc().database()->name == nsToDatabase(sourceNS.c_str()));
/* we can't build a new index for the ns if a build is already in progress in the background -
EVEN IF this is a foreground build.
*/
uassert(12588, "cannot add index with a background operation in progress",
!BackgroundOperation::inProgForNs(sourceNS.c_str()));
BSONObj key = io.getObjectField("key");
uassert(12524, "index key pattern too large", key.objsize() <= 2048);
if( !validKeyPattern(key) ) {
string s = string("bad index key pattern ") + key.toString();
uasserted(10098 , s.c_str());
}
if ( sourceNS.empty() || key.isEmpty() ) {
log(2) << "bad add index attempt name:" << (name?name:"") << "\n ns:" <<
sourceNS << "\n idxobj:" << io.toString() << endl;
string s = "bad add index attempt " + sourceNS + " key:" + key.toString();
uasserted(12504, s);
}
sourceCollection = nsdetails(sourceNS.c_str());
if( sourceCollection == 0 ) {
// try to create it
string err;
if ( !userCreateNS(sourceNS.c_str(), BSONObj(), err, false) ) {
problem() << "ERROR: failed to create collection while adding its index. " << sourceNS << endl;
return false;
}
sourceCollection = nsdetails(sourceNS.c_str());
log() << "info: creating collection " << sourceNS << " on add index\n";
assert( sourceCollection );
}
if ( sourceCollection->findIndexByName(name) >= 0 ) {
// index already exists.
return false;
}
if( sourceCollection->findIndexByKeyPattern(key) >= 0 ) {
log(2) << "index already exists with diff name " << name << ' ' << key.toString() << endl;
return false;
}
if ( sourceCollection->nIndexes >= NamespaceDetails::NIndexesMax ) {
stringstream ss;
ss << "add index fails, too many indexes for " << sourceNS << " key:" << key.toString();
string s = ss.str();
log() << s << '\n';
uasserted(12505,s);
}
/* this is because we want key patterns like { _id : 1 } and { _id : <someobjid> } to
all be treated as the same pattern.
*/
if ( !god && IndexDetails::isIdIndexPattern(key) ) {
ensureHaveIdIndex( sourceNS.c_str() );
return false;
}
return true;
}
bool run(const char *dbname, BSONObj& cmd, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
Timer t;
Client::GodScope cg;
Client& client = cc();
CurOp * op = client.curop();
MRSetup mr( nsToDatabase( dbname ) , cmd );
log(1) << "mr ns: " << mr.ns << endl;
if ( ! db.exists( mr.ns ) ){
errmsg = "ns doesn't exist";
return false;
}
bool shouldHaveData = false;
long long num = 0;
long long inReduce = 0;
BSONObjBuilder countsBuilder;
BSONObjBuilder timingBuilder;
try {
MRState state( mr );
state.scope->injectNative( "emit" , fast_emit );
MRTL * mrtl = new MRTL( state );
_tlmr.reset( mrtl );
ProgressMeter & pm = op->setMessage( "m/r: (1/3) emit phase" , db.count( mr.ns , mr.filter ) );
long long mapTime = 0;
{
readlock lock( mr.ns );
Client::Context ctx( mr.ns );
auto_ptr<Cursor> temp = QueryPlanSet(mr.ns.c_str() , mr.filter , BSONObj() ).getBestGuess()->newCursor();
auto_ptr<ClientCursor> cursor( new ClientCursor( QueryOption_NoCursorTimeout , temp , mr.ns.c_str() ) );
if ( ! mr.filter.isEmpty() )
cursor->matcher.reset( new CoveredIndexMatcher( mr.filter , cursor->indexKeyPattern() ) );
Timer mt;
while ( cursor->ok() ){
if ( ! cursor->currentMatches() ){
cursor->advance();
continue;
}
BSONObj o = cursor->current();
cursor->advance();
if ( mr.verbose ) mt.reset();
state.scope->setThis( &o );
if ( state.scope->invoke( state.map , state.setup.mapparams , 0 , true ) )
throw UserException( 9014, (string)"map invoke failed: " + state.scope->getError() );
if ( mr.verbose ) mapTime += mt.micros();
num++;
if ( num % 100 == 0 ){
ClientCursor::YieldLock yield = cursor->yieldHold();
Timer t;
mrtl->checkSize();
inReduce += t.micros();
if ( ! yield.stillOk() ){
cursor.release();
break;
}
killCurrentOp.checkForInterrupt();
}
pm.hit();
if ( mr.limit && num >= mr.limit )
break;
}
}
pm.finished();
killCurrentOp.checkForInterrupt();
countsBuilder.appendNumber( "input" , num );
countsBuilder.appendNumber( "emit" , mrtl->numEmits );
if ( mrtl->numEmits )
shouldHaveData = true;
timingBuilder.append( "mapTime" , mapTime / 1000 );
timingBuilder.append( "emitLoop" , t.millis() );
// final reduce
op->setMessage( "m/r: (2/3) final reduce in memory" );
mrtl->reduceInMemory();
mrtl->dump();
BSONObj sortKey = BSON( "0" << 1 );
db.ensureIndex( mr.incLong , sortKey );
{
writelock lock( mr.tempLong.c_str() );
Client::Context ctx( mr.tempLong.c_str() );
assert( userCreateNS( mr.tempLong.c_str() , BSONObj() , errmsg , mr.replicate ) );
}
{
readlock rl(mr.incLong.c_str());
Client::Context ctx( mr.incLong );
BSONObj prev;
BSONList all;
pm = op->setMessage( "m/r: (3/3) final reduce to collection" , db.count( mr.incLong ) );
auto_ptr<Cursor> temp = QueryPlanSet(mr.incLong.c_str() , BSONObj() , sortKey ).getBestGuess()->newCursor();
auto_ptr<ClientCursor> cursor( new ClientCursor( QueryOption_NoCursorTimeout , temp , mr.incLong.c_str() ) );
while ( cursor->ok() ){
BSONObj o = cursor->current().getOwned();
cursor->advance();
pm.hit();
if ( o.woSortOrder( prev , sortKey ) == 0 ){
all.push_back( o );
if ( pm.hits() % 1000 == 0 ){
if ( ! cursor->yield() ){
cursor.release();
break;
}
killCurrentOp.checkForInterrupt();
}
continue;
}
ClientCursor::YieldLock yield = cursor->yieldHold();
state.finalReduce( all );
all.clear();
prev = o;
all.push_back( o );
if ( ! yield.stillOk() ){
cursor.release();
break;
}
killCurrentOp.checkForInterrupt();
}
{
dbtemprelease tl;
state.finalReduce( all );
}
pm.finished();
}
_tlmr.reset( 0 );
}
catch ( ... ){
log() << "mr failed, removing collection" << endl;
db.dropCollection( mr.tempLong );
db.dropCollection( mr.incLong );
throw;
}
long long finalCount = 0;
{
dblock lock;
db.dropCollection( mr.incLong );
finalCount = mr.renameIfNeeded( db );
}
timingBuilder.append( "total" , t.millis() );
result.append( "result" , mr.finalShort );
result.append( "timeMillis" , t.millis() );
countsBuilder.appendNumber( "output" , finalCount );
if ( mr.verbose ) result.append( "timing" , timingBuilder.obj() );
result.append( "counts" , countsBuilder.obj() );
if ( finalCount == 0 && shouldHaveData ){
result.append( "cmd" , cmd );
errmsg = "there were emits but no data!";
return false;
}
return true;
}
void create() {
Lock::GlobalWrite lk;
DurTransaction txn;
ASSERT( userCreateNS( &txn, db(), ns(), fromjson( spec() ), false ).isOK() );
}
Status cloneCollectionAsCapped(OperationContext* txn,
Database* db,
const std::string& shortFrom,
const std::string& shortTo,
double size,
bool temp) {
std::string fromNs = db->name() + "." + shortFrom;
std::string toNs = db->name() + "." + shortTo;
Collection* fromCollection = db->getCollection(fromNs);
if (!fromCollection)
return Status(ErrorCodes::NamespaceNotFound,
str::stream() << "source collection " << fromNs << " does not exist");
if (db->getCollection(toNs))
return Status(ErrorCodes::NamespaceExists, "to collection already exists");
// create new collection
MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
const auto fromOptions =
fromCollection->getCatalogEntry()->getCollectionOptions(txn).toBSON();
OldClientContext ctx(txn, toNs);
BSONObjBuilder spec;
spec.appendBool("capped", true);
spec.append("size", size);
if (temp)
spec.appendBool("temp", true);
spec.appendElementsUnique(fromOptions);
WriteUnitOfWork wunit(txn);
Status status = userCreateNS(txn, ctx.db(), toNs, spec.done());
if (!status.isOK())
return status;
wunit.commit();
}
MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "cloneCollectionAsCapped", fromNs);
Collection* toCollection = db->getCollection(toNs);
invariant(toCollection); // we created above
// how much data to ignore because it won't fit anyway
// datasize and extentSize can't be compared exactly, so add some padding to 'size'
long long allocatedSpaceGuess =
std::max(static_cast<long long>(size * 2),
static_cast<long long>(toCollection->getRecordStore()->storageSize(txn) * 2));
long long excessSize = fromCollection->dataSize(txn) - allocatedSpaceGuess;
std::unique_ptr<PlanExecutor> exec(
InternalPlanner::collectionScan(txn, fromNs, fromCollection, InternalPlanner::FORWARD));
exec->setYieldPolicy(PlanExecutor::WRITE_CONFLICT_RETRY_ONLY);
Snapshotted<BSONObj> objToClone;
RecordId loc;
PlanExecutor::ExecState state = PlanExecutor::FAILURE; // suppress uninitialized warnings
DisableDocumentValidation validationDisabler(txn);
int retries = 0; // non-zero when retrying our last document.
while (true) {
if (!retries) {
state = exec->getNextSnapshotted(&objToClone, &loc);
}
switch (state) {
case PlanExecutor::IS_EOF:
return Status::OK();
case PlanExecutor::ADVANCED: {
if (excessSize > 0) {
// 4x is for padding, power of 2, etc...
excessSize -= (4 * objToClone.value().objsize());
continue;
}
break;
}
default:
// Unreachable as:
// 1) We require a read lock (at a minimum) on the "from" collection
// and won't yield, preventing collection drop and PlanExecutor::DEAD
// 2) PlanExecutor::FAILURE is only returned on PlanStage::FAILURE. The
// CollectionScan PlanStage does not have a FAILURE scenario.
// 3) All other PlanExecutor states are handled above
invariant(false);
}
try {
// Make sure we are working with the latest version of the document.
if (objToClone.snapshotId() != txn->recoveryUnit()->getSnapshotId() &&
!fromCollection->findDoc(txn, loc, &objToClone)) {
// doc was deleted so don't clone it.
retries = 0;
continue;
}
WriteUnitOfWork wunit(txn);
toCollection->insertDocument(txn, objToClone.value(), true, txn->writesAreReplicated());
wunit.commit();
// Go to the next document
retries = 0;
} catch (const WriteConflictException& wce) {
CurOp::get(txn)->debug().writeConflicts++;
retries++; // logAndBackoff expects this to be 1 on first call.
wce.logAndBackoff(retries, "cloneCollectionAsCapped", fromNs);
// Can't use WRITE_CONFLICT_RETRY_LOOP macros since we need to save/restore exec
// around call to abandonSnapshot.
exec->saveState();
txn->recoveryUnit()->abandonSnapshot();
exec->restoreState(txn); // Handles any WCEs internally.
}
}
invariant(false); // unreachable
}
void create() {
Client::Context c(_ns);
string err;
ASSERT(userCreateNS( _ns.c_str(), fromjson( spec() ), err, false ));
}
/* Prepare to build an index. Does not actually build it (except for a special _id case).
- We validate that the params are good
- That the index does not already exist
- Creates the source collection if it DNE
example of 'io':
{ ns : 'test.foo', name : 'z', key : { z : 1 } }
throws DBException
@param sourceNS - source NS we are indexing
@param sourceCollection - its details ptr
@return true if ok to continue. when false we stop/fail silently (index already exists)
*/
bool prepareToBuildIndex(const BSONObj& io, bool god, string& sourceNS, NamespaceDetails *&sourceCollection, BSONObj& fixedIndexObject ) {
sourceCollection = 0;
// logical name of the index. todo: get rid of the name, we don't need it!
const char *name = io.getStringField("name");
uassert(12523, "no index name specified", *name);
// the collection for which we are building an index
sourceNS = io.getStringField("ns");
uassert(10096, "invalid ns to index", sourceNS.find( '.' ) != string::npos);
uassert(10097, "bad table to index name on add index attempt",
cc().database()->name == nsToDatabase(sourceNS.c_str()));
BSONObj key = io.getObjectField("key");
uassert(12524, "index key pattern too large", key.objsize() <= 2048);
if( !validKeyPattern(key) ) {
string s = string("bad index key pattern ") + key.toString();
uasserted(10098 , s.c_str());
}
if ( sourceNS.empty() || key.isEmpty() ) {
log(2) << "bad add index attempt name:" << (name?name:"") << "\n ns:" <<
sourceNS << "\n idxobj:" << io.toString() << endl;
string s = "bad add index attempt " + sourceNS + " key:" + key.toString();
uasserted(12504, s);
}
sourceCollection = nsdetails(sourceNS.c_str());
if( sourceCollection == 0 ) {
// try to create it
string err;
if ( !userCreateNS(sourceNS.c_str(), BSONObj(), err, false) ) {
problem() << "ERROR: failed to create collection while adding its index. " << sourceNS << endl;
return false;
}
sourceCollection = nsdetails(sourceNS.c_str());
tlog() << "info: creating collection " << sourceNS << " on add index" << endl;
assert( sourceCollection );
}
if ( sourceCollection->findIndexByName(name) >= 0 ) {
// index already exists.
return false;
}
if( sourceCollection->findIndexByKeyPattern(key) >= 0 ) {
log(2) << "index already exists with diff name " << name << ' ' << key.toString() << endl;
return false;
}
if ( sourceCollection->nIndexes >= NamespaceDetails::NIndexesMax ) {
stringstream ss;
ss << "add index fails, too many indexes for " << sourceNS << " key:" << key.toString();
string s = ss.str();
log() << s << '\n';
uasserted(12505,s);
}
/* we can't build a new index for the ns if a build is already in progress in the background -
EVEN IF this is a foreground build.
*/
uassert(12588, "cannot add index with a background operation in progress",
!BackgroundOperation::inProgForNs(sourceNS.c_str()));
/* this is because we want key patterns like { _id : 1 } and { _id : <someobjid> } to
all be treated as the same pattern.
*/
if ( IndexDetails::isIdIndexPattern(key) ) {
if( !god ) {
ensureHaveIdIndex( sourceNS.c_str() );
return false;
}
}
else {
/* is buildIndexes:false set for this replica set member?
if so we don't build any indexes except _id
*/
if( theReplSet && !theReplSet->buildIndexes() )
return false;
}
string pluginName = IndexPlugin::findPluginName( key );
IndexPlugin * plugin = pluginName.size() ? IndexPlugin::get( pluginName ) : 0;
{
BSONObj o = io;
if ( plugin ) {
o = plugin->adjustIndexSpec(o);
}
BSONObjBuilder b;
int v = DefaultIndexVersionNumber;
if( !o["v"].eoo() ) {
double vv = o["v"].Number();
// note (one day) we may be able to fresh build less versions than we can use
// isASupportedIndexVersionNumber() is what we can use
uassert(14803, str::stream() << "this version of mongod cannot build new indexes of version number " << vv,
vv == 0 || vv == 1);
v = (int) vv;
}
// idea is to put things we use a lot earlier
b.append("v", v);
b.append(o["key"]);
if( o["unique"].trueValue() )
b.appendBool("unique", true); // normalize to bool true in case was int 1 or something...
b.append(o["ns"]);
{
// stripping _id
BSONObjIterator i(o);
while ( i.more() ) {
BSONElement e = i.next();
string s = e.fieldName();
if( s != "_id" && s != "v" && s != "ns" && s != "unique" && s != "key" )
b.append(e);
}
}
fixedIndexObject = b.obj();
}
return true;
}
bool Cloner::go(const char *masterHost, string& errmsg, const string& fromdb, bool logForRepl, bool slaveOk, bool useReplAuth, bool snapshot) {
massert( 10289 , "useReplAuth is not written to replication log", !useReplAuth || !logForRepl );
string todb = cc().database()->name;
stringstream a,b;
a << "localhost:" << cmdLine.port;
b << "127.0.0.1:" << cmdLine.port;
bool masterSameProcess = ( a.str() == masterHost || b.str() == masterHost );
if ( masterSameProcess ) {
if ( fromdb == todb && cc().database()->path == dbpath ) {
// guard against an "infinite" loop
/* if you are replicating, the local.sources config may be wrong if you get this */
errmsg = "can't clone from self (localhost).";
return false;
}
}
/* todo: we can put these releases inside dbclient or a dbclient specialization.
or just wait until we get rid of global lock anyway.
*/
string ns = fromdb + ".system.namespaces";
list<BSONObj> toClone;
{
dbtemprelease r;
auto_ptr<DBClientCursor> c;
{
if ( conn.get() ) {
// nothing to do
} else if ( !masterSameProcess ) {
auto_ptr< DBClientConnection > c( new DBClientConnection() );
if ( !c->connect( masterHost, errmsg ) )
return false;
if( !replAuthenticate(c.get()) )
return false;
conn = c;
} else {
conn.reset( new DBDirectClient() );
}
c = conn->query( ns.c_str(), BSONObj(), 0, 0, 0, slaveOk ? QueryOption_SlaveOk : 0 );
}
if ( c.get() == 0 ) {
errmsg = "query failed " + ns;
return false;
}
while ( c->more() ){
BSONObj collection = c->next();
log(2) << "\t cloner got " << collection << endl;
BSONElement e = collection.getField("name");
if ( e.eoo() ) {
string s = "bad system.namespaces object " + collection.toString();
massert( 10290 , s.c_str(), false);
}
assert( !e.eoo() );
assert( e.type() == String );
const char *from_name = e.valuestr();
if( strstr(from_name, ".system.") ) {
/* system.users is cloned -- but nothing else from system. */
if( legalClientSystemNS( from_name , true ) == 0 ){
log(2) << "\t\t not cloning because system collection" << endl;
continue;
}
}
if( ! nsDollarCheck( from_name ) ){
log(2) << "\t\t not cloning because has $ " << endl;
continue;
}
toClone.push_back( collection.getOwned() );
}
}
for ( list<BSONObj>::iterator i=toClone.begin(); i != toClone.end(); i++ ){
{
dbtemprelease r;
}
BSONObj collection = *i;
log(2) << " really will clone: " << collection << endl;
const char * from_name = collection["name"].valuestr();
BSONObj options = collection.getObjectField("options");
/* change name "<fromdb>.collection" -> <todb>.collection */
const char *p = strchr(from_name, '.');
assert(p);
string to_name = todb + p;
bool wantIdIndex = false;
{
string err;
const char *toname = to_name.c_str();
/* we defer building id index for performance - building it in batch is much faster */
userCreateNS(toname, options, err, logForRepl, &wantIdIndex);
}
log(1) << "\t\t cloning " << from_name << " -> " << to_name << endl;
Query q;
if( snapshot )
q.snapshot();
copy(from_name, to_name.c_str(), false, logForRepl, masterSameProcess, slaveOk, q);
if( wantIdIndex ) {
/* we need dropDups to be true as we didn't do a true snapshot and this is before applying oplog operations
that occur during the initial sync. inDBRepair makes dropDups be true.
*/
bool old = inDBRepair;
try {
inDBRepair = true;
ensureIdIndexForNewNs(to_name.c_str());
inDBRepair = old;
}
catch(...) {
inDBRepair = old;
throw;
}
}
}
// now build the indexes
string system_indexes_from = fromdb + ".system.indexes";
string system_indexes_to = todb + ".system.indexes";
/* [dm]: is the ID index sometimes not called "_id_"? There is other code in the system that looks for a "_id" prefix
rather than this exact value. we should standardize. OR, remove names - which is in the bugdb. Anyway, this
is dubious here at the moment.
*/
copy(system_indexes_from.c_str(), system_indexes_to.c_str(), true, logForRepl, masterSameProcess, slaveOk, BSON( "name" << NE << "_id_" ) );
return true;
}
bool Cloner::go(const char *masterHost, string& errmsg, const string& fromdb, bool logForRepl, bool slaveOk, bool useReplAuth) {
massert( "useReplAuth is not written to replication log", !useReplAuth || !logForRepl );
string todb = database->name;
stringstream a,b;
a << "localhost:" << port;
b << "127.0.0.1:" << port;
bool masterSameProcess = ( a.str() == masterHost || b.str() == masterHost );
if ( masterSameProcess ) {
if ( fromdb == todb && database->path == dbpath ) {
// guard against an "infinite" loop
/* if you are replicating, the local.sources config may be wrong if you get this */
errmsg = "can't clone from self (localhost).";
return false;
}
}
/* todo: we can put thesee releases inside dbclient or a dbclient specialization.
or just wait until we get rid of global lock anyway.
*/
string ns = fromdb + ".system.namespaces";
auto_ptr<DBClientCursor> c;
{
dbtemprelease r;
if ( !masterSameProcess ) {
auto_ptr< DBClientConnection > c( new DBClientConnection() );
if ( !c->connect( masterHost, errmsg ) )
return false;
if( !replAuthenticate(c.get()) )
return false;
conn = c;
} else {
conn.reset( new DBDirectClient() );
}
c = conn->query( ns.c_str(), BSONObj(), 0, 0, 0, slaveOk ? Option_SlaveOk : 0 );
}
if ( c.get() == 0 ) {
errmsg = "query failed " + ns;
return false;
}
while ( 1 ) {
{
dbtemprelease r;
if ( !c->more() )
break;
}
BSONObj collection = c->next();
BSONElement e = collection.findElement("name");
if ( e.eoo() ) {
string s = "bad system.namespaces object " + collection.toString();
/* temp
out() << masterHost << endl;
out() << ns << endl;
out() << e.toString() << endl;
exit(1);*/
massert(s.c_str(), false);
}
assert( !e.eoo() );
assert( e.type() == String );
const char *from_name = e.valuestr();
if( strstr(from_name, ".system.") ) {
/* system.users is cloned -- but nothing else from system. */
if( strstr(from_name, ".system.users") == 0 )
continue;
}
else if( strchr(from_name, '$') ) {
// don't clone index namespaces -- we take care of those separately below.
continue;
}
BSONObj options = collection.getObjectField("options");
/* change name "<fromdb>.collection" -> <todb>.collection */
const char *p = strchr(from_name, '.');
assert(p);
string to_name = todb + p;
//if( !options.isEmpty() )
{
string err;
const char *toname = to_name.c_str();
userCreateNS(toname, options, err, logForRepl);
/* chunks are big enough that we should create the _id index up front, that should
be faster. perhaps we should do that for everything? Not doing that yet -- not sure
how we want to handle _id-less collections, and we might not want to create the index
there.
*/
if ( strstr(toname, "._chunks") )
ensureHaveIdIndex(toname);
}
copy(from_name, to_name.c_str(), false, logForRepl, masterSameProcess, slaveOk);
}
// now build the indexes
string system_indexes_from = fromdb + ".system.indexes";
string system_indexes_to = todb + ".system.indexes";
copy(system_indexes_from.c_str(), system_indexes_to.c_str(), true, logForRepl, masterSameProcess, slaveOk, BSON( "name" << NE << "_id_" ) );
return true;
}
bool Cloner::startCloneCollection( const char *fromhost, const char *ns, const BSONObj &query, string &errmsg, bool logForRepl, bool copyIndexes, int logSizeMb, long long &cursorId ) {
char db[256];
nsToDatabase( ns, db );
NamespaceDetails *nsd = nsdetails( ns );
if ( nsd ){
/** note: its ok to clone into a collection, but only if the range you're copying
doesn't exist on this server */
string err;
if ( runCount( ns , BSON( "query" << query ) , err ) > 0 ){
log() << "WARNING: data already exists for: " << ns << " in range : " << query << " deleting..." << endl;
deleteObjects( ns , query , false , logForRepl , false );
}
}
{
dbtemprelease r;
auto_ptr< DBClientConnection > c( new DBClientConnection() );
if ( !c->connect( fromhost, errmsg ) )
return false;
if( !replAuthenticate(c.get()) )
return false;
conn = c;
// Start temporary op log
BSONObjBuilder cmdSpec;
cmdSpec << "logCollection" << ns << "start" << 1;
if ( logSizeMb != INT_MIN )
cmdSpec << "logSizeMb" << logSizeMb;
BSONObj info;
if ( !conn->runCommand( db, cmdSpec.done(), info ) ) {
errmsg = "logCollection failed: " + (string)info;
return false;
}
}
if ( ! nsd ) {
BSONObj spec = conn->findOne( string( db ) + ".system.namespaces", BSON( "name" << ns ) );
if ( !userCreateNS( ns, spec.getObjectField( "options" ), errmsg, true ) )
return false;
}
copy( ns, ns, false, logForRepl, false, false, query );
if ( copyIndexes ) {
string indexNs = string( db ) + ".system.indexes";
copy( indexNs.c_str(), indexNs.c_str(), true, logForRepl, false, false, BSON( "ns" << ns << "name" << NE << "_id_" ) );
}
auto_ptr< DBClientCursor > c;
{
dbtemprelease r;
string logNS = "local.temp.oplog." + string( ns );
c = conn->query( logNS.c_str(), Query(), 0, 0, 0, QueryOption_CursorTailable );
}
if ( c->more() ) {
replayOpLog( c.get(), query );
cursorId = c->getCursorId();
massert( 10291 , "Expected valid tailing cursor", cursorId != 0 );
} else {
massert( 10292 , "Did not expect valid cursor for empty query result", c->getCursorId() == 0 );
cursorId = 0;
}
c->decouple();
return true;
}
virtual bool run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj,
int options,
string& errmsg,
BSONObjBuilder& result,
bool fromRepl) {
const std::string ns = parseNsCollectionRequired(dbname, cmdObj);
const BSONObj query = cmdObj.getObjectField("query");
const BSONObj fields = cmdObj.getObjectField("fields");
const BSONObj update = cmdObj.getObjectField("update");
const BSONObj sort = cmdObj.getObjectField("sort");
bool upsert = cmdObj["upsert"].trueValue();
bool returnNew = cmdObj["new"].trueValue();
bool remove = cmdObj["remove"].trueValue();
if ( remove ) {
if ( upsert ) {
errmsg = "remove and upsert can't co-exist";
return false;
}
if ( !update.isEmpty() ) {
errmsg = "remove and update can't co-exist";
return false;
}
if ( returnNew ) {
errmsg = "remove and returnNew can't co-exist";
return false;
}
}
else if ( !cmdObj.hasField("update") ) {
errmsg = "need remove or update";
return false;
}
bool ok = false;
MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
errmsg = "";
// We can always retry because we only ever modify one document
ok = runImpl(txn,
dbname,
ns,
query,
fields,
update,
sort,
upsert,
returnNew,
remove,
result,
errmsg);
} MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "findAndModify", ns);
if ( !ok && errmsg == "no-collection" ) {
// Take X lock so we can create collection, then re-run operation.
ScopedTransaction transaction(txn, MODE_IX);
Lock::DBLock lk(txn->lockState(), dbname, MODE_X);
Client::Context ctx(txn, ns, false /* don't check version */);
if (!fromRepl &&
!repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase(dbname)) {
return appendCommandStatus(result, Status(ErrorCodes::NotMaster, str::stream()
<< "Not primary while creating collection " << ns
<< " during findAndModify"));
}
Database* db = ctx.db();
if ( db->getCollection( ns ) ) {
// someone else beat us to it, that's ok
// we might race while we unlock if someone drops
// but that's ok, we'll just do nothing and error out
}
else {
MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
WriteUnitOfWork wuow(txn);
uassertStatusOK( userCreateNS( txn, db,
ns, BSONObj(),
!fromRepl ) );
wuow.commit();
} MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "findAndModify", ns);
}
errmsg = "";
ok = runImpl(txn,
dbname,
ns,
query,
fields,
update,
sort,
upsert,
returnNew,
remove,
result,
errmsg);
}
void create() {
Lock::GlobalWrite lk;
OperationContextImpl txn;
ASSERT( userCreateNS( &txn, db(), ns(), fromjson( spec() ), false ).isOK() );
}
