void run() {
OperationContextImpl txn;
Client::WriteContext ctx(&txn, ns());
DBDirectClient db(&txn);
db.insert(ns(), BSON("x" << 1 << "y" << 2));
db.insert(ns(), BSON("x" << 2 << "y" << 2));
Collection* collection = ctx.ctx().db()->getCollection( &txn, ns() );
ASSERT( collection );
IndexCatalog* indexCatalog = collection->getIndexCatalog();
ASSERT_EQUALS(1, indexCatalog->numIndexesReady(&txn));
// _id index
ASSERT_EQUALS(1U, db.count("test.system.indexes"));
// test.buildindex
// test.buildindex_$id
// test.system.indexes
ASSERT_EQUALS(3U, db.count("test.system.namespaces"));
db.ensureIndex(ns(), BSON("y" << 1), true);
ASSERT_EQUALS(1, indexCatalog->numIndexesReady(&txn));
ASSERT_EQUALS(1U, db.count("test.system.indexes"));
ASSERT_EQUALS(3U, db.count("test.system.namespaces"));
db.ensureIndex(ns(), BSON("x" << 1), true);
ctx.commit();
ASSERT_EQUALS(2, indexCatalog->numIndexesReady(&txn));
ASSERT_EQUALS(2U, db.count("test.system.indexes"));
ASSERT_EQUALS(4U, db.count("test.system.namespaces"));
}
bool Helpers::findById(OperationContext* opCtx,
Database* database,
StringData ns,
BSONObj query,
BSONObj& result,
bool* nsFound,
bool* indexFound) {
invariant(database);
Collection* collection = database->getCollection(opCtx, ns);
if (!collection) {
return false;
}
if (nsFound)
*nsFound = true;
IndexCatalog* catalog = collection->getIndexCatalog();
const IndexDescriptor* desc = catalog->findIdIndex(opCtx);
if (!desc)
return false;
if (indexFound)
*indexFound = 1;
RecordId loc = catalog->getIndex(desc)->findSingle(opCtx, query["_id"].wrap());
if (loc.isNull())
return false;
result = collection->docFor(opCtx, loc).value();
return true;
}
void run() {
const ServiceContext::UniqueOperationContext txnPtr = cc().makeOperationContext();
OperationContext& txn = *txnPtr;
OldClientWriteContext ctx(&txn, ns());
DBDirectClient db(&txn);
db.insert(ns(), BSON("x" << 1 << "y" << 2));
db.insert(ns(), BSON("x" << 2 << "y" << 2));
Collection* collection = ctx.getCollection();
ASSERT(collection);
IndexCatalog* indexCatalog = collection->getIndexCatalog();
ASSERT_EQUALS(1, indexCatalog->numIndexesReady(&txn));
// _id index
ASSERT_EQUALS(1U, db.getIndexSpecs(ns()).size());
ASSERT_EQUALS(ErrorCodes::DuplicateKey,
dbtests::createIndex(&txn, ns(), BSON("y" << 1), true));
ASSERT_EQUALS(1, indexCatalog->numIndexesReady(&txn));
ASSERT_EQUALS(1U, db.getIndexSpecs(ns()).size());
ASSERT_OK(dbtests::createIndex(&txn, ns(), BSON("x" << 1), true));
ASSERT_EQUALS(2, indexCatalog->numIndexesReady(&txn));
ASSERT_EQUALS(2U, db.getIndexSpecs(ns()).size());
}
void configureSystemIndexes(OperationContext* txn, const StringData& dbname) {
int authzVersion;
Status status = getGlobalAuthorizationManager()->getAuthorizationVersion(
txn, &authzVersion);
if (!status.isOK()) {
return;
}
if (dbname == "admin" && authzVersion >= AuthorizationManager::schemaVersion26Final) {
NamespaceString systemUsers(dbname, "system.users");
// Make sure the old unique index from v2.4 on system.users doesn't exist.
Client::WriteContext wctx(txn, systemUsers);
Collection* collection = wctx.ctx().db()->getCollection(txn,
NamespaceString(systemUsers));
if (!collection) {
return;
}
IndexCatalog* indexCatalog = collection->getIndexCatalog();
IndexDescriptor* oldIndex = NULL;
while ((oldIndex = indexCatalog->findIndexByKeyPattern(txn, v1SystemUsersKeyPattern))) {
indexCatalog->dropIndex(txn, oldIndex);
}
wctx.commit();
}
}
RecordId Helpers::findById(OperationContext* txn, Collection* collection, const BSONObj& idquery) {
verify(collection);
IndexCatalog* catalog = collection->getIndexCatalog();
const IndexDescriptor* desc = catalog->findIdIndex(txn);
uassert(13430, "no _id index", desc);
return catalog->getIndex(desc)->findSingle(txn, idquery["_id"].wrap());
}
void run() {
OperationContextImpl txn;
Client::WriteContext ctx(&txn, ns());
DBDirectClient db(&txn);
db.insert(ns(), BSON("x" << 1 << "y" << 2));
db.insert(ns(), BSON("x" << 2 << "y" << 2));
Collection* collection = ctx.ctx().db()->getCollection( &txn, ns() );
ASSERT( collection );
IndexCatalog* indexCatalog = collection->getIndexCatalog();
ASSERT_EQUALS(1, indexCatalog->numIndexesReady(&txn));
// _id index
ASSERT_EQUALS(1U, db.getIndexSpecs(ns()).size());
db.ensureIndex(ns(), BSON("y" << 1), true);
ASSERT_EQUALS(1, indexCatalog->numIndexesReady(&txn));
ASSERT_EQUALS(1U, db.getIndexSpecs(ns()).size());
db.ensureIndex(ns(), BSON("x" << 1), true);
ctx.commit();
ASSERT_EQUALS(2, indexCatalog->numIndexesReady(&txn));
ASSERT_EQUALS(2U, db.getIndexSpecs(ns()).size());
}
bool Helpers::findById(Database* database, const char *ns, BSONObj query, BSONObj& result ,
bool* nsFound , bool* indexFound ) {
Lock::assertAtLeastReadLocked(ns);
invariant( database );
Collection* collection = database->getCollection( ns );
if ( !collection ) {
return false;
}
if ( nsFound )
*nsFound = true;
IndexCatalog* catalog = collection->getIndexCatalog();
const IndexDescriptor* desc = catalog->findIdIndex();
if ( !desc )
return false;
if ( indexFound )
*indexFound = 1;
// See SERVER-12397. This may not always be true.
BtreeBasedAccessMethod* accessMethod =
static_cast<BtreeBasedAccessMethod*>(catalog->getIndex( desc ));
DiskLoc loc = accessMethod->findSingle( query["_id"].wrap() );
if ( loc.isNull() )
return false;
result = collection->docFor( loc );
return true;
}
bool Helpers::findById(Client& c, const char *ns, BSONObj query, BSONObj& result ,
bool* nsFound , bool* indexFound ) {
Lock::assertAtLeastReadLocked(ns);
Database *database = c.database();
verify( database );
Collection* collection = database->getCollection( ns );
if ( !collection ) {
return false;
}
if ( nsFound )
*nsFound = true;
IndexCatalog* catalog = collection->getIndexCatalog();
const IndexDescriptor* desc = catalog->findIdIndex();
if ( !desc )
return false;
if ( indexFound )
*indexFound = 1;
BtreeBasedAccessMethod* accessMethod = catalog->getBtreeBasedIndex( desc );
DiskLoc loc = accessMethod->findSingle( query["_id"].wrap() );
if ( loc.isNull() )
return false;
result = collection->docFor( loc );
return true;
}
Collection* Database::createCollection(OperationContext* txn,
StringData ns,
const CollectionOptions& options,
bool createIdIndex) {
massert(17399, "collection already exists", getCollection(ns) == NULL);
massertNamespaceNotIndex(ns, "createCollection");
invariant(txn->lockState()->isDbLockedForMode(name(), MODE_X));
if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer &&
!(ns.startsWith("config.") || ns.startsWith("local.") || ns.startsWith("admin."))) {
uasserted(14037, "can't create user databases on a --configsvr instance");
}
if (NamespaceString::normal(ns)) {
// This check only applies for actual collections, not indexes or other types of ns.
uassert(17381,
str::stream() << "fully qualified namespace " << ns << " is too long "
<< "(max is "
<< NamespaceString::MaxNsCollectionLen
<< " bytes)",
ns.size() <= NamespaceString::MaxNsCollectionLen);
}
NamespaceString nss(ns);
uassert(17316, "cannot create a blank collection", nss.coll() > 0);
uassert(28838, "cannot create a non-capped oplog collection", options.capped || !nss.isOplog());
audit::logCreateCollection(&cc(), ns);
txn->recoveryUnit()->registerChange(new AddCollectionChange(txn, this, ns));
Status status = _dbEntry->createCollection(txn, ns, options, true /*allocateDefaultSpace*/);
massertNoTraceStatusOK(status);
Collection* collection = _getOrCreateCollectionInstance(txn, ns);
invariant(collection);
_collections[ns] = collection;
if (createIdIndex) {
if (collection->requiresIdIndex()) {
if (options.autoIndexId == CollectionOptions::YES ||
options.autoIndexId == CollectionOptions::DEFAULT) {
IndexCatalog* ic = collection->getIndexCatalog();
uassertStatusOK(ic->createIndexOnEmptyCollection(txn, ic->getDefaultIdIndexSpec()));
}
}
if (nss.isSystem()) {
authindex::createSystemIndexes(txn, collection);
}
}
auto opObserver = getGlobalServiceContext()->getOpObserver();
if (opObserver)
opObserver->onCreateCollection(txn, nss, options);
return collection;
}
DiskLoc Helpers::findById(Collection* collection, const BSONObj& idquery) {
verify(collection);
IndexCatalog* catalog = collection->getIndexCatalog();
const IndexDescriptor* desc = catalog->findIdIndex();
uassert(13430, "no _id index", desc);
BtreeBasedAccessMethod* accessMethod = catalog->getBtreeBasedIndex( desc );
return accessMethod->findSingle( idquery["_id"].wrap() );
}
void run() {
string ns = "unittests.rollback_set_index_head";
const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext();
OperationContext& opCtx = *opCtxPtr;
NamespaceString nss(ns);
dropDatabase(&opCtx, nss);
createCollection(&opCtx, nss);
AutoGetDb autoDb(&opCtx, nss.db(), MODE_X);
Collection* coll = autoDb.getDb()->getCollection(&opCtx, nss);
IndexCatalog* catalog = coll->getIndexCatalog();
string idxName = "a";
BSONObj spec = BSON("ns" << ns << "key" << BSON("a" << 1) << "name" << idxName << "v"
<< static_cast<int>(kIndexVersion));
{
WriteUnitOfWork uow(&opCtx);
ASSERT_OK(catalog->createIndexOnEmptyCollection(&opCtx, spec));
uow.commit();
}
IndexDescriptor* indexDesc = catalog->findIndexByName(&opCtx, idxName);
invariant(indexDesc);
const IndexCatalogEntry* ice = catalog->getEntry(indexDesc);
invariant(ice);
HeadManager* headManager = ice->headManager();
const RecordId oldHead = headManager->getHead(&opCtx);
ASSERT_EQ(oldHead, ice->head(&opCtx));
const RecordId dummyHead(123, 456);
ASSERT_NE(oldHead, dummyHead);
// END SETUP / START TEST
{
WriteUnitOfWork uow(&opCtx);
headManager->setHead(&opCtx, dummyHead);
ASSERT_EQ(ice->head(&opCtx), dummyHead);
ASSERT_EQ(headManager->getHead(&opCtx), dummyHead);
if (!rollback) {
uow.commit();
}
}
if (rollback) {
ASSERT_EQ(ice->head(&opCtx), oldHead);
ASSERT_EQ(headManager->getHead(&opCtx), oldHead);
} else {
ASSERT_EQ(ice->head(&opCtx), dummyHead);
ASSERT_EQ(headManager->getHead(&opCtx), dummyHead);
}
}
DiskLoc Helpers::findById(Collection* collection, const BSONObj& idquery) {
verify(collection);
IndexCatalog* catalog = collection->getIndexCatalog();
const IndexDescriptor* desc = catalog->findIdIndex();
uassert(13430, "no _id index", desc);
// See SERVER-12397. This may not always be true.
BtreeBasedAccessMethod* accessMethod =
static_cast<BtreeBasedAccessMethod*>(catalog->getIndex( desc ));
return accessMethod->findSingle( idquery["_id"].wrap() );
}
Collection* Database::createCollection( OperationContext* txn,
const StringData& ns,
const CollectionOptions& options,
bool allocateDefaultSpace,
bool createIdIndex ) {
massert( 17399, "collection already exists", getCollection( txn, ns ) == NULL );
massertNamespaceNotIndex( ns, "createCollection" );
if ( serverGlobalParams.configsvr &&
!( ns.startsWith( "config." ) ||
ns.startsWith( "local." ) ||
ns.startsWith( "admin." ) ) ) {
uasserted(14037, "can't create user databases on a --configsvr instance");
}
if (NamespaceString::normal(ns)) {
// This check only applies for actual collections, not indexes or other types of ns.
uassert(17381, str::stream() << "fully qualified namespace " << ns << " is too long "
<< "(max is " << NamespaceString::MaxNsCollectionLen << " bytes)",
ns.size() <= NamespaceString::MaxNsCollectionLen);
}
NamespaceString nss( ns );
uassert( 17316, "cannot create a blank collection", nss.coll() > 0 );
audit::logCreateCollection( currentClient.get(), ns );
txn->recoveryUnit()->registerChange( new AddCollectionChange(this, ns) );
Status status = _dbEntry->createCollection(txn, ns,
options, allocateDefaultSpace);
massertStatusOK(status);
Collection* collection = getCollection(txn, ns);
invariant(collection);
if ( createIdIndex ) {
if ( collection->requiresIdIndex() ) {
if ( options.autoIndexId == CollectionOptions::YES ||
options.autoIndexId == CollectionOptions::DEFAULT ) {
IndexCatalog* ic = collection->getIndexCatalog();
uassertStatusOK(
ic->createIndexOnEmptyCollection(txn, ic->getDefaultIdIndexSpec()));
}
}
if ( nss.isSystem() ) {
authindex::createSystemIndexes( txn, collection );
}
}
return collection;
}
Collection* Database::createCollection(OperationContext* txn,
StringData ns,
const CollectionOptions& options,
bool createIdIndex) {
invariant(txn->lockState()->isDbLockedForMode(name(), MODE_X));
invariant(!options.isView());
NamespaceString nss(ns);
_checkCanCreateCollection(nss, options);
audit::logCreateCollection(&cc(), ns);
txn->recoveryUnit()->registerChange(new AddCollectionChange(txn, this, ns));
Status status = _dbEntry->createCollection(txn, ns, options, true /*allocateDefaultSpace*/);
massertNoTraceStatusOK(status);
Collection* collection = _getOrCreateCollectionInstance(txn, ns);
invariant(collection);
_collections[ns] = collection;
if (createIdIndex) {
if (collection->requiresIdIndex()) {
if (options.autoIndexId == CollectionOptions::YES ||
options.autoIndexId == CollectionOptions::DEFAULT) {
// The creation of the _id index isn't replicated and is instead implicit in the
// creation of the collection. This means that the version of the _id index to build
// is technically unspecified. However, we're able to use the
// featureCompatibilityVersion of this server to determine the default index version
// to use because we apply commands (opType == 'c') in their own batch. This
// guarantees the write to the admin.system.version collection from the
// "setFeatureCompatibilityVersion" command either happens entirely before the
// collection creation or it happens entirely after.
const auto featureCompatibilityVersion =
serverGlobalParams.featureCompatibilityVersion.load();
IndexCatalog* ic = collection->getIndexCatalog();
uassertStatusOK(ic->createIndexOnEmptyCollection(
txn, ic->getDefaultIdIndexSpec(featureCompatibilityVersion)));
}
}
if (nss.isSystem()) {
authindex::createSystemIndexes(txn, collection);
}
}
auto opObserver = getGlobalServiceContext()->getOpObserver();
if (opObserver)
opObserver->onCreateCollection(txn, nss, options);
return collection;
}
void run() {
string ns = "unittests.rollback_create_collection_and_indexes";
const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext();
OperationContext& opCtx = *opCtxPtr;
NamespaceString nss(ns);
dropDatabase(&opCtx, nss);
Lock::DBLock dbXLock(&opCtx, nss.db(), MODE_X);
OldClientContext ctx(&opCtx, nss.ns());
string idxNameA = "indexA";
string idxNameB = "indexB";
string idxNameC = "indexC";
BSONObj specA = BSON("ns" << ns << "key" << BSON("a" << 1) << "name" << idxNameA << "v"
<< static_cast<int>(kIndexVersion));
BSONObj specB = BSON("ns" << ns << "key" << BSON("b" << 1) << "name" << idxNameB << "v"
<< static_cast<int>(kIndexVersion));
BSONObj specC = BSON("ns" << ns << "key" << BSON("c" << 1) << "name" << idxNameC << "v"
<< static_cast<int>(kIndexVersion));
// END SETUP / START TEST
{
WriteUnitOfWork uow(&opCtx);
ASSERT(!collectionExists(&ctx, nss.ns()));
ASSERT_OK(userCreateNS(
&opCtx, ctx.db(), nss.ns(), BSONObj(), CollectionOptions::parseForCommand, false));
ASSERT(collectionExists(&ctx, nss.ns()));
Collection* coll = ctx.db()->getCollection(&opCtx, nss);
IndexCatalog* catalog = coll->getIndexCatalog();
ASSERT_OK(catalog->createIndexOnEmptyCollection(&opCtx, specA));
ASSERT_OK(catalog->createIndexOnEmptyCollection(&opCtx, specB));
ASSERT_OK(catalog->createIndexOnEmptyCollection(&opCtx, specC));
if (!rollback) {
uow.commit();
}
} // uow
if (rollback) {
ASSERT(!collectionExists(&ctx, ns));
} else {
ASSERT(collectionExists(&ctx, ns));
ASSERT(indexReady(&opCtx, nss, idxNameA));
ASSERT(indexReady(&opCtx, nss, idxNameB));
ASSERT(indexReady(&opCtx, nss, idxNameC));
}
}
void run() {
string ns = "unittests.rollback_drop_index";
const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext();
OperationContext& opCtx = *opCtxPtr;
NamespaceString nss(ns);
dropDatabase(&opCtx, nss);
createCollection(&opCtx, nss);
AutoGetDb autoDb(&opCtx, nss.db(), MODE_X);
Collection* coll = autoDb.getDb()->getCollection(&opCtx, nss);
IndexCatalog* catalog = coll->getIndexCatalog();
string idxName = "a";
BSONObj spec = BSON("ns" << ns << "key" << BSON("a" << 1) << "name" << idxName << "v"
<< static_cast<int>(kIndexVersion));
{
WriteUnitOfWork uow(&opCtx);
ASSERT_OK(catalog->createIndexOnEmptyCollection(&opCtx, spec));
insertRecord(&opCtx, nss, BSON("a" << 1));
insertRecord(&opCtx, nss, BSON("a" << 2));
insertRecord(&opCtx, nss, BSON("a" << 3));
uow.commit();
}
ASSERT(indexReady(&opCtx, nss, idxName));
ASSERT_EQ(3u, getNumIndexEntries(&opCtx, nss, idxName));
// END SETUP / START TEST
{
WriteUnitOfWork uow(&opCtx);
dropIndex(&opCtx, nss, idxName);
ASSERT(!indexExists(&opCtx, nss, idxName));
if (!rollback) {
uow.commit();
}
}
if (rollback) {
ASSERT(indexExists(&opCtx, nss, idxName));
ASSERT(indexReady(&opCtx, nss, idxName));
ASSERT_EQ(3u, getNumIndexEntries(&opCtx, nss, idxName));
} else {
ASSERT(!indexExists(&opCtx, nss, idxName));
}
}
Collection* Database::createCollection(OperationContext* txn,
StringData ns,
const CollectionOptions& options,
bool createIdIndex,
const BSONObj& idIndex) {
invariant(txn->lockState()->isDbLockedForMode(name(), MODE_X));
invariant(!options.isView());
NamespaceString nss(ns);
_checkCanCreateCollection(nss, options);
audit::logCreateCollection(&cc(), ns);
Status status = _dbEntry->createCollection(txn, ns, options, true /*allocateDefaultSpace*/);
massertNoTraceStatusOK(status);
txn->recoveryUnit()->registerChange(new AddCollectionChange(txn, this, ns));
Collection* collection = _getOrCreateCollectionInstance(txn, ns);
invariant(collection);
_collections[ns] = collection;
BSONObj fullIdIndexSpec;
if (createIdIndex) {
if (collection->requiresIdIndex()) {
if (options.autoIndexId == CollectionOptions::YES ||
options.autoIndexId == CollectionOptions::DEFAULT) {
const auto featureCompatibilityVersion =
serverGlobalParams.featureCompatibility.version.load();
IndexCatalog* ic = collection->getIndexCatalog();
fullIdIndexSpec = uassertStatusOK(ic->createIndexOnEmptyCollection(
txn,
!idIndex.isEmpty() ? idIndex
: ic->getDefaultIdIndexSpec(featureCompatibilityVersion)));
}
}
if (nss.isSystem()) {
authindex::createSystemIndexes(txn, collection);
}
}
auto opObserver = getGlobalServiceContext()->getOpObserver();
if (opObserver)
opObserver->onCreateCollection(txn, nss, options, fullIdIndexSpec);
return collection;
}
IndexScan* createIndexScan(MatchExpression* expr, WorkingSet* ws) {
IndexCatalog* catalog = _coll->getIndexCatalog();
IndexDescriptor* descriptor = catalog->findIndexByKeyPattern(&_txn, BSON("x" << 1));
invariant(descriptor);
// We are not testing indexing here so use maximal bounds
IndexScanParams params;
params.descriptor = descriptor;
params.bounds.isSimpleRange = true;
params.bounds.startKey = BSON("" << 0);
params.bounds.endKey = BSON("" << kDocuments+1);
params.bounds.endKeyInclusive = true;
params.direction = 1;
// This child stage gets owned and freed by its parent CountStage
return new IndexScan(&_txn, params, ws, expr);
}
IndexScan* createIndexScanSimpleRange(BSONObj startKey, BSONObj endKey) {
IndexCatalog* catalog = _coll->getIndexCatalog();
std::vector<IndexDescriptor*> indexes;
catalog->findIndexesByKeyPattern(&_opCtx, BSON("x" << 1), false, &indexes);
ASSERT_EQ(indexes.size(), 1U);
// We are not testing indexing here so use maximal bounds
IndexScanParams params(&_opCtx, *indexes[0]);
params.bounds.isSimpleRange = true;
params.bounds.startKey = startKey;
params.bounds.endKey = endKey;
params.bounds.boundInclusion = BoundInclusion::kIncludeBothStartAndEndKeys;
params.direction = 1;
// This child stage gets owned and freed by the caller.
MatchExpression* filter = NULL;
return new IndexScan(&_opCtx, params, &_ws, filter);
}
IndexScan* createIndexScanSimpleRange(BSONObj startKey, BSONObj endKey) {
IndexCatalog* catalog = _coll->getIndexCatalog();
IndexDescriptor* descriptor = catalog->findIndexByKeyPattern(&_txn, BSON("x" << 1));
invariant(descriptor);
// We are not testing indexing here so use maximal bounds
IndexScanParams params;
params.descriptor = descriptor;
params.bounds.isSimpleRange = true;
params.bounds.startKey = startKey;
params.bounds.endKey = endKey;
params.bounds.endKeyInclusive = true;
params.direction = 1;
// This child stage gets owned and freed by the caller.
MatchExpression* filter = NULL;
return new IndexScan(&_txn, params, &_ws, filter);
}
uint64_t Collection::getIndexSize(OperationContext* opCtx, BSONObjBuilder* details, int scale) {
IndexCatalog* idxCatalog = getIndexCatalog();
IndexCatalog::IndexIterator ii = idxCatalog->getIndexIterator(opCtx, true);
uint64_t totalSize = 0;
while (ii.more()) {
IndexDescriptor* d = ii.next();
IndexAccessMethod* iam = idxCatalog->getIndex(d);
long long ds = iam->getSpaceUsedBytes(opCtx);
totalSize += ds;
if (details) {
details->appendNumber(d->indexName(), ds / scale);
}
}
return totalSize;
}
IndexScan* createIndexScan(BSONObj startKey,
BSONObj endKey,
bool startInclusive,
bool endInclusive,
int direction = 1) {
IndexCatalog* catalog = _coll->getIndexCatalog();
IndexDescriptor* descriptor = catalog->findIndexByKeyPattern(&_txn, BSON("x" << 1));
invariant(descriptor);
IndexScanParams params;
params.descriptor = descriptor;
params.direction = direction;
OrderedIntervalList oil("x");
BSONObjBuilder bob;
bob.appendAs(startKey.firstElement(), "");
bob.appendAs(endKey.firstElement(), "");
oil.intervals.push_back(Interval(bob.obj(), startInclusive, endInclusive));
params.bounds.fields.push_back(oil);
MatchExpression* filter = NULL;
return new IndexScan(&_txn, params, &_ws, filter);
}
IndexScan* createIndexScan(BSONObj startKey,
BSONObj endKey,
bool startInclusive,
bool endInclusive,
int direction = 1) {
IndexCatalog* catalog = _coll->getIndexCatalog();
std::vector<IndexDescriptor*> indexes;
catalog->findIndexesByKeyPattern(&_opCtx, BSON("x" << 1), false, &indexes);
ASSERT_EQ(indexes.size(), 1U);
IndexScanParams params(&_opCtx, *indexes[0]);
params.direction = direction;
OrderedIntervalList oil("x");
BSONObjBuilder bob;
bob.appendAs(startKey.firstElement(), "");
bob.appendAs(endKey.firstElement(), "");
oil.intervals.push_back(Interval(bob.obj(), startInclusive, endInclusive));
params.bounds.fields.push_back(oil);
MatchExpression* filter = NULL;
return new IndexScan(&_opCtx, params, &_ws, filter);
}
bool Helpers::findById(OperationContext* txn,
Database* database,
const char *ns,
BSONObj query,
BSONObj& result,
bool* nsFound,
bool* indexFound) {
invariant(database);
Collection* collection = database->getCollection( ns );
if ( !collection ) {
return false;
}
if ( nsFound )
*nsFound = true;
IndexCatalog* catalog = collection->getIndexCatalog();
const IndexDescriptor* desc = catalog->findIdIndex( txn );
if ( !desc )
return false;
if ( indexFound )
*indexFound = 1;
// See SERVER-12397. This may not always be true.
BtreeBasedAccessMethod* accessMethod =
static_cast<BtreeBasedAccessMethod*>(catalog->getIndex( desc ));
RecordId loc = accessMethod->findSingle( txn, query["_id"].wrap() );
if ( loc.isNull() )
return false;
result = collection->docFor(txn, loc).value();
return true;
}
PlanStage::StageState TwoDNear::work(WorkingSetID* out) {
++_commonStats.works;
// Adds the amount of time taken by work() to executionTimeMillis.
ScopedTimer timer(&_commonStats.executionTimeMillis);
if (!_initted) {
_initted = true;
if ( !_params.collection )
return PlanStage::IS_EOF;
IndexCatalog* indexCatalog = _params.collection->getIndexCatalog();
IndexDescriptor* desc = indexCatalog->findIndexByKeyPattern(_params.indexKeyPattern);
if ( desc == NULL )
return PlanStage::IS_EOF;
TwoDAccessMethod* am = static_cast<TwoDAccessMethod*>( indexCatalog->getIndex( desc ) );
auto_ptr<twod_exec::GeoSearch> search;
search.reset(new twod_exec::GeoSearch(_params.collection,
am,
_params.nearQuery.centroid.oldPoint,
_params.numWanted,
_params.filter,
_params.nearQuery.maxDistance,
_params.nearQuery.isNearSphere ? twod_exec::GEO_SPHERE
: twod_exec::GEO_PLANE));
// This is where all the work is done. :(
search->exec();
_specificStats.objectsLoaded = search->_objectsLoaded;
_specificStats.nscanned = search->_lookedAt;
for (twod_exec::GeoHopper::Holder::iterator it = search->_points.begin();
it != search->_points.end(); it++) {
WorkingSetID id = _workingSet->allocate();
WorkingSetMember* member = _workingSet->get(id);
member->loc = it->_loc;
member->obj = _params.collection->docFor(member->loc);
member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ;
if (_params.addDistMeta) {
member->addComputed(new GeoDistanceComputedData(it->_distance));
}
if (_params.addPointMeta) {
member->addComputed(new GeoNearPointComputedData(it->_pt));
}
_results.push(Result(id, it->_distance));
_invalidationMap.insert(pair<DiskLoc, WorkingSetID>(it->_loc, id));
}
}
if (isEOF()) { return PlanStage::IS_EOF; }
Result result = _results.top();
_results.pop();
*out = result.id;
// Remove from invalidation map.
WorkingSetMember* member = _workingSet->get(*out);
// The WSM may have been mutated or deleted so it may not have a loc.
if (member->hasLoc()) {
typedef multimap<DiskLoc, WorkingSetID>::iterator MMIT;
pair<MMIT, MMIT> range = _invalidationMap.equal_range(member->loc);
for (MMIT it = range.first; it != range.second; ++it) {
if (it->second == *out) {
_invalidationMap.erase(it);
break;
}
}
}
++_commonStats.advanced;
return PlanStage::ADVANCED;
}
void IndexRebuilder::checkNS(const std::list<std::string>& nsToCheck) {
bool firstTime = true;
for (std::list<std::string>::const_iterator it = nsToCheck.begin();
it != nsToCheck.end();
++it) {
string ns = *it;
LOG(3) << "IndexRebuilder::checkNS: " << ns;
OperationContextImpl txn; // XXX???
// This write lock is held throughout the index building process
// for this namespace.
Client::WriteContext ctx(&txn, ns);
Collection* collection = ctx.ctx().db()->getCollection( &txn, ns );
if ( collection == NULL )
continue;
IndexCatalog* indexCatalog = collection->getIndexCatalog();
if ( collection->ns().isOplog() && indexCatalog->numIndexesTotal() > 0 ) {
warning() << ns << " had illegal indexes, removing";
indexCatalog->dropAllIndexes(&txn, true);
continue;
}
vector<BSONObj> indexesToBuild = indexCatalog->getAndClearUnfinishedIndexes(&txn);
// The indexes have now been removed from system.indexes, so the only record is
// in-memory. If there is a journal commit between now and when insert() rewrites
// the entry and the db crashes before the new system.indexes entry is journalled,
// the index will be lost forever. Thus, we're assuming no journaling will happen
// between now and the entry being re-written.
if ( indexesToBuild.size() == 0 ) {
continue;
}
log() << "found " << indexesToBuild.size()
<< " interrupted index build(s) on " << ns;
if (firstTime) {
log() << "note: restart the server with --noIndexBuildRetry to skip index rebuilds";
firstTime = false;
}
if (!serverGlobalParams.indexBuildRetry) {
log() << " not rebuilding interrupted indexes";
continue;
}
// TODO: these can/should/must be done in parallel
for ( size_t i = 0; i < indexesToBuild.size(); i++ ) {
BSONObj indexObj = indexesToBuild[i];
log() << "going to rebuild: " << indexObj;
Status status = indexCatalog->createIndex(&txn, indexObj, false);
if ( !status.isOK() ) {
log() << "building index failed: " << status.toString() << " index: " << indexObj;
}
}
}
}
Status appendCollectionStorageStats(OperationContext* opCtx,
const NamespaceString& nss,
const BSONObj& param,
BSONObjBuilder* result) {
int scale = 1;
if (param["scale"].isNumber()) {
scale = param["scale"].numberInt();
if (scale < 1) {
return {ErrorCodes::BadValue, "scale has to be >= 1"};
}
} else if (param["scale"].trueValue()) {
return {ErrorCodes::BadValue, "scale has to be a number >= 1"};
}
bool verbose = param["verbose"].trueValue();
AutoGetCollectionForReadCommand ctx(opCtx, nss);
Collection* collection = ctx.getCollection(); // Will be set if present
if (!ctx.getDb() || !collection) {
result->appendNumber("size", 0);
result->appendNumber("count", 0);
result->appendNumber("storageSize", 0);
result->append("nindexes", 0);
result->appendNumber("totalIndexSize", 0);
result->append("indexDetails", BSONObj());
result->append("indexSizes", BSONObj());
std::string errmsg = !(ctx.getDb()) ? "Database [" + nss.db().toString() + "] not found."
: "Collection [" + nss.toString() + "] not found.";
return {ErrorCodes::NamespaceNotFound, errmsg};
}
long long size = collection->dataSize(opCtx) / scale;
result->appendNumber("size", size);
long long numRecords = collection->numRecords(opCtx);
result->appendNumber("count", numRecords);
if (numRecords)
result->append("avgObjSize", collection->averageObjectSize(opCtx));
RecordStore* recordStore = collection->getRecordStore();
result->appendNumber(
"storageSize",
static_cast<long long>(recordStore->storageSize(opCtx, result, verbose ? 1 : 0)) / scale);
recordStore->appendCustomStats(opCtx, result, scale);
IndexCatalog* indexCatalog = collection->getIndexCatalog();
result->append("nindexes", indexCatalog->numIndexesReady(opCtx));
BSONObjBuilder indexDetails;
IndexCatalog::IndexIterator i = indexCatalog->getIndexIterator(opCtx, false);
while (i.more()) {
const IndexDescriptor* descriptor = i.next();
IndexAccessMethod* iam = indexCatalog->getIndex(descriptor);
invariant(iam);
BSONObjBuilder bob;
if (iam->appendCustomStats(opCtx, &bob, scale)) {
indexDetails.append(descriptor->indexName(), bob.obj());
}
}
result->append("indexDetails", indexDetails.obj());
BSONObjBuilder indexSizes;
long long indexSize = collection->getIndexSize(opCtx, &indexSizes, scale);
result->appendNumber("totalIndexSize", indexSize / scale);
result->append("indexSizes", indexSizes.obj());
return Status::OK();
}
bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& anObjBuilder, bool /*fromRepl*/) {
BSONElement e = jsobj.firstElement();
const string toDeleteNs = dbname + '.' + e.valuestr();
if (!serverGlobalParams.quiet) {
MONGO_TLOG(0) << "CMD: dropIndexes " << toDeleteNs << endl;
}
Lock::DBWrite dbXLock(dbname);
Client::Context ctx(toDeleteNs);
Collection* collection = cc().database()->getCollection( toDeleteNs );
if ( ! collection ) {
errmsg = "ns not found";
return false;
}
stopIndexBuilds(cc().database(), jsobj);
IndexCatalog* indexCatalog = collection->getIndexCatalog();
anObjBuilder.appendNumber("nIndexesWas", indexCatalog->numIndexesTotal() );
BSONElement f = jsobj.getField("index");
if ( f.type() == String ) {
string indexToDelete = f.valuestr();
if ( indexToDelete == "*" ) {
Status s = indexCatalog->dropAllIndexes( false );
if ( !s.isOK() ) {
appendCommandStatus( anObjBuilder, s );
return false;
}
anObjBuilder.append("msg", "non-_id indexes dropped for collection");
return true;
}
IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByName( indexToDelete );
if ( desc == NULL ) {
errmsg = str::stream() << "index not found with name [" << indexToDelete << "]";
return false;
}
if ( desc->isIdIndex() ) {
errmsg = "cannot drop _id index";
return false;
}
Status s = indexCatalog->dropIndex( desc );
if ( !s.isOK() ) {
appendCommandStatus( anObjBuilder, s );
return false;
}
return true;
}
if ( f.type() == Object ) {
IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByKeyPattern( f.embeddedObject() );
if ( desc == NULL ) {
errmsg = "can't find index with key:";
errmsg += f.embeddedObject().toString();
return false;
}
if ( desc->isIdIndex() ) {
errmsg = "cannot drop _id index";
return false;
}
Status s = indexCatalog->dropIndex( desc );
if ( !s.isOK() ) {
appendCommandStatus( anObjBuilder, s );
return false;
}
return true;
}
errmsg = "invalid index name spec";
return false;
}
bool wrappedRun(OperationContext* txn,
const string& dbname,
BSONObj& jsobj,
string& errmsg,
BSONObjBuilder& anObjBuilder) {
const std::string coll = jsobj.firstElement().valuestrsafe();
if (coll.empty()) {
errmsg = "no collection name specified";
return false;
}
const std::string toDeleteNs = dbname + '.' + coll;
if (!serverGlobalParams.quiet) {
LOG(0) << "CMD: dropIndexes " << toDeleteNs << endl;
}
Client::Context ctx(txn, toDeleteNs);
Database* db = ctx.db();
Collection* collection = db->getCollection( txn, toDeleteNs );
if ( ! collection ) {
errmsg = "ns not found";
return false;
}
stopIndexBuilds(txn, db, jsobj);
IndexCatalog* indexCatalog = collection->getIndexCatalog();
anObjBuilder.appendNumber("nIndexesWas", indexCatalog->numIndexesTotal(txn) );
BSONElement f = jsobj.getField("index");
if ( f.type() == String ) {
string indexToDelete = f.valuestr();
if ( indexToDelete == "*" ) {
Status s = indexCatalog->dropAllIndexes(txn, false);
if ( !s.isOK() ) {
appendCommandStatus( anObjBuilder, s );
return false;
}
anObjBuilder.append("msg", "non-_id indexes dropped for collection");
return true;
}
IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByName( txn,
indexToDelete );
if ( desc == NULL ) {
errmsg = str::stream() << "index not found with name [" << indexToDelete << "]";
return false;
}
if ( desc->isIdIndex() ) {
errmsg = "cannot drop _id index";
return false;
}
Status s = indexCatalog->dropIndex(txn, desc);
if ( !s.isOK() ) {
appendCommandStatus( anObjBuilder, s );
return false;
}
return true;
}
if ( f.type() == Object ) {
IndexDescriptor* desc =
collection->getIndexCatalog()->findIndexByKeyPattern( txn, f.embeddedObject() );
if ( desc == NULL ) {
errmsg = "can't find index with key:";
errmsg += f.embeddedObject().toString();
return false;
}
if ( desc->isIdIndex() ) {
errmsg = "cannot drop _id index";
return false;
}
Status s = indexCatalog->dropIndex(txn, desc);
if ( !s.isOK() ) {
appendCommandStatus( anObjBuilder, s );
return false;
}
return true;
}
errmsg = "invalid index name spec";
return false;
}
void validateNS(const string& ns,
Collection* collection,
const BSONObj& cmdObj,
BSONObjBuilder& result) {
const bool full = cmdObj["full"].trueValue();
const bool scanData = full || cmdObj["scandata"].trueValue();
NamespaceDetails* nsd = collection->details();
bool valid = true;
BSONArrayBuilder errors; // explanation(s) for why valid = false
if ( collection->isCapped() ){
result.append("capped", nsd->isCapped());
result.appendNumber("max", nsd->maxCappedDocs());
}
if ( nsd->firstExtent().isNull() )
result.append( "firstExtent", "null" );
else
result.append( "firstExtent", str::stream() << nsd->firstExtent().toString()
<< " ns:" << nsd->firstExtent().ext()->nsDiagnostic.toString());
if ( nsd->lastExtent().isNull() )
result.append( "lastExtent", "null" );
else
result.append( "lastExtent", str::stream() << nsd->lastExtent().toString()
<< " ns:" << nsd->lastExtent().ext()->nsDiagnostic.toString());
BSONArrayBuilder extentData;
int extentCount = 0;
try {
if ( !nsd->firstExtent().isNull() ) {
nsd->firstExtent().ext()->assertOk();
nsd->lastExtent().ext()->assertOk();
}
DiskLoc extentDiskLoc = nsd->firstExtent();
while (!extentDiskLoc.isNull()) {
Extent* thisExtent = extentDiskLoc.ext();
if (full) {
extentData << thisExtent->dump();
}
if (!thisExtent->validates(extentDiskLoc, &errors)) {
valid = false;
}
DiskLoc nextDiskLoc = thisExtent->xnext;
if (extentCount > 0 && !nextDiskLoc.isNull()
&& nextDiskLoc.ext()->xprev != extentDiskLoc) {
StringBuilder sb;
sb << "'xprev' pointer " << nextDiskLoc.ext()->xprev.toString()
<< " in extent " << nextDiskLoc.toString()
<< " does not point to extent " << extentDiskLoc.toString();
errors << sb.str();
valid = false;
}
if (nextDiskLoc.isNull() && extentDiskLoc != nsd->lastExtent()) {
StringBuilder sb;
sb << "'lastExtent' pointer " << nsd->lastExtent().toString()
<< " does not point to last extent in list " << extentDiskLoc.toString();
errors << sb.str();
valid = false;
}
extentDiskLoc = nextDiskLoc;
extentCount++;
killCurrentOp.checkForInterrupt();
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception validating extent " << extentCount
<< ": " << e.what();
errors << sb.str();
valid = false;
}
result.append("extentCount", extentCount);
if ( full )
result.appendArray( "extents" , extentData.arr() );
result.appendNumber("datasize", nsd->dataSize());
result.appendNumber("nrecords", nsd->numRecords());
result.appendNumber("lastExtentSize", nsd->lastExtentSize());
result.appendNumber("padding", nsd->paddingFactor());
try {
bool testingLastExtent = false;
try {
if (nsd->firstExtent().isNull()) {
// this is ok
}
else {
result.append("firstExtentDetails", nsd->firstExtent().ext()->dump());
if (!nsd->firstExtent().ext()->xprev.isNull()) {
StringBuilder sb;
sb << "'xprev' pointer in 'firstExtent' " << nsd->firstExtent().toString()
<< " is " << nsd->firstExtent().ext()->xprev.toString()
<< ", should be null";
errors << sb.str();
valid=false;
}
}
testingLastExtent = true;
if (nsd->lastExtent().isNull()) {
// this is ok
}
else {
if (nsd->firstExtent() != nsd->lastExtent()) {
result.append("lastExtentDetails", nsd->lastExtent().ext()->dump());
if (!nsd->lastExtent().ext()->xnext.isNull()) {
StringBuilder sb;
sb << "'xnext' pointer in 'lastExtent' " << nsd->lastExtent().toString()
<< " is " << nsd->lastExtent().ext()->xnext.toString()
<< ", should be null";
errors << sb.str();
valid = false;
}
}
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception processing '"
<< (testingLastExtent ? "lastExtent" : "firstExtent")
<< "': " << e.what();
errors << sb.str();
valid = false;
}
set<DiskLoc> recs;
if( scanData ) {
int n = 0;
int nInvalid = 0;
long long nQuantizedSize = 0;
long long nPowerOf2QuantizedSize = 0;
long long len = 0;
long long nlen = 0;
long long bsonLen = 0;
int outOfOrder = 0;
DiskLoc cl_last;
DiskLoc cl;
Runner::RunnerState state;
auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns));
while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &cl))) {
n++;
if ( n < 1000000 )
recs.insert(cl);
if ( nsd->isCapped() ) {
if ( cl < cl_last )
outOfOrder++;
cl_last = cl;
}
Record *r = cl.rec();
len += r->lengthWithHeaders();
nlen += r->netLength();
if ( r->lengthWithHeaders() ==
NamespaceDetails::quantizeAllocationSpace
( r->lengthWithHeaders() ) ) {
// Count the number of records having a size consistent with
// the quantizeAllocationSpace quantization implementation.
++nQuantizedSize;
}
if ( r->lengthWithHeaders() ==
NamespaceDetails::quantizePowerOf2AllocationSpace
( r->lengthWithHeaders() - 1 ) ) {
// Count the number of records having a size consistent with the
// quantizePowerOf2AllocationSpace quantization implementation.
// Because of SERVER-8311, power of 2 quantization is not idempotent and
// r->lengthWithHeaders() - 1 must be checked instead of the record
// length itself.
++nPowerOf2QuantizedSize;
}
if (full){
BSONObj obj = BSONObj::make(r);
if (!obj.isValid() || !obj.valid()){ // both fast and deep checks
valid = false;
if (nInvalid == 0) // only log once;
errors << "invalid bson object detected (see logs for more info)";
nInvalid++;
if (strcmp("_id", obj.firstElementFieldName()) == 0){
try {
obj.firstElement().validate(); // throws on error
log() << "Invalid bson detected in " << ns << " with _id: " << obj.firstElement().toString(false) << endl;
}
catch(...){
log() << "Invalid bson detected in " << ns << " with corrupt _id" << endl;
}
}
else {
log() << "Invalid bson detected in " << ns << " and couldn't find _id" << endl;
}
}
else {
bsonLen += obj.objsize();
}
}
}
if (Runner::RUNNER_EOF != state) {
// TODO: more descriptive logging.
warning() << "Internal error while reading collection " << ns << endl;
}
if ( nsd->isCapped() && !nsd->capLooped() ) {
result.append("cappedOutOfOrder", outOfOrder);
if ( outOfOrder > 1 ) {
valid = false;
errors << "too many out of order records";
}
}
result.append("objectsFound", n);
if (full) {
result.append("invalidObjects", nInvalid);
}
result.appendNumber("nQuantizedSize", nQuantizedSize);
result.appendNumber("nPowerOf2QuantizedSize", nPowerOf2QuantizedSize);
result.appendNumber("bytesWithHeaders", len);
result.appendNumber("bytesWithoutHeaders", nlen);
if (full) {
result.appendNumber("bytesBson", bsonLen);
}
}
BSONArrayBuilder deletedListArray;
for ( int i = 0; i < Buckets; i++ ) {
deletedListArray << nsd->deletedListEntry(i).isNull();
}
int ndel = 0;
long long delSize = 0;
BSONArrayBuilder delBucketSizes;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = nsd->deletedListEntry(i);
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( nsd->isCapped() && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
string err( str::stream() << "bad pointer in deleted record list: "
<< loc.toString()
<< " bucket: " << i
<< " k: " << k );
errors << err;
valid = false;
break;
}
DeletedRecord *d = loc.drec();
delSize += d->lengthWithHeaders();
loc = d->nextDeleted();
k++;
killCurrentOp.checkForInterrupt();
}
delBucketSizes << k;
}
catch (...) {
errors << ("exception in deleted chain for bucket " + BSONObjBuilder::numStr(i));
valid = false;
}
}
result.appendNumber("deletedCount", ndel);
result.appendNumber("deletedSize", delSize);
if ( full ) {
result << "delBucketSizes" << delBucketSizes.arr();
}
if ( incorrect ) {
errors << (BSONObjBuilder::numStr(incorrect) + " records from datafile are in deleted list");
valid = false;
}
int idxn = 0;
try {
IndexCatalog* indexCatalog = collection->getIndexCatalog();
result.append("nIndexes", nsd->getCompletedIndexCount());
BSONObjBuilder indexes; // not using subObjStart to be exception safe
NamespaceDetails::IndexIterator i = nsd->ii();
while( i.more() ) {
IndexDetails& id = i.next();
log() << "validating index " << idxn << ": " << id.indexNamespace() << endl;
IndexDescriptor* descriptor = indexCatalog->getDescriptor( idxn );
verify( descriptor );
IndexAccessMethod* iam = indexCatalog->getIndex( descriptor );
verify( iam );
int64_t keys;
iam->validate(&keys);
indexes.appendNumber(id.indexNamespace(), static_cast<long long>(keys));
idxn++;
}
result.append("keysPerIndex", indexes.done());
}
catch (...) {
errors << ("exception during index validate idxn " + BSONObjBuilder::numStr(idxn));
valid=false;
}
}
catch (AssertionException) {
errors << "exception during validate";
valid = false;
}
result.appendBool("valid", valid);
result.append("errors", errors.arr());
if ( !full ){
result.append("warning", "Some checks omitted for speed. use {full:true} option to do more thorough scan.");
}
if ( !valid ) {
result.append("advice", "ns corrupt, requires repair");
}
}
