void run() { OperationContextImpl txn; Client::WriteContext ctx(&txn, ns()); DBDirectClient db(&txn); db.insert(ns(), BSON("x" << 1 << "y" << 2)); db.insert(ns(), BSON("x" << 2 << "y" << 2)); Collection* collection = ctx.ctx().db()->getCollection( &txn, ns() ); ASSERT( collection ); IndexCatalog* indexCatalog = collection->getIndexCatalog(); ASSERT_EQUALS(1, indexCatalog->numIndexesReady(&txn)); // _id index ASSERT_EQUALS(1U, db.count("test.system.indexes")); // test.buildindex // test.buildindex_$id // test.system.indexes ASSERT_EQUALS(3U, db.count("test.system.namespaces")); db.ensureIndex(ns(), BSON("y" << 1), true); ASSERT_EQUALS(1, indexCatalog->numIndexesReady(&txn)); ASSERT_EQUALS(1U, db.count("test.system.indexes")); ASSERT_EQUALS(3U, db.count("test.system.namespaces")); db.ensureIndex(ns(), BSON("x" << 1), true); ctx.commit(); ASSERT_EQUALS(2, indexCatalog->numIndexesReady(&txn)); ASSERT_EQUALS(2U, db.count("test.system.indexes")); ASSERT_EQUALS(4U, db.count("test.system.namespaces")); }
bool Helpers::findById(OperationContext* opCtx, Database* database, StringData ns, BSONObj query, BSONObj& result, bool* nsFound, bool* indexFound) { invariant(database); Collection* collection = database->getCollection(opCtx, ns); if (!collection) { return false; } if (nsFound) *nsFound = true; IndexCatalog* catalog = collection->getIndexCatalog(); const IndexDescriptor* desc = catalog->findIdIndex(opCtx); if (!desc) return false; if (indexFound) *indexFound = 1; RecordId loc = catalog->getIndex(desc)->findSingle(opCtx, query["_id"].wrap()); if (loc.isNull()) return false; result = collection->docFor(opCtx, loc).value(); return true; }
void run() { const ServiceContext::UniqueOperationContext txnPtr = cc().makeOperationContext(); OperationContext& txn = *txnPtr; OldClientWriteContext ctx(&txn, ns()); DBDirectClient db(&txn); db.insert(ns(), BSON("x" << 1 << "y" << 2)); db.insert(ns(), BSON("x" << 2 << "y" << 2)); Collection* collection = ctx.getCollection(); ASSERT(collection); IndexCatalog* indexCatalog = collection->getIndexCatalog(); ASSERT_EQUALS(1, indexCatalog->numIndexesReady(&txn)); // _id index ASSERT_EQUALS(1U, db.getIndexSpecs(ns()).size()); ASSERT_EQUALS(ErrorCodes::DuplicateKey, dbtests::createIndex(&txn, ns(), BSON("y" << 1), true)); ASSERT_EQUALS(1, indexCatalog->numIndexesReady(&txn)); ASSERT_EQUALS(1U, db.getIndexSpecs(ns()).size()); ASSERT_OK(dbtests::createIndex(&txn, ns(), BSON("x" << 1), true)); ASSERT_EQUALS(2, indexCatalog->numIndexesReady(&txn)); ASSERT_EQUALS(2U, db.getIndexSpecs(ns()).size()); }
void configureSystemIndexes(OperationContext* txn, const StringData& dbname) { int authzVersion; Status status = getGlobalAuthorizationManager()->getAuthorizationVersion( txn, &authzVersion); if (!status.isOK()) { return; } if (dbname == "admin" && authzVersion >= AuthorizationManager::schemaVersion26Final) { NamespaceString systemUsers(dbname, "system.users"); // Make sure the old unique index from v2.4 on system.users doesn't exist. Client::WriteContext wctx(txn, systemUsers); Collection* collection = wctx.ctx().db()->getCollection(txn, NamespaceString(systemUsers)); if (!collection) { return; } IndexCatalog* indexCatalog = collection->getIndexCatalog(); IndexDescriptor* oldIndex = NULL; while ((oldIndex = indexCatalog->findIndexByKeyPattern(txn, v1SystemUsersKeyPattern))) { indexCatalog->dropIndex(txn, oldIndex); } wctx.commit(); } }
RecordId Helpers::findById(OperationContext* txn, Collection* collection, const BSONObj& idquery) { verify(collection); IndexCatalog* catalog = collection->getIndexCatalog(); const IndexDescriptor* desc = catalog->findIdIndex(txn); uassert(13430, "no _id index", desc); return catalog->getIndex(desc)->findSingle(txn, idquery["_id"].wrap()); }
void run() { OperationContextImpl txn; Client::WriteContext ctx(&txn, ns()); DBDirectClient db(&txn); db.insert(ns(), BSON("x" << 1 << "y" << 2)); db.insert(ns(), BSON("x" << 2 << "y" << 2)); Collection* collection = ctx.ctx().db()->getCollection( &txn, ns() ); ASSERT( collection ); IndexCatalog* indexCatalog = collection->getIndexCatalog(); ASSERT_EQUALS(1, indexCatalog->numIndexesReady(&txn)); // _id index ASSERT_EQUALS(1U, db.getIndexSpecs(ns()).size()); db.ensureIndex(ns(), BSON("y" << 1), true); ASSERT_EQUALS(1, indexCatalog->numIndexesReady(&txn)); ASSERT_EQUALS(1U, db.getIndexSpecs(ns()).size()); db.ensureIndex(ns(), BSON("x" << 1), true); ctx.commit(); ASSERT_EQUALS(2, indexCatalog->numIndexesReady(&txn)); ASSERT_EQUALS(2U, db.getIndexSpecs(ns()).size()); }
bool Helpers::findById(Database* database, const char *ns, BSONObj query, BSONObj& result , bool* nsFound , bool* indexFound ) { Lock::assertAtLeastReadLocked(ns); invariant( database ); Collection* collection = database->getCollection( ns ); if ( !collection ) { return false; } if ( nsFound ) *nsFound = true; IndexCatalog* catalog = collection->getIndexCatalog(); const IndexDescriptor* desc = catalog->findIdIndex(); if ( !desc ) return false; if ( indexFound ) *indexFound = 1; // See SERVER-12397. This may not always be true. BtreeBasedAccessMethod* accessMethod = static_cast<BtreeBasedAccessMethod*>(catalog->getIndex( desc )); DiskLoc loc = accessMethod->findSingle( query["_id"].wrap() ); if ( loc.isNull() ) return false; result = collection->docFor( loc ); return true; }
bool Helpers::findById(Client& c, const char *ns, BSONObj query, BSONObj& result , bool* nsFound , bool* indexFound ) { Lock::assertAtLeastReadLocked(ns); Database *database = c.database(); verify( database ); Collection* collection = database->getCollection( ns ); if ( !collection ) { return false; } if ( nsFound ) *nsFound = true; IndexCatalog* catalog = collection->getIndexCatalog(); const IndexDescriptor* desc = catalog->findIdIndex(); if ( !desc ) return false; if ( indexFound ) *indexFound = 1; BtreeBasedAccessMethod* accessMethod = catalog->getBtreeBasedIndex( desc ); DiskLoc loc = accessMethod->findSingle( query["_id"].wrap() ); if ( loc.isNull() ) return false; result = collection->docFor( loc ); return true; }
Collection* Database::createCollection(OperationContext* txn, StringData ns, const CollectionOptions& options, bool createIdIndex) { massert(17399, "collection already exists", getCollection(ns) == NULL); massertNamespaceNotIndex(ns, "createCollection"); invariant(txn->lockState()->isDbLockedForMode(name(), MODE_X)); if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer && !(ns.startsWith("config.") || ns.startsWith("local.") || ns.startsWith("admin."))) { uasserted(14037, "can't create user databases on a --configsvr instance"); } if (NamespaceString::normal(ns)) { // This check only applies for actual collections, not indexes or other types of ns. uassert(17381, str::stream() << "fully qualified namespace " << ns << " is too long " << "(max is " << NamespaceString::MaxNsCollectionLen << " bytes)", ns.size() <= NamespaceString::MaxNsCollectionLen); } NamespaceString nss(ns); uassert(17316, "cannot create a blank collection", nss.coll() > 0); uassert(28838, "cannot create a non-capped oplog collection", options.capped || !nss.isOplog()); audit::logCreateCollection(&cc(), ns); txn->recoveryUnit()->registerChange(new AddCollectionChange(txn, this, ns)); Status status = _dbEntry->createCollection(txn, ns, options, true /*allocateDefaultSpace*/); massertNoTraceStatusOK(status); Collection* collection = _getOrCreateCollectionInstance(txn, ns); invariant(collection); _collections[ns] = collection; if (createIdIndex) { if (collection->requiresIdIndex()) { if (options.autoIndexId == CollectionOptions::YES || options.autoIndexId == CollectionOptions::DEFAULT) { IndexCatalog* ic = collection->getIndexCatalog(); uassertStatusOK(ic->createIndexOnEmptyCollection(txn, ic->getDefaultIdIndexSpec())); } } if (nss.isSystem()) { authindex::createSystemIndexes(txn, collection); } } auto opObserver = getGlobalServiceContext()->getOpObserver(); if (opObserver) opObserver->onCreateCollection(txn, nss, options); return collection; }
DiskLoc Helpers::findById(Collection* collection, const BSONObj& idquery) { verify(collection); IndexCatalog* catalog = collection->getIndexCatalog(); const IndexDescriptor* desc = catalog->findIdIndex(); uassert(13430, "no _id index", desc); BtreeBasedAccessMethod* accessMethod = catalog->getBtreeBasedIndex( desc ); return accessMethod->findSingle( idquery["_id"].wrap() ); }
void run() { string ns = "unittests.rollback_set_index_head"; const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext(); OperationContext& opCtx = *opCtxPtr; NamespaceString nss(ns); dropDatabase(&opCtx, nss); createCollection(&opCtx, nss); AutoGetDb autoDb(&opCtx, nss.db(), MODE_X); Collection* coll = autoDb.getDb()->getCollection(&opCtx, nss); IndexCatalog* catalog = coll->getIndexCatalog(); string idxName = "a"; BSONObj spec = BSON("ns" << ns << "key" << BSON("a" << 1) << "name" << idxName << "v" << static_cast<int>(kIndexVersion)); { WriteUnitOfWork uow(&opCtx); ASSERT_OK(catalog->createIndexOnEmptyCollection(&opCtx, spec)); uow.commit(); } IndexDescriptor* indexDesc = catalog->findIndexByName(&opCtx, idxName); invariant(indexDesc); const IndexCatalogEntry* ice = catalog->getEntry(indexDesc); invariant(ice); HeadManager* headManager = ice->headManager(); const RecordId oldHead = headManager->getHead(&opCtx); ASSERT_EQ(oldHead, ice->head(&opCtx)); const RecordId dummyHead(123, 456); ASSERT_NE(oldHead, dummyHead); // END SETUP / START TEST { WriteUnitOfWork uow(&opCtx); headManager->setHead(&opCtx, dummyHead); ASSERT_EQ(ice->head(&opCtx), dummyHead); ASSERT_EQ(headManager->getHead(&opCtx), dummyHead); if (!rollback) { uow.commit(); } } if (rollback) { ASSERT_EQ(ice->head(&opCtx), oldHead); ASSERT_EQ(headManager->getHead(&opCtx), oldHead); } else { ASSERT_EQ(ice->head(&opCtx), dummyHead); ASSERT_EQ(headManager->getHead(&opCtx), dummyHead); } }
DiskLoc Helpers::findById(Collection* collection, const BSONObj& idquery) { verify(collection); IndexCatalog* catalog = collection->getIndexCatalog(); const IndexDescriptor* desc = catalog->findIdIndex(); uassert(13430, "no _id index", desc); // See SERVER-12397. This may not always be true. BtreeBasedAccessMethod* accessMethod = static_cast<BtreeBasedAccessMethod*>(catalog->getIndex( desc )); return accessMethod->findSingle( idquery["_id"].wrap() ); }
Collection* Database::createCollection( OperationContext* txn, const StringData& ns, const CollectionOptions& options, bool allocateDefaultSpace, bool createIdIndex ) { massert( 17399, "collection already exists", getCollection( txn, ns ) == NULL ); massertNamespaceNotIndex( ns, "createCollection" ); if ( serverGlobalParams.configsvr && !( ns.startsWith( "config." ) || ns.startsWith( "local." ) || ns.startsWith( "admin." ) ) ) { uasserted(14037, "can't create user databases on a --configsvr instance"); } if (NamespaceString::normal(ns)) { // This check only applies for actual collections, not indexes or other types of ns. uassert(17381, str::stream() << "fully qualified namespace " << ns << " is too long " << "(max is " << NamespaceString::MaxNsCollectionLen << " bytes)", ns.size() <= NamespaceString::MaxNsCollectionLen); } NamespaceString nss( ns ); uassert( 17316, "cannot create a blank collection", nss.coll() > 0 ); audit::logCreateCollection( currentClient.get(), ns ); txn->recoveryUnit()->registerChange( new AddCollectionChange(this, ns) ); Status status = _dbEntry->createCollection(txn, ns, options, allocateDefaultSpace); massertStatusOK(status); Collection* collection = getCollection(txn, ns); invariant(collection); if ( createIdIndex ) { if ( collection->requiresIdIndex() ) { if ( options.autoIndexId == CollectionOptions::YES || options.autoIndexId == CollectionOptions::DEFAULT ) { IndexCatalog* ic = collection->getIndexCatalog(); uassertStatusOK( ic->createIndexOnEmptyCollection(txn, ic->getDefaultIdIndexSpec())); } } if ( nss.isSystem() ) { authindex::createSystemIndexes( txn, collection ); } } return collection; }
Collection* Database::createCollection(OperationContext* txn, StringData ns, const CollectionOptions& options, bool createIdIndex) { invariant(txn->lockState()->isDbLockedForMode(name(), MODE_X)); invariant(!options.isView()); NamespaceString nss(ns); _checkCanCreateCollection(nss, options); audit::logCreateCollection(&cc(), ns); txn->recoveryUnit()->registerChange(new AddCollectionChange(txn, this, ns)); Status status = _dbEntry->createCollection(txn, ns, options, true /*allocateDefaultSpace*/); massertNoTraceStatusOK(status); Collection* collection = _getOrCreateCollectionInstance(txn, ns); invariant(collection); _collections[ns] = collection; if (createIdIndex) { if (collection->requiresIdIndex()) { if (options.autoIndexId == CollectionOptions::YES || options.autoIndexId == CollectionOptions::DEFAULT) { // The creation of the _id index isn't replicated and is instead implicit in the // creation of the collection. This means that the version of the _id index to build // is technically unspecified. However, we're able to use the // featureCompatibilityVersion of this server to determine the default index version // to use because we apply commands (opType == 'c') in their own batch. This // guarantees the write to the admin.system.version collection from the // "setFeatureCompatibilityVersion" command either happens entirely before the // collection creation or it happens entirely after. const auto featureCompatibilityVersion = serverGlobalParams.featureCompatibilityVersion.load(); IndexCatalog* ic = collection->getIndexCatalog(); uassertStatusOK(ic->createIndexOnEmptyCollection( txn, ic->getDefaultIdIndexSpec(featureCompatibilityVersion))); } } if (nss.isSystem()) { authindex::createSystemIndexes(txn, collection); } } auto opObserver = getGlobalServiceContext()->getOpObserver(); if (opObserver) opObserver->onCreateCollection(txn, nss, options); return collection; }
void run() { string ns = "unittests.rollback_create_collection_and_indexes"; const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext(); OperationContext& opCtx = *opCtxPtr; NamespaceString nss(ns); dropDatabase(&opCtx, nss); Lock::DBLock dbXLock(&opCtx, nss.db(), MODE_X); OldClientContext ctx(&opCtx, nss.ns()); string idxNameA = "indexA"; string idxNameB = "indexB"; string idxNameC = "indexC"; BSONObj specA = BSON("ns" << ns << "key" << BSON("a" << 1) << "name" << idxNameA << "v" << static_cast<int>(kIndexVersion)); BSONObj specB = BSON("ns" << ns << "key" << BSON("b" << 1) << "name" << idxNameB << "v" << static_cast<int>(kIndexVersion)); BSONObj specC = BSON("ns" << ns << "key" << BSON("c" << 1) << "name" << idxNameC << "v" << static_cast<int>(kIndexVersion)); // END SETUP / START TEST { WriteUnitOfWork uow(&opCtx); ASSERT(!collectionExists(&ctx, nss.ns())); ASSERT_OK(userCreateNS( &opCtx, ctx.db(), nss.ns(), BSONObj(), CollectionOptions::parseForCommand, false)); ASSERT(collectionExists(&ctx, nss.ns())); Collection* coll = ctx.db()->getCollection(&opCtx, nss); IndexCatalog* catalog = coll->getIndexCatalog(); ASSERT_OK(catalog->createIndexOnEmptyCollection(&opCtx, specA)); ASSERT_OK(catalog->createIndexOnEmptyCollection(&opCtx, specB)); ASSERT_OK(catalog->createIndexOnEmptyCollection(&opCtx, specC)); if (!rollback) { uow.commit(); } } // uow if (rollback) { ASSERT(!collectionExists(&ctx, ns)); } else { ASSERT(collectionExists(&ctx, ns)); ASSERT(indexReady(&opCtx, nss, idxNameA)); ASSERT(indexReady(&opCtx, nss, idxNameB)); ASSERT(indexReady(&opCtx, nss, idxNameC)); } }
void run() { string ns = "unittests.rollback_drop_index"; const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext(); OperationContext& opCtx = *opCtxPtr; NamespaceString nss(ns); dropDatabase(&opCtx, nss); createCollection(&opCtx, nss); AutoGetDb autoDb(&opCtx, nss.db(), MODE_X); Collection* coll = autoDb.getDb()->getCollection(&opCtx, nss); IndexCatalog* catalog = coll->getIndexCatalog(); string idxName = "a"; BSONObj spec = BSON("ns" << ns << "key" << BSON("a" << 1) << "name" << idxName << "v" << static_cast<int>(kIndexVersion)); { WriteUnitOfWork uow(&opCtx); ASSERT_OK(catalog->createIndexOnEmptyCollection(&opCtx, spec)); insertRecord(&opCtx, nss, BSON("a" << 1)); insertRecord(&opCtx, nss, BSON("a" << 2)); insertRecord(&opCtx, nss, BSON("a" << 3)); uow.commit(); } ASSERT(indexReady(&opCtx, nss, idxName)); ASSERT_EQ(3u, getNumIndexEntries(&opCtx, nss, idxName)); // END SETUP / START TEST { WriteUnitOfWork uow(&opCtx); dropIndex(&opCtx, nss, idxName); ASSERT(!indexExists(&opCtx, nss, idxName)); if (!rollback) { uow.commit(); } } if (rollback) { ASSERT(indexExists(&opCtx, nss, idxName)); ASSERT(indexReady(&opCtx, nss, idxName)); ASSERT_EQ(3u, getNumIndexEntries(&opCtx, nss, idxName)); } else { ASSERT(!indexExists(&opCtx, nss, idxName)); } }
Collection* Database::createCollection(OperationContext* txn, StringData ns, const CollectionOptions& options, bool createIdIndex, const BSONObj& idIndex) { invariant(txn->lockState()->isDbLockedForMode(name(), MODE_X)); invariant(!options.isView()); NamespaceString nss(ns); _checkCanCreateCollection(nss, options); audit::logCreateCollection(&cc(), ns); Status status = _dbEntry->createCollection(txn, ns, options, true /*allocateDefaultSpace*/); massertNoTraceStatusOK(status); txn->recoveryUnit()->registerChange(new AddCollectionChange(txn, this, ns)); Collection* collection = _getOrCreateCollectionInstance(txn, ns); invariant(collection); _collections[ns] = collection; BSONObj fullIdIndexSpec; if (createIdIndex) { if (collection->requiresIdIndex()) { if (options.autoIndexId == CollectionOptions::YES || options.autoIndexId == CollectionOptions::DEFAULT) { const auto featureCompatibilityVersion = serverGlobalParams.featureCompatibility.version.load(); IndexCatalog* ic = collection->getIndexCatalog(); fullIdIndexSpec = uassertStatusOK(ic->createIndexOnEmptyCollection( txn, !idIndex.isEmpty() ? idIndex : ic->getDefaultIdIndexSpec(featureCompatibilityVersion))); } } if (nss.isSystem()) { authindex::createSystemIndexes(txn, collection); } } auto opObserver = getGlobalServiceContext()->getOpObserver(); if (opObserver) opObserver->onCreateCollection(txn, nss, options, fullIdIndexSpec); return collection; }
IndexScan* createIndexScan(MatchExpression* expr, WorkingSet* ws) { IndexCatalog* catalog = _coll->getIndexCatalog(); IndexDescriptor* descriptor = catalog->findIndexByKeyPattern(&_txn, BSON("x" << 1)); invariant(descriptor); // We are not testing indexing here so use maximal bounds IndexScanParams params; params.descriptor = descriptor; params.bounds.isSimpleRange = true; params.bounds.startKey = BSON("" << 0); params.bounds.endKey = BSON("" << kDocuments+1); params.bounds.endKeyInclusive = true; params.direction = 1; // This child stage gets owned and freed by its parent CountStage return new IndexScan(&_txn, params, ws, expr); }
IndexScan* createIndexScanSimpleRange(BSONObj startKey, BSONObj endKey) { IndexCatalog* catalog = _coll->getIndexCatalog(); std::vector<IndexDescriptor*> indexes; catalog->findIndexesByKeyPattern(&_opCtx, BSON("x" << 1), false, &indexes); ASSERT_EQ(indexes.size(), 1U); // We are not testing indexing here so use maximal bounds IndexScanParams params(&_opCtx, *indexes[0]); params.bounds.isSimpleRange = true; params.bounds.startKey = startKey; params.bounds.endKey = endKey; params.bounds.boundInclusion = BoundInclusion::kIncludeBothStartAndEndKeys; params.direction = 1; // This child stage gets owned and freed by the caller. MatchExpression* filter = NULL; return new IndexScan(&_opCtx, params, &_ws, filter); }
IndexScan* createIndexScanSimpleRange(BSONObj startKey, BSONObj endKey) { IndexCatalog* catalog = _coll->getIndexCatalog(); IndexDescriptor* descriptor = catalog->findIndexByKeyPattern(&_txn, BSON("x" << 1)); invariant(descriptor); // We are not testing indexing here so use maximal bounds IndexScanParams params; params.descriptor = descriptor; params.bounds.isSimpleRange = true; params.bounds.startKey = startKey; params.bounds.endKey = endKey; params.bounds.endKeyInclusive = true; params.direction = 1; // This child stage gets owned and freed by the caller. MatchExpression* filter = NULL; return new IndexScan(&_txn, params, &_ws, filter); }
uint64_t Collection::getIndexSize(OperationContext* opCtx, BSONObjBuilder* details, int scale) { IndexCatalog* idxCatalog = getIndexCatalog(); IndexCatalog::IndexIterator ii = idxCatalog->getIndexIterator(opCtx, true); uint64_t totalSize = 0; while (ii.more()) { IndexDescriptor* d = ii.next(); IndexAccessMethod* iam = idxCatalog->getIndex(d); long long ds = iam->getSpaceUsedBytes(opCtx); totalSize += ds; if (details) { details->appendNumber(d->indexName(), ds / scale); } } return totalSize; }
IndexScan* createIndexScan(BSONObj startKey, BSONObj endKey, bool startInclusive, bool endInclusive, int direction = 1) { IndexCatalog* catalog = _coll->getIndexCatalog(); IndexDescriptor* descriptor = catalog->findIndexByKeyPattern(&_txn, BSON("x" << 1)); invariant(descriptor); IndexScanParams params; params.descriptor = descriptor; params.direction = direction; OrderedIntervalList oil("x"); BSONObjBuilder bob; bob.appendAs(startKey.firstElement(), ""); bob.appendAs(endKey.firstElement(), ""); oil.intervals.push_back(Interval(bob.obj(), startInclusive, endInclusive)); params.bounds.fields.push_back(oil); MatchExpression* filter = NULL; return new IndexScan(&_txn, params, &_ws, filter); }
IndexScan* createIndexScan(BSONObj startKey, BSONObj endKey, bool startInclusive, bool endInclusive, int direction = 1) { IndexCatalog* catalog = _coll->getIndexCatalog(); std::vector<IndexDescriptor*> indexes; catalog->findIndexesByKeyPattern(&_opCtx, BSON("x" << 1), false, &indexes); ASSERT_EQ(indexes.size(), 1U); IndexScanParams params(&_opCtx, *indexes[0]); params.direction = direction; OrderedIntervalList oil("x"); BSONObjBuilder bob; bob.appendAs(startKey.firstElement(), ""); bob.appendAs(endKey.firstElement(), ""); oil.intervals.push_back(Interval(bob.obj(), startInclusive, endInclusive)); params.bounds.fields.push_back(oil); MatchExpression* filter = NULL; return new IndexScan(&_opCtx, params, &_ws, filter); }
bool Helpers::findById(OperationContext* txn, Database* database, const char *ns, BSONObj query, BSONObj& result, bool* nsFound, bool* indexFound) { invariant(database); Collection* collection = database->getCollection( ns ); if ( !collection ) { return false; } if ( nsFound ) *nsFound = true; IndexCatalog* catalog = collection->getIndexCatalog(); const IndexDescriptor* desc = catalog->findIdIndex( txn ); if ( !desc ) return false; if ( indexFound ) *indexFound = 1; // See SERVER-12397. This may not always be true. BtreeBasedAccessMethod* accessMethod = static_cast<BtreeBasedAccessMethod*>(catalog->getIndex( desc )); RecordId loc = accessMethod->findSingle( txn, query["_id"].wrap() ); if ( loc.isNull() ) return false; result = collection->docFor(txn, loc).value(); return true; }
PlanStage::StageState TwoDNear::work(WorkingSetID* out) { ++_commonStats.works; // Adds the amount of time taken by work() to executionTimeMillis. ScopedTimer timer(&_commonStats.executionTimeMillis); if (!_initted) { _initted = true; if ( !_params.collection ) return PlanStage::IS_EOF; IndexCatalog* indexCatalog = _params.collection->getIndexCatalog(); IndexDescriptor* desc = indexCatalog->findIndexByKeyPattern(_params.indexKeyPattern); if ( desc == NULL ) return PlanStage::IS_EOF; TwoDAccessMethod* am = static_cast<TwoDAccessMethod*>( indexCatalog->getIndex( desc ) ); auto_ptr<twod_exec::GeoSearch> search; search.reset(new twod_exec::GeoSearch(_params.collection, am, _params.nearQuery.centroid.oldPoint, _params.numWanted, _params.filter, _params.nearQuery.maxDistance, _params.nearQuery.isNearSphere ? twod_exec::GEO_SPHERE : twod_exec::GEO_PLANE)); // This is where all the work is done. :( search->exec(); _specificStats.objectsLoaded = search->_objectsLoaded; _specificStats.nscanned = search->_lookedAt; for (twod_exec::GeoHopper::Holder::iterator it = search->_points.begin(); it != search->_points.end(); it++) { WorkingSetID id = _workingSet->allocate(); WorkingSetMember* member = _workingSet->get(id); member->loc = it->_loc; member->obj = _params.collection->docFor(member->loc); member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; if (_params.addDistMeta) { member->addComputed(new GeoDistanceComputedData(it->_distance)); } if (_params.addPointMeta) { member->addComputed(new GeoNearPointComputedData(it->_pt)); } _results.push(Result(id, it->_distance)); _invalidationMap.insert(pair<DiskLoc, WorkingSetID>(it->_loc, id)); } } if (isEOF()) { return PlanStage::IS_EOF; } Result result = _results.top(); _results.pop(); *out = result.id; // Remove from invalidation map. WorkingSetMember* member = _workingSet->get(*out); // The WSM may have been mutated or deleted so it may not have a loc. if (member->hasLoc()) { typedef multimap<DiskLoc, WorkingSetID>::iterator MMIT; pair<MMIT, MMIT> range = _invalidationMap.equal_range(member->loc); for (MMIT it = range.first; it != range.second; ++it) { if (it->second == *out) { _invalidationMap.erase(it); break; } } } ++_commonStats.advanced; return PlanStage::ADVANCED; }
void IndexRebuilder::checkNS(const std::list<std::string>& nsToCheck) { bool firstTime = true; for (std::list<std::string>::const_iterator it = nsToCheck.begin(); it != nsToCheck.end(); ++it) { string ns = *it; LOG(3) << "IndexRebuilder::checkNS: " << ns; OperationContextImpl txn; // XXX??? // This write lock is held throughout the index building process // for this namespace. Client::WriteContext ctx(&txn, ns); Collection* collection = ctx.ctx().db()->getCollection( &txn, ns ); if ( collection == NULL ) continue; IndexCatalog* indexCatalog = collection->getIndexCatalog(); if ( collection->ns().isOplog() && indexCatalog->numIndexesTotal() > 0 ) { warning() << ns << " had illegal indexes, removing"; indexCatalog->dropAllIndexes(&txn, true); continue; } vector<BSONObj> indexesToBuild = indexCatalog->getAndClearUnfinishedIndexes(&txn); // The indexes have now been removed from system.indexes, so the only record is // in-memory. If there is a journal commit between now and when insert() rewrites // the entry and the db crashes before the new system.indexes entry is journalled, // the index will be lost forever. Thus, we're assuming no journaling will happen // between now and the entry being re-written. if ( indexesToBuild.size() == 0 ) { continue; } log() << "found " << indexesToBuild.size() << " interrupted index build(s) on " << ns; if (firstTime) { log() << "note: restart the server with --noIndexBuildRetry to skip index rebuilds"; firstTime = false; } if (!serverGlobalParams.indexBuildRetry) { log() << " not rebuilding interrupted indexes"; continue; } // TODO: these can/should/must be done in parallel for ( size_t i = 0; i < indexesToBuild.size(); i++ ) { BSONObj indexObj = indexesToBuild[i]; log() << "going to rebuild: " << indexObj; Status status = indexCatalog->createIndex(&txn, indexObj, false); if ( !status.isOK() ) { log() << "building index failed: " << status.toString() << " index: " << indexObj; } } } }
Status appendCollectionStorageStats(OperationContext* opCtx, const NamespaceString& nss, const BSONObj& param, BSONObjBuilder* result) { int scale = 1; if (param["scale"].isNumber()) { scale = param["scale"].numberInt(); if (scale < 1) { return {ErrorCodes::BadValue, "scale has to be >= 1"}; } } else if (param["scale"].trueValue()) { return {ErrorCodes::BadValue, "scale has to be a number >= 1"}; } bool verbose = param["verbose"].trueValue(); AutoGetCollectionForReadCommand ctx(opCtx, nss); Collection* collection = ctx.getCollection(); // Will be set if present if (!ctx.getDb() || !collection) { result->appendNumber("size", 0); result->appendNumber("count", 0); result->appendNumber("storageSize", 0); result->append("nindexes", 0); result->appendNumber("totalIndexSize", 0); result->append("indexDetails", BSONObj()); result->append("indexSizes", BSONObj()); std::string errmsg = !(ctx.getDb()) ? "Database [" + nss.db().toString() + "] not found." : "Collection [" + nss.toString() + "] not found."; return {ErrorCodes::NamespaceNotFound, errmsg}; } long long size = collection->dataSize(opCtx) / scale; result->appendNumber("size", size); long long numRecords = collection->numRecords(opCtx); result->appendNumber("count", numRecords); if (numRecords) result->append("avgObjSize", collection->averageObjectSize(opCtx)); RecordStore* recordStore = collection->getRecordStore(); result->appendNumber( "storageSize", static_cast<long long>(recordStore->storageSize(opCtx, result, verbose ? 1 : 0)) / scale); recordStore->appendCustomStats(opCtx, result, scale); IndexCatalog* indexCatalog = collection->getIndexCatalog(); result->append("nindexes", indexCatalog->numIndexesReady(opCtx)); BSONObjBuilder indexDetails; IndexCatalog::IndexIterator i = indexCatalog->getIndexIterator(opCtx, false); while (i.more()) { const IndexDescriptor* descriptor = i.next(); IndexAccessMethod* iam = indexCatalog->getIndex(descriptor); invariant(iam); BSONObjBuilder bob; if (iam->appendCustomStats(opCtx, &bob, scale)) { indexDetails.append(descriptor->indexName(), bob.obj()); } } result->append("indexDetails", indexDetails.obj()); BSONObjBuilder indexSizes; long long indexSize = collection->getIndexSize(opCtx, &indexSizes, scale); result->appendNumber("totalIndexSize", indexSize / scale); result->append("indexSizes", indexSizes.obj()); return Status::OK(); }
bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& anObjBuilder, bool /*fromRepl*/) { BSONElement e = jsobj.firstElement(); const string toDeleteNs = dbname + '.' + e.valuestr(); if (!serverGlobalParams.quiet) { MONGO_TLOG(0) << "CMD: dropIndexes " << toDeleteNs << endl; } Lock::DBWrite dbXLock(dbname); Client::Context ctx(toDeleteNs); Collection* collection = cc().database()->getCollection( toDeleteNs ); if ( ! collection ) { errmsg = "ns not found"; return false; } stopIndexBuilds(cc().database(), jsobj); IndexCatalog* indexCatalog = collection->getIndexCatalog(); anObjBuilder.appendNumber("nIndexesWas", indexCatalog->numIndexesTotal() ); BSONElement f = jsobj.getField("index"); if ( f.type() == String ) { string indexToDelete = f.valuestr(); if ( indexToDelete == "*" ) { Status s = indexCatalog->dropAllIndexes( false ); if ( !s.isOK() ) { appendCommandStatus( anObjBuilder, s ); return false; } anObjBuilder.append("msg", "non-_id indexes dropped for collection"); return true; } IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByName( indexToDelete ); if ( desc == NULL ) { errmsg = str::stream() << "index not found with name [" << indexToDelete << "]"; return false; } if ( desc->isIdIndex() ) { errmsg = "cannot drop _id index"; return false; } Status s = indexCatalog->dropIndex( desc ); if ( !s.isOK() ) { appendCommandStatus( anObjBuilder, s ); return false; } return true; } if ( f.type() == Object ) { IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByKeyPattern( f.embeddedObject() ); if ( desc == NULL ) { errmsg = "can't find index with key:"; errmsg += f.embeddedObject().toString(); return false; } if ( desc->isIdIndex() ) { errmsg = "cannot drop _id index"; return false; } Status s = indexCatalog->dropIndex( desc ); if ( !s.isOK() ) { appendCommandStatus( anObjBuilder, s ); return false; } return true; } errmsg = "invalid index name spec"; return false; }
bool wrappedRun(OperationContext* txn, const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& anObjBuilder) { const std::string coll = jsobj.firstElement().valuestrsafe(); if (coll.empty()) { errmsg = "no collection name specified"; return false; } const std::string toDeleteNs = dbname + '.' + coll; if (!serverGlobalParams.quiet) { LOG(0) << "CMD: dropIndexes " << toDeleteNs << endl; } Client::Context ctx(txn, toDeleteNs); Database* db = ctx.db(); Collection* collection = db->getCollection( txn, toDeleteNs ); if ( ! collection ) { errmsg = "ns not found"; return false; } stopIndexBuilds(txn, db, jsobj); IndexCatalog* indexCatalog = collection->getIndexCatalog(); anObjBuilder.appendNumber("nIndexesWas", indexCatalog->numIndexesTotal(txn) ); BSONElement f = jsobj.getField("index"); if ( f.type() == String ) { string indexToDelete = f.valuestr(); if ( indexToDelete == "*" ) { Status s = indexCatalog->dropAllIndexes(txn, false); if ( !s.isOK() ) { appendCommandStatus( anObjBuilder, s ); return false; } anObjBuilder.append("msg", "non-_id indexes dropped for collection"); return true; } IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByName( txn, indexToDelete ); if ( desc == NULL ) { errmsg = str::stream() << "index not found with name [" << indexToDelete << "]"; return false; } if ( desc->isIdIndex() ) { errmsg = "cannot drop _id index"; return false; } Status s = indexCatalog->dropIndex(txn, desc); if ( !s.isOK() ) { appendCommandStatus( anObjBuilder, s ); return false; } return true; } if ( f.type() == Object ) { IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByKeyPattern( txn, f.embeddedObject() ); if ( desc == NULL ) { errmsg = "can't find index with key:"; errmsg += f.embeddedObject().toString(); return false; } if ( desc->isIdIndex() ) { errmsg = "cannot drop _id index"; return false; } Status s = indexCatalog->dropIndex(txn, desc); if ( !s.isOK() ) { appendCommandStatus( anObjBuilder, s ); return false; } return true; } errmsg = "invalid index name spec"; return false; }
void validateNS(const string& ns, Collection* collection, const BSONObj& cmdObj, BSONObjBuilder& result) { const bool full = cmdObj["full"].trueValue(); const bool scanData = full || cmdObj["scandata"].trueValue(); NamespaceDetails* nsd = collection->details(); bool valid = true; BSONArrayBuilder errors; // explanation(s) for why valid = false if ( collection->isCapped() ){ result.append("capped", nsd->isCapped()); result.appendNumber("max", nsd->maxCappedDocs()); } if ( nsd->firstExtent().isNull() ) result.append( "firstExtent", "null" ); else result.append( "firstExtent", str::stream() << nsd->firstExtent().toString() << " ns:" << nsd->firstExtent().ext()->nsDiagnostic.toString()); if ( nsd->lastExtent().isNull() ) result.append( "lastExtent", "null" ); else result.append( "lastExtent", str::stream() << nsd->lastExtent().toString() << " ns:" << nsd->lastExtent().ext()->nsDiagnostic.toString()); BSONArrayBuilder extentData; int extentCount = 0; try { if ( !nsd->firstExtent().isNull() ) { nsd->firstExtent().ext()->assertOk(); nsd->lastExtent().ext()->assertOk(); } DiskLoc extentDiskLoc = nsd->firstExtent(); while (!extentDiskLoc.isNull()) { Extent* thisExtent = extentDiskLoc.ext(); if (full) { extentData << thisExtent->dump(); } if (!thisExtent->validates(extentDiskLoc, &errors)) { valid = false; } DiskLoc nextDiskLoc = thisExtent->xnext; if (extentCount > 0 && !nextDiskLoc.isNull() && nextDiskLoc.ext()->xprev != extentDiskLoc) { StringBuilder sb; sb << "'xprev' pointer " << nextDiskLoc.ext()->xprev.toString() << " in extent " << nextDiskLoc.toString() << " does not point to extent " << extentDiskLoc.toString(); errors << sb.str(); valid = false; } if (nextDiskLoc.isNull() && extentDiskLoc != nsd->lastExtent()) { StringBuilder sb; sb << "'lastExtent' pointer " << nsd->lastExtent().toString() << " does not point to last extent in list " << extentDiskLoc.toString(); errors << sb.str(); valid = false; } extentDiskLoc = nextDiskLoc; extentCount++; killCurrentOp.checkForInterrupt(); } } catch (const DBException& e) { StringBuilder sb; sb << "exception validating extent " << extentCount << ": " << e.what(); errors << sb.str(); valid = false; } result.append("extentCount", extentCount); if ( full ) result.appendArray( "extents" , extentData.arr() ); result.appendNumber("datasize", nsd->dataSize()); result.appendNumber("nrecords", nsd->numRecords()); result.appendNumber("lastExtentSize", nsd->lastExtentSize()); result.appendNumber("padding", nsd->paddingFactor()); try { bool testingLastExtent = false; try { if (nsd->firstExtent().isNull()) { // this is ok } else { result.append("firstExtentDetails", nsd->firstExtent().ext()->dump()); if (!nsd->firstExtent().ext()->xprev.isNull()) { StringBuilder sb; sb << "'xprev' pointer in 'firstExtent' " << nsd->firstExtent().toString() << " is " << nsd->firstExtent().ext()->xprev.toString() << ", should be null"; errors << sb.str(); valid=false; } } testingLastExtent = true; if (nsd->lastExtent().isNull()) { // this is ok } else { if (nsd->firstExtent() != nsd->lastExtent()) { result.append("lastExtentDetails", nsd->lastExtent().ext()->dump()); if (!nsd->lastExtent().ext()->xnext.isNull()) { StringBuilder sb; sb << "'xnext' pointer in 'lastExtent' " << nsd->lastExtent().toString() << " is " << nsd->lastExtent().ext()->xnext.toString() << ", should be null"; errors << sb.str(); valid = false; } } } } catch (const DBException& e) { StringBuilder sb; sb << "exception processing '" << (testingLastExtent ? "lastExtent" : "firstExtent") << "': " << e.what(); errors << sb.str(); valid = false; } set<DiskLoc> recs; if( scanData ) { int n = 0; int nInvalid = 0; long long nQuantizedSize = 0; long long nPowerOf2QuantizedSize = 0; long long len = 0; long long nlen = 0; long long bsonLen = 0; int outOfOrder = 0; DiskLoc cl_last; DiskLoc cl; Runner::RunnerState state; auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns)); while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &cl))) { n++; if ( n < 1000000 ) recs.insert(cl); if ( nsd->isCapped() ) { if ( cl < cl_last ) outOfOrder++; cl_last = cl; } Record *r = cl.rec(); len += r->lengthWithHeaders(); nlen += r->netLength(); if ( r->lengthWithHeaders() == NamespaceDetails::quantizeAllocationSpace ( r->lengthWithHeaders() ) ) { // Count the number of records having a size consistent with // the quantizeAllocationSpace quantization implementation. ++nQuantizedSize; } if ( r->lengthWithHeaders() == NamespaceDetails::quantizePowerOf2AllocationSpace ( r->lengthWithHeaders() - 1 ) ) { // Count the number of records having a size consistent with the // quantizePowerOf2AllocationSpace quantization implementation. // Because of SERVER-8311, power of 2 quantization is not idempotent and // r->lengthWithHeaders() - 1 must be checked instead of the record // length itself. ++nPowerOf2QuantizedSize; } if (full){ BSONObj obj = BSONObj::make(r); if (!obj.isValid() || !obj.valid()){ // both fast and deep checks valid = false; if (nInvalid == 0) // only log once; errors << "invalid bson object detected (see logs for more info)"; nInvalid++; if (strcmp("_id", obj.firstElementFieldName()) == 0){ try { obj.firstElement().validate(); // throws on error log() << "Invalid bson detected in " << ns << " with _id: " << obj.firstElement().toString(false) << endl; } catch(...){ log() << "Invalid bson detected in " << ns << " with corrupt _id" << endl; } } else { log() << "Invalid bson detected in " << ns << " and couldn't find _id" << endl; } } else { bsonLen += obj.objsize(); } } } if (Runner::RUNNER_EOF != state) { // TODO: more descriptive logging. warning() << "Internal error while reading collection " << ns << endl; } if ( nsd->isCapped() && !nsd->capLooped() ) { result.append("cappedOutOfOrder", outOfOrder); if ( outOfOrder > 1 ) { valid = false; errors << "too many out of order records"; } } result.append("objectsFound", n); if (full) { result.append("invalidObjects", nInvalid); } result.appendNumber("nQuantizedSize", nQuantizedSize); result.appendNumber("nPowerOf2QuantizedSize", nPowerOf2QuantizedSize); result.appendNumber("bytesWithHeaders", len); result.appendNumber("bytesWithoutHeaders", nlen); if (full) { result.appendNumber("bytesBson", bsonLen); } } BSONArrayBuilder deletedListArray; for ( int i = 0; i < Buckets; i++ ) { deletedListArray << nsd->deletedListEntry(i).isNull(); } int ndel = 0; long long delSize = 0; BSONArrayBuilder delBucketSizes; int incorrect = 0; for ( int i = 0; i < Buckets; i++ ) { DiskLoc loc = nsd->deletedListEntry(i); try { int k = 0; while ( !loc.isNull() ) { if ( recs.count(loc) ) incorrect++; ndel++; if ( loc.questionable() ) { if( nsd->isCapped() && !loc.isValid() && i == 1 ) { /* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid see comments in namespace.h */ break; } string err( str::stream() << "bad pointer in deleted record list: " << loc.toString() << " bucket: " << i << " k: " << k ); errors << err; valid = false; break; } DeletedRecord *d = loc.drec(); delSize += d->lengthWithHeaders(); loc = d->nextDeleted(); k++; killCurrentOp.checkForInterrupt(); } delBucketSizes << k; } catch (...) { errors << ("exception in deleted chain for bucket " + BSONObjBuilder::numStr(i)); valid = false; } } result.appendNumber("deletedCount", ndel); result.appendNumber("deletedSize", delSize); if ( full ) { result << "delBucketSizes" << delBucketSizes.arr(); } if ( incorrect ) { errors << (BSONObjBuilder::numStr(incorrect) + " records from datafile are in deleted list"); valid = false; } int idxn = 0; try { IndexCatalog* indexCatalog = collection->getIndexCatalog(); result.append("nIndexes", nsd->getCompletedIndexCount()); BSONObjBuilder indexes; // not using subObjStart to be exception safe NamespaceDetails::IndexIterator i = nsd->ii(); while( i.more() ) { IndexDetails& id = i.next(); log() << "validating index " << idxn << ": " << id.indexNamespace() << endl; IndexDescriptor* descriptor = indexCatalog->getDescriptor( idxn ); verify( descriptor ); IndexAccessMethod* iam = indexCatalog->getIndex( descriptor ); verify( iam ); int64_t keys; iam->validate(&keys); indexes.appendNumber(id.indexNamespace(), static_cast<long long>(keys)); idxn++; } result.append("keysPerIndex", indexes.done()); } catch (...) { errors << ("exception during index validate idxn " + BSONObjBuilder::numStr(idxn)); valid=false; } } catch (AssertionException) { errors << "exception during validate"; valid = false; } result.appendBool("valid", valid); result.append("errors", errors.arr()); if ( !full ){ result.append("warning", "Some checks omitted for speed. use {full:true} option to do more thorough scan."); } if ( !valid ) { result.append("advice", "ns corrupt, requires repair"); } }