TEST_F(KVStorageEngineTest, RecreateIndexes) {
repl::setGlobalReplicationCoordinator(
new repl::ReplicationCoordinatorMock(getGlobalServiceContext(), repl::ReplSettings()));
auto opCtx = cc().makeOperationContext();
// Create two indexes for `db.coll1` in the catalog named `foo` and `bar`. Verify the indexes
// appear as idents in the KVEngine.
ASSERT_OK(createCollection(opCtx.get(), NamespaceString("db.coll1")).getStatus());
ASSERT_OK(createIndex(opCtx.get(), NamespaceString("db.coll1"), "foo"));
ASSERT_OK(createIndex(opCtx.get(), NamespaceString("db.coll1"), "bar"));
auto kvIdents = getAllKVEngineIdents(opCtx.get());
ASSERT_EQUALS(2, std::count_if(kvIdents.begin(), kvIdents.end(), [](const std::string& str) {
return str.find("index-") == 0;
}));
// Use the `getIndexNameObjs` to find the `foo` index in the IndexCatalog.
DatabaseCatalogEntry* dbce = _storageEngine->getDatabaseCatalogEntry(opCtx.get(), "db");
CollectionCatalogEntry* cce = dbce->getCollectionCatalogEntry("db.coll1");
auto swIndexNameObjs = getIndexNameObjs(
opCtx.get(), dbce, cce, [](const std::string& indexName) { return indexName == "foo"; });
ASSERT_OK(swIndexNameObjs.getStatus());
auto& indexNameObjs = swIndexNameObjs.getValue();
// There's one index that matched the name `foo`.
ASSERT_EQUALS(static_cast<const unsigned long>(1), indexNameObjs.first.size());
// Assert the parallel vectors have matching sizes.
ASSERT_EQUALS(static_cast<const unsigned long>(1), indexNameObjs.second.size());
// The index that matched should be named `foo`.
ASSERT_EQUALS("foo", indexNameObjs.first[0]);
ASSERT_EQUALS("db.coll1"_sd, indexNameObjs.second[0].getStringField("ns"));
ASSERT_EQUALS("foo"_sd, indexNameObjs.second[0].getStringField("name"));
ASSERT_EQUALS(2, indexNameObjs.second[0].getIntField("v"));
ASSERT_EQUALS(1, indexNameObjs.second[0].getObjectField("key").getIntField("foo"));
// Drop the `foo` index table. Count one remaining index ident according to the KVEngine.
ASSERT_OK(dropIndexTable(opCtx.get(), NamespaceString("db.coll1"), "foo"));
kvIdents = getAllKVEngineIdents(opCtx.get());
ASSERT_EQUALS(1, std::count_if(kvIdents.begin(), kvIdents.end(), [](const std::string& str) {
return str.find("index-") == 0;
}));
AutoGetCollection coll(opCtx.get(), NamespaceString("db.coll1"), LockMode::MODE_X);
// Find the `foo` index in the catalog. Rebuild it. Count two indexes in the KVEngine.
ASSERT_OK(rebuildIndexesOnCollection(opCtx.get(), dbce, cce, indexNameObjs));
ASSERT_TRUE(cce->isIndexReady(opCtx.get(), "foo"));
kvIdents = getAllKVEngineIdents(opCtx.get());
ASSERT_EQUALS(2, std::count_if(kvIdents.begin(), kvIdents.end(), [](const std::string& str) {
return str.find("index-") == 0;
}));
}
void openCatalog(OperationContext* opCtx, const MinVisibleTimestampMap& minVisibleTimestampMap) {
invariant(opCtx->lockState()->isW());
// Load the catalog in the storage engine.
log() << "openCatalog: loading storage engine catalog";
auto storageEngine = opCtx->getServiceContext()->getStorageEngine();
storageEngine->loadCatalog(opCtx);
log() << "openCatalog: reconciling catalog and idents";
auto indexesToRebuild = storageEngine->reconcileCatalogAndIdents(opCtx);
fassert(40688, indexesToRebuild.getStatus());
// Determine which indexes need to be rebuilt. rebuildIndexesOnCollection() requires that all
// indexes on that collection are done at once, so we use a map to group them together.
StringMap<IndexNameObjs> nsToIndexNameObjMap;
for (auto indexNamespace : indexesToRebuild.getValue()) {
NamespaceString collNss(indexNamespace.first);
auto indexName = indexNamespace.second;
auto dbCatalogEntry = storageEngine->getDatabaseCatalogEntry(opCtx, collNss.db());
invariant(dbCatalogEntry,
str::stream() << "couldn't get database catalog entry for database "
<< collNss.db());
auto collCatalogEntry = dbCatalogEntry->getCollectionCatalogEntry(collNss.toString());
invariant(collCatalogEntry,
str::stream() << "couldn't get collection catalog entry for collection "
<< collNss.toString());
auto indexSpecs = getIndexNameObjs(
opCtx, dbCatalogEntry, collCatalogEntry, [&indexName](const std::string& name) {
return name == indexName;
});
if (!indexSpecs.isOK() || indexSpecs.getValue().first.empty()) {
fassert(40689,
{ErrorCodes::InternalError,
str::stream() << "failed to get index spec for index " << indexName
<< " in collection "
<< collNss.toString()});
}
auto indexesToRebuild = indexSpecs.getValue();
invariant(
indexesToRebuild.first.size() == 1,
str::stream() << "expected to find a list containing exactly 1 index name, but found "
<< indexesToRebuild.first.size());
invariant(
indexesToRebuild.second.size() == 1,
str::stream() << "expected to find a list containing exactly 1 index spec, but found "
<< indexesToRebuild.second.size());
auto& ino = nsToIndexNameObjMap[collNss.ns()];
ino.first.emplace_back(std::move(indexesToRebuild.first.back()));
ino.second.emplace_back(std::move(indexesToRebuild.second.back()));
}
for (const auto& entry : nsToIndexNameObjMap) {
NamespaceString collNss(entry.first);
auto dbCatalogEntry = storageEngine->getDatabaseCatalogEntry(opCtx, collNss.db());
invariant(dbCatalogEntry,
str::stream() << "couldn't get database catalog entry for database "
<< collNss.db());
auto collCatalogEntry = dbCatalogEntry->getCollectionCatalogEntry(collNss.toString());
invariant(collCatalogEntry,
str::stream() << "couldn't get collection catalog entry for collection "
<< collNss.toString());
for (const auto& indexName : entry.second.first) {
log() << "openCatalog: rebuilding index: collection: " << collNss.toString()
<< ", index: " << indexName;
}
std::vector<BSONObj> indexSpecs = entry.second.second;
fassert(40690,
rebuildIndexesOnCollection(opCtx, dbCatalogEntry, collCatalogEntry, indexSpecs));
}
// Open all databases and repopulate the UUID catalog.
log() << "openCatalog: reopening all databases";
auto databaseHolder = DatabaseHolder::get(opCtx);
std::vector<std::string> databasesToOpen;
storageEngine->listDatabases(&databasesToOpen);
for (auto&& dbName : databasesToOpen) {
LOG(1) << "openCatalog: dbholder reopening database " << dbName;
auto db = databaseHolder->openDb(opCtx, dbName);
invariant(db, str::stream() << "failed to reopen database " << dbName);
std::list<std::string> collections;
db->getDatabaseCatalogEntry()->getCollectionNamespaces(&collections);
for (auto&& collName : collections) {
// Note that the collection name already includes the database component.
NamespaceString collNss(collName);
auto collection = db->getCollection(opCtx, collName);
invariant(collection,
str::stream() << "failed to get valid collection pointer for namespace "
<< collName);
auto uuid = collection->uuid();
invariant(uuid);
if (minVisibleTimestampMap.count(*uuid) > 0) {
collection->setMinimumVisibleSnapshot(minVisibleTimestampMap.find(*uuid)->second);
}
// If this is the oplog collection, re-establish the replication system's cached pointer
// to the oplog.
if (collNss.isOplog()) {
log() << "openCatalog: updating cached oplog pointer";
collection->establishOplogCollectionForLogging(opCtx);
}
}
}
// Opening UUID Catalog: The UUID catalog is now in sync with the storage engine catalog. Clear
// the pre-closing state.
UUIDCatalog::get(opCtx).onOpenCatalog(opCtx);
LOG(1) << "openCatalog: finished reloading UUID catalog";
}
