bool doDBUpgrade( const string& dbName , string errmsg , DataFileHeader * h ) {
static DBDirectClient db;
if ( h->version == 4 && h->versionMinor == 4 ) {
verify( PDFILE_VERSION == 4 );
verify( PDFILE_VERSION_MINOR == 5 );
list<string> colls = db.getCollectionNames( dbName );
for ( list<string>::iterator i=colls.begin(); i!=colls.end(); i++) {
string c = *i;
log() << "\t upgrading collection:" << c << endl;
BSONObj out;
bool ok = db.runCommand( dbName , BSON( "reIndex" << c.substr( dbName.size() + 1 ) ) , out );
if ( ! ok ) {
errmsg = "reindex failed";
log() << "\t\t reindex failed: " << out << endl;
return false;
}
}
h->versionMinor = 5;
return true;
}
// do this in the general case
return repairDatabase( dbName.c_str(), errmsg );
}
void doDBUpgrade( const string& dbName, DataFileHeader* h ) {
static DBDirectClient db;
if ( h->version == 4 && h->versionMinor == 4 ) {
verify( PDFILE_VERSION == 4 );
verify( PDFILE_VERSION_MINOR_22_AND_OLDER == 5 );
list<string> colls = db.getCollectionNames( dbName );
for ( list<string>::iterator i=colls.begin(); i!=colls.end(); i++) {
string c = *i;
log() << "\t upgrading collection:" << c << endl;
BSONObj out;
bool ok = db.runCommand( dbName , BSON( "reIndex" << c.substr( dbName.size() + 1 ) ) , out );
if ( ! ok ) {
log() << "\t\t reindex failed: " << out;
fassertFailed( 17393 );
}
}
getDur().writingInt(h->versionMinor) = 5;
return;
}
// do this in the general case
fassert( 17401, repairDatabase( dbName ) );
}
Status ReplicationCoordinatorExternalStateImpl::runRepairOnLocalDB(OperationContext* opCtx) {
try {
Lock::GlobalWrite globalWrite(opCtx);
StorageEngine* engine = getGlobalServiceContext()->getGlobalStorageEngine();
if (!engine->isMmapV1()) {
return Status::OK();
}
UnreplicatedWritesBlock uwb(opCtx);
Status status = repairDatabase(opCtx, engine, localDbName, false, false);
// Open database before returning
dbHolder().openDb(opCtx, localDbName);
} catch (const DBException& ex) {
return ex.toStatus();
}
return Status::OK();
}
void repairDatabases() {
dblock lk;
vector< string > dbNames;
getDatabaseNames( dbNames );
for ( vector< string >::iterator i = dbNames.begin(); i != dbNames.end(); ++i ) {
string dbName = *i;
assert( !setClientTempNs( dbName.c_str() ) );
MongoDataFile *p = database->getFile( 0 );
MDFHeader *h = p->getHeader();
if ( !h->currentVersion() ) {
log() << "****" << endl;
log() << "****" << endl;
log() << "need to upgrade database " << dbName << " with pdfile version " << h->version << "." << h->versionMinor << ", "
<< "new version: " << VERSION << "." << VERSION_MINOR << endl;
if ( shouldRepairDatabases ){
// QUESTION: Repair even if file format is higher version than code?
log() << "\t starting repair" << endl;
string errmsg;
assert( repairDatabase( dbName.c_str(), errmsg ) );
}
else {
log() << "\t Not repairing, exiting!" << endl;
log() << "\t run --upgrade to upgrade dbs, then start again" << endl;
log() << "****" << endl;
dbexit( EXIT_NEED_UPGRADE );
shouldRepairDatabases = 1;
return;
}
} else {
closeClient( dbName.c_str() );
}
}
if ( shouldRepairDatabases ){
log() << "finished checking dbs" << endl;
dbexit( EXIT_CLEAN );
}
}
void repairDatabasesAndCheckVersion(OperationContext* txn) {
LOG(1) << "enter repairDatabases (to check pdfile version #)";
ScopedTransaction transaction(txn, MODE_X);
Lock::GlobalWrite lk(txn->lockState());
vector<string> dbNames;
StorageEngine* storageEngine = txn->getServiceContext()->getGlobalStorageEngine();
storageEngine->listDatabases(&dbNames);
// Repair all databases first, so that we do not try to open them if they are in bad shape
if (storageGlobalParams.repair) {
invariant(!storageGlobalParams.readOnly);
for (vector<string>::const_iterator i = dbNames.begin(); i != dbNames.end(); ++i) {
const string dbName = *i;
LOG(1) << " Repairing database: " << dbName;
fassert(18506, repairDatabase(txn, storageEngine, dbName));
}
}
const repl::ReplSettings& replSettings = repl::getGlobalReplicationCoordinator()->getSettings();
// On replica set members we only clear temp collections on DBs other than "local" during
// promotion to primary. On pure slaves, they are only cleared when the oplog tells them
// to. The local DB is special because it is not replicated. See SERVER-10927 for more
// details.
const bool shouldClearNonLocalTmpCollections =
!(checkIfReplMissingFromCommandLine(txn) || replSettings.usingReplSets() ||
replSettings.isSlave());
const bool shouldDoCleanupForSERVER23299 = isSubjectToSERVER23299(txn);
for (vector<string>::const_iterator i = dbNames.begin(); i != dbNames.end(); ++i) {
const string dbName = *i;
LOG(1) << " Recovering database: " << dbName;
Database* db = dbHolder().openDb(txn, dbName);
invariant(db);
// First thing after opening the database is to check for file compatibility,
// otherwise we might crash if this is a deprecated format.
auto status = db->getDatabaseCatalogEntry()->currentFilesCompatible(txn);
if (!status.isOK()) {
if (status.code() == ErrorCodes::CanRepairToDowngrade) {
// Convert CanRepairToDowngrade statuses to MustUpgrade statuses to avoid logging a
// potentially confusing and inaccurate message.
//
// TODO SERVER-24097: Log a message informing the user that they can start the
// current version of mongod with --repair and then proceed with normal startup.
status = {ErrorCodes::MustUpgrade, status.reason()};
}
severe() << "Unable to start mongod due to an incompatibility with the data files and"
" this version of mongod: "
<< redact(status);
severe() << "Please consult our documentation when trying to downgrade to a previous"
" major release";
quickExit(EXIT_NEED_UPGRADE);
return;
}
// Check if admin.system.version contains an invalid featureCompatibilityVersion.
// If a valid featureCompatibilityVersion is present, cache it as a server parameter.
if (dbName == "admin") {
if (Collection* versionColl =
db->getCollection(FeatureCompatibilityVersion::kCollection)) {
BSONObj featureCompatibilityVersion;
if (Helpers::findOne(txn,
versionColl,
BSON("_id" << FeatureCompatibilityVersion::kParameterName),
featureCompatibilityVersion)) {
auto version = FeatureCompatibilityVersion::parse(featureCompatibilityVersion);
if (!version.isOK()) {
severe() << version.getStatus();
fassertFailedNoTrace(40283);
}
serverGlobalParams.featureCompatibilityVersion.store(version.getValue());
}
}
}
// Major versions match, check indexes
const string systemIndexes = db->name() + ".system.indexes";
Collection* coll = db->getCollection(systemIndexes);
unique_ptr<PlanExecutor> exec(
InternalPlanner::collectionScan(txn, systemIndexes, coll, PlanExecutor::YIELD_MANUAL));
BSONObj index;
PlanExecutor::ExecState state;
while (PlanExecutor::ADVANCED == (state = exec->getNext(&index, NULL))) {
const BSONObj key = index.getObjectField("key");
const string plugin = IndexNames::findPluginName(key);
if (db->getDatabaseCatalogEntry()->isOlderThan24(txn)) {
if (IndexNames::existedBefore24(plugin)) {
continue;
}
log() << "Index " << index << " claims to be of type '" << plugin << "', "
<< "which is either invalid or did not exist before v2.4. "
<< "See the upgrade section: "
<< "http://dochub.mongodb.org/core/upgrade-2.4" << startupWarningsLog;
}
const Status keyStatus = validateKeyPattern(key);
if (!keyStatus.isOK()) {
log() << "Problem with index " << index << ": " << redact(keyStatus)
<< " This index can still be used however it cannot be rebuilt."
<< " For more info see"
<< " http://dochub.mongodb.org/core/index-validation" << startupWarningsLog;
}
if (index["v"].isNumber() && index["v"].numberInt() == 0) {
log() << "WARNING: The index: " << index << " was created with the deprecated"
<< " v:0 format. This format will not be supported in a future release."
<< startupWarningsLog;
log() << "\t To fix this, you need to rebuild this index."
<< " For instructions, see http://dochub.mongodb.org/core/rebuild-v0-indexes"
<< startupWarningsLog;
}
}
// Non-yielding collection scans from InternalPlanner will never error.
invariant(PlanExecutor::IS_EOF == state);
if (replSettings.usingReplSets()) {
// We only care about the _id index if we are in a replset
checkForIdIndexes(txn, db);
// Ensure oplog is capped (mmap does not guarantee order of inserts on noncapped
// collections)
if (db->name() == "local") {
checkForCappedOplog(txn, db);
}
}
if (shouldDoCleanupForSERVER23299) {
handleSERVER23299ForDb(txn, db);
}
if (!storageGlobalParams.readOnly &&
(shouldClearNonLocalTmpCollections || dbName == "local")) {
db->clearTmpCollections(txn);
}
}
LOG(1) << "done repairDatabases";
}
/**
* Return whether there are non-local databases. If there was an error becauses the wrong mongod
* version was used for these datafiles, a DBException with status ErrorCodes::MustDowngrade is
* thrown.
*/
bool repairDatabasesAndCheckVersion(OperationContext* opCtx) {
auto const storageEngine = opCtx->getServiceContext()->getStorageEngine();
Lock::GlobalWrite lk(opCtx);
std::vector<std::string> dbNames = storageEngine->listDatabases();
// Rebuilding indexes must be done before a database can be opened, except when using repair,
// which rebuilds all indexes when it is done.
if (!storageGlobalParams.readOnly && !storageGlobalParams.repair) {
// Determine whether this is a replica set node running in standalone mode. If we're in
// repair mode, we cannot set the flag yet as it needs to open a database and look through a
// collection. Rebuild the necessary indexes after setting the flag.
setReplSetMemberInStandaloneMode(opCtx);
rebuildIndexes(opCtx, storageEngine);
}
bool ensuredCollectionProperties = false;
// Repair all databases first, so that we do not try to open them if they are in bad shape
auto databaseHolder = DatabaseHolder::get(opCtx);
if (storageGlobalParams.repair) {
invariant(!storageGlobalParams.readOnly);
if (MONGO_FAIL_POINT(exitBeforeDataRepair)) {
log() << "Exiting because 'exitBeforeDataRepair' fail point was set.";
quickExit(EXIT_ABRUPT);
}
// Ensure that the local database is repaired first, if it exists, so that we can open it
// before any other database to be able to determine if this is a replica set node running
// in standalone mode before rebuilding any indexes.
auto dbNamesIt = std::find(dbNames.begin(), dbNames.end(), NamespaceString::kLocalDb);
if (dbNamesIt != dbNames.end()) {
std::swap(dbNames.front(), *dbNamesIt);
invariant(dbNames.front() == NamespaceString::kLocalDb);
}
stdx::function<void(const std::string& dbName)> onRecordStoreRepair =
[opCtx](const std::string& dbName) {
if (dbName == NamespaceString::kLocalDb) {
setReplSetMemberInStandaloneMode(opCtx);
}
};
for (const auto& dbName : dbNames) {
LOG(1) << " Repairing database: " << dbName;
fassertNoTrace(18506,
repairDatabase(opCtx, storageEngine, dbName, onRecordStoreRepair));
}
// All collections must have UUIDs before restoring the FCV document to a version that
// requires UUIDs.
uassertStatusOK(ensureCollectionProperties(opCtx, dbNames));
ensuredCollectionProperties = true;
// Attempt to restore the featureCompatibilityVersion document if it is missing.
NamespaceString fcvNSS(NamespaceString::kServerConfigurationNamespace);
auto db = databaseHolder->getDb(opCtx, fcvNSS.db());
Collection* versionColl;
BSONObj featureCompatibilityVersion;
if (!db || !(versionColl = db->getCollection(opCtx, fcvNSS)) ||
!Helpers::findOne(opCtx,
versionColl,
BSON("_id" << FeatureCompatibilityVersionParser::kParameterName),
featureCompatibilityVersion)) {
uassertStatusOK(restoreMissingFeatureCompatibilityVersionDocument(opCtx, dbNames));
}
}
if (!ensuredCollectionProperties) {
uassertStatusOK(ensureCollectionProperties(opCtx, dbNames));
}
if (!storageGlobalParams.readOnly) {
// We open the "local" database before calling hasReplSetConfigDoc() to ensure the in-memory
// catalog entries for the 'kSystemReplSetNamespace' collection have been populated if the
// collection exists. If the "local" database didn't exist at this point yet, then it will
// be created. If the mongod is running in a read-only mode, then it is fine to not open the
// "local" database and populate the catalog entries because we won't attempt to drop the
// temporary collections anyway.
Lock::DBLock dbLock(opCtx, NamespaceString::kSystemReplSetNamespace.db(), MODE_X);
databaseHolder->openDb(opCtx, NamespaceString::kSystemReplSetNamespace.db());
}
if (storageGlobalParams.repair) {
if (MONGO_FAIL_POINT(exitBeforeRepairInvalidatesConfig)) {
log() << "Exiting because 'exitBeforeRepairInvalidatesConfig' fail point was set.";
quickExit(EXIT_ABRUPT);
}
// This must be done after opening the "local" database as it modifies the replica set
// config.
auto repairObserver = StorageRepairObserver::get(opCtx->getServiceContext());
repairObserver->onRepairDone(opCtx);
if (repairObserver->isDataModified()) {
warning() << "Modifications made by repair:";
const auto& mods = repairObserver->getModifications();
for (const auto& mod : mods) {
warning() << " " << mod;
}
if (hasReplSetConfigDoc(opCtx)) {
warning() << "WARNING: Repair may have modified replicated data. This node will no "
"longer be able to join a replica set without a full re-sync";
}
}
}
const repl::ReplSettings& replSettings =
repl::ReplicationCoordinator::get(opCtx)->getSettings();
// On replica set members we only clear temp collections on DBs other than "local" during
// promotion to primary. On pure slaves, they are only cleared when the oplog tells them
// to. The local DB is special because it is not replicated. See SERVER-10927 for more
// details.
const bool shouldClearNonLocalTmpCollections =
!(hasReplSetConfigDoc(opCtx) || replSettings.usingReplSets());
// To check whether a featureCompatibilityVersion document exists.
bool fcvDocumentExists = false;
// To check whether we have databases other than local.
bool nonLocalDatabases = false;
// Refresh list of database names to include newly-created admin, if it exists.
dbNames = storageEngine->listDatabases();
for (const auto& dbName : dbNames) {
if (dbName != "local") {
nonLocalDatabases = true;
}
LOG(1) << " Recovering database: " << dbName;
auto db = databaseHolder->openDb(opCtx, dbName);
invariant(db);
// First thing after opening the database is to check for file compatibility,
// otherwise we might crash if this is a deprecated format.
auto status = storageEngine->currentFilesCompatible(opCtx);
if (!status.isOK()) {
if (status.code() == ErrorCodes::CanRepairToDowngrade) {
// Convert CanRepairToDowngrade statuses to MustUpgrade statuses to avoid logging a
// potentially confusing and inaccurate message.
//
// TODO SERVER-24097: Log a message informing the user that they can start the
// current version of mongod with --repair and then proceed with normal startup.
status = {ErrorCodes::MustUpgrade, status.reason()};
}
severe() << "Unable to start mongod due to an incompatibility with the data files and"
" this version of mongod: "
<< redact(status);
severe() << "Please consult our documentation when trying to downgrade to a previous"
" major release";
quickExit(EXIT_NEED_UPGRADE);
MONGO_UNREACHABLE;
}
// If the server configuration collection already contains a valid
// featureCompatibilityVersion document, cache it in-memory as a server parameter.
if (dbName == "admin") {
if (Collection* versionColl =
db->getCollection(opCtx, NamespaceString::kServerConfigurationNamespace)) {
BSONObj featureCompatibilityVersion;
if (Helpers::findOne(
opCtx,
versionColl,
BSON("_id" << FeatureCompatibilityVersionParser::kParameterName),
featureCompatibilityVersion)) {
auto swVersion =
FeatureCompatibilityVersionParser::parse(featureCompatibilityVersion);
// Note this error path captures all cases of an FCV document existing,
// but with any value other than "4.0" or "4.2". This includes unexpected
// cases with no path forward such as the FCV value not being a string.
uassert(ErrorCodes::MustDowngrade,
str::stream()
<< "UPGRADE PROBLEM: Found an invalid "
"featureCompatibilityVersion document (ERROR: "
<< swVersion.getStatus()
<< "). If the current featureCompatibilityVersion is below "
"4.0, see the documentation on upgrading at "
<< feature_compatibility_version_documentation::kUpgradeLink
<< ".",
swVersion.isOK());
fcvDocumentExists = true;
auto version = swVersion.getValue();
serverGlobalParams.featureCompatibility.setVersion(version);
FeatureCompatibilityVersion::updateMinWireVersion();
// On startup, if the version is in an upgrading or downrading state, print a
// warning.
if (version ==
ServerGlobalParams::FeatureCompatibility::Version::kUpgradingTo42) {
log() << "** WARNING: A featureCompatibilityVersion upgrade did not "
<< "complete. " << startupWarningsLog;
log() << "** The current featureCompatibilityVersion is "
<< FeatureCompatibilityVersionParser::toString(version) << "."
<< startupWarningsLog;
log() << "** To fix this, use the setFeatureCompatibilityVersion "
<< "command to resume upgrade to 4.2." << startupWarningsLog;
} else if (version == ServerGlobalParams::FeatureCompatibility::Version::
kDowngradingTo40) {
log() << "** WARNING: A featureCompatibilityVersion downgrade did not "
<< "complete. " << startupWarningsLog;
log() << "** The current featureCompatibilityVersion is "
<< FeatureCompatibilityVersionParser::toString(version) << "."
<< startupWarningsLog;
log() << "** To fix this, use the setFeatureCompatibilityVersion "
<< "command to resume downgrade to 4.0." << startupWarningsLog;
}
}
}
}
if (replSettings.usingReplSets()) {
// We only care about _id indexes and drop-pending collections if we are in a replset.
db->checkForIdIndexesAndDropPendingCollections(opCtx);
// Ensure oplog is capped (mongodb does not guarantee order of inserts on noncapped
// collections)
if (db->name() == "local") {
checkForCappedOplog(opCtx, db);
}
}
if (!storageGlobalParams.readOnly &&
(shouldClearNonLocalTmpCollections || dbName == "local")) {
db->clearTmpCollections(opCtx);
}
}
// Fail to start up if there is no featureCompatibilityVersion document and there are non-local
// databases present.
if (!fcvDocumentExists && nonLocalDatabases) {
severe()
<< "Unable to start up mongod due to missing featureCompatibilityVersion document.";
severe() << "Please run with --repair to restore the document.";
fassertFailedNoTrace(40652);
}
LOG(1) << "done repairDatabases";
return nonLocalDatabases;
}
