/**
* Upgrade v0 to v7 described here
*
* This upgrade takes the config server from empty to an initial version.
*/
bool doUpgradeV0ToV7(CatalogManager* catalogManager,
const VersionType& lastVersionInfo,
string* errMsg) {
string dummy;
if (!errMsg)
errMsg = &dummy;
verify(lastVersionInfo.getCurrentVersion() == UpgradeHistory_EmptyVersion);
//
// Even though the initial config write is a single-document update, that single document
// is on multiple config servers and requests can interleave. The upgrade lock prevents
// this.
//
log() << "writing initial config version at v" << CURRENT_CONFIG_VERSION;
OID newClusterId = OID::gen();
VersionType versionInfo;
// Upgrade to new version
versionInfo.setMinCompatibleVersion(MIN_COMPATIBLE_CONFIG_VERSION);
versionInfo.setCurrentVersion(CURRENT_CONFIG_VERSION);
versionInfo.setClusterId(newClusterId);
verify(versionInfo.isValid(NULL));
// If the cluster has not previously been initialized, we need to set the version before
// using so subsequent mongoses use the config data the same way. This requires all three
// config servers online initially.
Status result = catalogManager->update(VersionType::ConfigNS,
BSON("_id" << 1),
versionInfo.toBSON(),
true, // upsert
false, // multi
NULL);
if (!result.isOK()) {
*errMsg = stream() << "error writing initial config version: " << result.reason();
return false;
}
return true;
}
/**
* Upgrade v3 to v4 described here.
*
* This upgrade takes a config server without collection epochs (potentially) and adds
* epochs to all mongo processes.
*
*/
bool doUpgradeV3ToV4(const ConnectionString& configLoc,
const VersionType& lastVersionInfo,
string* errMsg)
{
string dummy;
if (!errMsg) errMsg = &dummy;
verify(lastVersionInfo.getCurrentVersion() == UpgradeHistory_NoEpochVersion);
if (lastVersionInfo.isUpgradeIdSet() && lastVersionInfo.getUpgradeId().isSet()) {
//
// Another upgrade failed, so cleanup may be necessary
//
BSONObj lastUpgradeState = lastVersionInfo.getUpgradeState();
bool inCriticalSection;
if (!FieldParser::extract(lastUpgradeState,
inCriticalSectionField,
&inCriticalSection,
errMsg))
{
*errMsg = stream() << "problem reading previous upgrade state" << causedBy(errMsg);
return false;
}
if (inCriticalSection) {
// Manual intervention is needed here. Somehow our upgrade didn't get applied
// consistently across config servers.
*errMsg = cannotCleanupMessage;
return false;
}
if (!_cleanupUpgradeState(configLoc, lastVersionInfo.getUpgradeId(), errMsg)) {
// If we can't cleanup the old upgrade state, the user might have done it for us,
// not a fatal problem (we'll just end up with extra collections).
warning() << "could not cleanup previous upgrade state" << causedBy(errMsg) << endl;
*errMsg = "";
}
}
//
// Check the versions of other mongo processes in the cluster before upgrade.
// We can't upgrade if there are active pre-v2.2 processes in the cluster
//
Status mongoVersionStatus = checkClusterMongoVersions(configLoc,
string(minMongoProcessVersion));
if (!mongoVersionStatus.isOK()) {
*errMsg = stream() << "cannot upgrade with pre-v" << minMongoProcessVersion
<< " mongo processes active in the cluster"
<< causedBy(mongoVersionStatus);
return false;
}
VersionType newVersionInfo;
lastVersionInfo.cloneTo(&newVersionInfo);
// Set our upgrade id and state
OID upgradeId = OID::gen();
newVersionInfo.setUpgradeId(upgradeId);
newVersionInfo.setUpgradeState(BSONObj());
// Write our upgrade id and state
{
scoped_ptr<ScopedDbConnection> connPtr;
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
verify(newVersionInfo.isValid(NULL));
conn->update(VersionType::ConfigNS,
BSON("_id" << 1 << VersionType::version_DEPRECATED(3)),
newVersionInfo.toBSON());
_checkGLE(conn);
}
catch (const DBException& e) {
*errMsg = stream() << "could not initialize version info for upgrade"
<< causedBy(e);
return false;
}
connPtr->done();
}
//
// First lock all collection namespaces that exist
//
OwnedPointerMap<string, CollectionType> ownedCollections;
const map<string, CollectionType*>& collections = ownedCollections.map();
Status findCollectionsStatus = findAllCollectionsV3(configLoc, &ownedCollections);
if (!findCollectionsStatus.isOK()) {
*errMsg = stream() << "could not read collections from config server"
<< causedBy(findCollectionsStatus);
return false;
}
//
// Acquire locks for all sharded collections
// Something that didn't involve getting thousands of locks would be better.
//
OwnedPointerVector<ScopedDistributedLock> collectionLocks;
log() << "acquiring locks for " << collections.size() << " sharded collections..." << endl;
// WARNING - this string is used programmatically when forcing locks, be careful when
// changing!
// TODO: Add programmatic "why" field to lock collection
string lockMessage = str::stream() << "ensuring epochs for config upgrade"
<< " (" << upgradeId.toString() << ")";
if (!_acquireAllCollectionLocks(configLoc,
collections,
lockMessage,
20 * 60 * 1000,
&collectionLocks,
errMsg))
{
*errMsg = stream() << "could not acquire all namespace locks for upgrade"
<< " (" << upgradeId.toString() << ")"
<< causedBy(errMsg);
return false;
}
// We are now preventing all splits and migrates for all sharded collections
// Get working and backup suffixes
string workingSuffix = genWorkingSuffix(upgradeId);
string backupSuffix = genBackupSuffix(upgradeId);
log() << "copying collection and chunk metadata to working and backup collections..."
<< endl;
// Get a backup and working copy of the config.collections and config.chunks collections
Status copyStatus = copyFrozenCollection(configLoc,
CollectionType::ConfigNS,
CollectionType::ConfigNS + workingSuffix);
if (!copyStatus.isOK()) {
*errMsg = stream() << "could not copy " << CollectionType::ConfigNS << " to "
<< (CollectionType::ConfigNS + workingSuffix)
<< causedBy(copyStatus);
return false;
}
copyStatus = copyFrozenCollection(configLoc,
CollectionType::ConfigNS,
CollectionType::ConfigNS + backupSuffix);
if (!copyStatus.isOK()) {
*errMsg = stream() << "could not copy " << CollectionType::ConfigNS << " to "
<< (CollectionType::ConfigNS + backupSuffix) << causedBy(copyStatus);
return false;
}
copyStatus = copyFrozenCollection(configLoc,
ChunkType::ConfigNS,
ChunkType::ConfigNS + workingSuffix);
if (!copyStatus.isOK()) {
*errMsg = stream() << "could not copy " << ChunkType::ConfigNS << " to "
<< (ChunkType::ConfigNS + workingSuffix) << causedBy(copyStatus);
return false;
}
copyStatus = copyFrozenCollection(configLoc,
ChunkType::ConfigNS,
ChunkType::ConfigNS + backupSuffix);
if (!copyStatus.isOK()) {
*errMsg = stream() << "could not copy " << ChunkType::ConfigNS << " to "
<< (ChunkType::ConfigNS + backupSuffix) << causedBy(copyStatus);
return false;
}
//
// Go through sharded collections one-by-one and add epochs where missing
//
for (map<string, CollectionType*>::const_iterator it = collections.begin();
it != collections.end(); ++it)
{
// Create a copy so that we can change the epoch later
CollectionType collection;
it->second->cloneTo(&collection);
log() << "checking epochs for " << collection.getNS() << " collection..." << endl;
OID epoch = collection.getEpoch();
//
// Go through chunks to find epoch if we haven't found it or to verify epoch is the same
//
OwnedPointerVector<ChunkType> ownedChunks;
const vector<ChunkType*>& chunks = ownedChunks.vector();
Status findChunksStatus = findAllChunks(configLoc, collection.getNS(), &ownedChunks);
if (!findChunksStatus.isOK()) {
*errMsg = stream() << "could not read chunks from config server"
<< causedBy(findChunksStatus);
return false;
}
for (vector<ChunkType*>::const_iterator chunkIt = chunks.begin();
chunkIt != chunks.end(); ++chunkIt)
{
const ChunkType& chunk = *(*chunkIt);
// If our chunk epoch is set and doesn't match
if (epoch.isSet() && chunk.getVersion().epoch().isSet()
&& chunk.getVersion().epoch() != epoch)
{
*errMsg = stream() << "chunk epoch for " << chunk.toString() << " in "
<< collection.getNS() << " does not match found epoch "
<< epoch;
return false;
}
else if (!epoch.isSet() && chunk.getVersion().epoch().isSet()) {
epoch = chunk.getVersion().epoch();
}
}
//
// Write collection epoch if needed
//
if (!collection.getEpoch().isSet()) {
OID newEpoch = OID::gen();
log() << "writing new epoch " << newEpoch << " for " << collection.getNS()
<< " collection..." << endl;
scoped_ptr<ScopedDbConnection> connPtr;
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
conn->update(CollectionType::ConfigNS + workingSuffix,
BSON(CollectionType::ns(collection.getNS())),
BSON("$set" << BSON(CollectionType::DEPRECATED_lastmodEpoch(newEpoch))));
_checkGLE(conn);
}
catch (const DBException& e) {
*errMsg = stream() << "could not write a new epoch for " << collection.getNS()
<< causedBy(e);
return false;
}
connPtr->done();
collection.setEpoch(newEpoch);
}
epoch = collection.getEpoch();
verify(epoch.isSet());
//
// Now write verified epoch to all chunks
//
log() << "writing epoch " << epoch << " for " << chunks.size() << " chunks in "
<< collection.getNS() << " collection..." << endl;
{
scoped_ptr<ScopedDbConnection> connPtr;
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
// Multi-update of all chunks
conn->update(ChunkType::ConfigNS + workingSuffix,
BSON(ChunkType::ns(collection.getNS())),
BSON("$set" << BSON(ChunkType::DEPRECATED_epoch(epoch))),
false,
true); // multi
_checkGLE(conn);
}
catch (const DBException& e) {
*errMsg = stream() << "could not write a new epoch " << epoch.toString()
<< " for chunks in " << collection.getNS() << causedBy(e);
return false;
}
connPtr->done();
}
}
//
// Paranoid verify the collection writes
//
{
scoped_ptr<ScopedDbConnection> connPtr;
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
// Find collections with no epochs
BSONObj emptyDoc =
conn->findOne(CollectionType::ConfigNS + workingSuffix,
BSON("$unset" << BSON(CollectionType::DEPRECATED_lastmodEpoch() << 1)));
if (!emptyDoc.isEmpty()) {
*errMsg = stream() << "collection " << emptyDoc
<< " is still missing epoch after config upgrade";
connPtr->done();
return false;
}
// Find collections with empty epochs
emptyDoc = conn->findOne(CollectionType::ConfigNS + workingSuffix,
BSON(CollectionType::DEPRECATED_lastmodEpoch(OID())));
if (!emptyDoc.isEmpty()) {
*errMsg = stream() << "collection " << emptyDoc
<< " still has empty epoch after config upgrade";
connPtr->done();
return false;
}
// Find chunks with no epochs
emptyDoc =
conn->findOne(ChunkType::ConfigNS + workingSuffix,
BSON("$unset" << BSON(ChunkType::DEPRECATED_epoch() << 1)));
if (!emptyDoc.isEmpty()) {
*errMsg = stream() << "chunk " << emptyDoc
<< " is still missing epoch after config upgrade";
connPtr->done();
return false;
}
// Find chunks with empty epochs
emptyDoc = conn->findOne(ChunkType::ConfigNS + workingSuffix,
BSON(ChunkType::DEPRECATED_epoch(OID())));
if (!emptyDoc.isEmpty()) {
*errMsg = stream() << "chunk " << emptyDoc
<< " still has empty epoch after config upgrade";
connPtr->done();
return false;
}
}
catch (const DBException& e) {
*errMsg = stream() << "could not verify epoch writes" << causedBy(e);
return false;
}
connPtr->done();
}
//
// Double check that our collections haven't changed
//
Status idCheckStatus = checkIdsTheSame(configLoc,
CollectionType::ConfigNS,
CollectionType::ConfigNS + workingSuffix);
if (!idCheckStatus.isOK()) {
*errMsg = stream() << CollectionType::ConfigNS
<< " was modified while working on upgrade"
<< causedBy(idCheckStatus);
return false;
}
idCheckStatus = checkIdsTheSame(configLoc,
ChunkType::ConfigNS,
ChunkType::ConfigNS + workingSuffix);
if (!idCheckStatus.isOK()) {
*errMsg = stream() << ChunkType::ConfigNS << " was modified while working on upgrade"
<< causedBy(idCheckStatus);
return false;
}
//
// ENTER CRITICAL SECTION
//
newVersionInfo.setUpgradeState(BSON(inCriticalSectionField(true)));
{
scoped_ptr<ScopedDbConnection> connPtr;
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
verify(newVersionInfo.isValid(NULL));
conn->update(VersionType::ConfigNS,
BSON("_id" << 1 << VersionType::version_DEPRECATED(3)),
newVersionInfo.toBSON());
_checkGLE(conn);
}
catch (const DBException& e) {
// No cleanup message here since we're not sure if we wrote or not, and
// not dangerous either way except to prevent further updates (at which point
// the message is printed)
*errMsg = stream()
<< "could not update version info to enter critical update section"
<< causedBy(e);
return false;
}
// AT THIS POINT ANY FAILURE REQUIRES MANUAL INTERVENTION!
connPtr->done();
}
log() << "entered critical section for config upgrade" << endl;
Status overwriteStatus = overwriteCollection(configLoc,
CollectionType::ConfigNS + workingSuffix,
CollectionType::ConfigNS);
if (!overwriteStatus.isOK()) {
error() << cleanupMessage << endl;
*errMsg = stream() << "could not overwrite collection " << CollectionType::ConfigNS
<< " with working collection "
<< (CollectionType::ConfigNS + workingSuffix)
<< causedBy(overwriteStatus);
return false;
}
overwriteStatus = overwriteCollection(configLoc,
ChunkType::ConfigNS + workingSuffix,
ChunkType::ConfigNS);
if (!overwriteStatus.isOK()) {
error() << cleanupMessage << endl;
*errMsg = stream() << "could not overwrite collection " << ChunkType::ConfigNS
<< " with working collection "
<< (ChunkType::ConfigNS + workingSuffix)
<< causedBy(overwriteStatus);
return false;
}
//
// Finally update the version to latest and add clusterId to version
//
OID newClusterId = OID::gen();
// Note: hardcoded versions, since this is a very particular upgrade
// Note: DO NOT CLEAR the config version unless bumping the minCompatibleVersion,
// we want to save the excludes that were set.
newVersionInfo.setMinCompatibleVersion(UpgradeHistory_NoEpochVersion);
newVersionInfo.setCurrentVersion(UpgradeHistory_MandatoryEpochVersion);
newVersionInfo.setClusterId(newClusterId);
// Leave critical section
newVersionInfo.unsetUpgradeId();
newVersionInfo.unsetUpgradeState();
log() << "writing new version info and clusterId " << newClusterId << "..." << endl;
{
scoped_ptr<ScopedDbConnection> connPtr;
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
verify(newVersionInfo.isValid(NULL));
conn->update(VersionType::ConfigNS,
BSON("_id" << 1 << VersionType::version_DEPRECATED(UpgradeHistory_NoEpochVersion)),
newVersionInfo.toBSON());
_checkGLE(conn);
}
catch (const DBException& e) {
error() << cleanupMessage << endl;
*errMsg = stream() << "could not write new version info "
<< "and exit critical upgrade section" << causedBy(e);
return false;
}
connPtr->done();
}
//
// END CRITICAL SECTION
//
return true;
}
Status CatalogManagerReplicaSet::initConfigVersion(OperationContext* txn) {
for (int x = 0; x < kMaxConfigVersionInitRetry; x++) {
auto versionStatus = _getConfigVersion(txn);
if (!versionStatus.isOK()) {
return versionStatus.getStatus();
}
auto versionInfo = versionStatus.getValue();
if (versionInfo.getMinCompatibleVersion() > CURRENT_CONFIG_VERSION) {
return {ErrorCodes::IncompatibleShardingConfigVersion,
str::stream() << "current version v" << CURRENT_CONFIG_VERSION
<< " is older than the cluster min compatible v"
<< versionInfo.getMinCompatibleVersion()};
}
if (versionInfo.getCurrentVersion() == UpgradeHistory_EmptyVersion) {
VersionType newVersion;
newVersion.setClusterId(OID::gen());
newVersion.setMinCompatibleVersion(MIN_COMPATIBLE_CONFIG_VERSION);
newVersion.setCurrentVersion(CURRENT_CONFIG_VERSION);
BSONObj versionObj(newVersion.toBSON());
BatchedCommandResponse response;
auto upsertStatus = update(txn,
VersionType::ConfigNS,
versionObj,
versionObj,
true /* upsert*/,
false /* multi */,
&response);
if ((upsertStatus.isOK() && response.getN() < 1) ||
upsertStatus == ErrorCodes::DuplicateKey) {
// Do the check again as someone inserted a new config version document
// and the upsert neither inserted nor updated a config version document.
// Note: you can get duplicate key errors on upsert because of SERVER-14322.
continue;
}
return upsertStatus;
}
if (versionInfo.getCurrentVersion() == UpgradeHistory_UnreportedVersion) {
return {ErrorCodes::IncompatibleShardingConfigVersion,
"Assuming config data is old since the version document cannot be found in the "
"config server and it contains databases aside 'local' and 'admin'. "
"Please upgrade if this is the case. Otherwise, make sure that the config "
"server is clean."};
}
if (versionInfo.getCurrentVersion() < CURRENT_CONFIG_VERSION) {
return {ErrorCodes::IncompatibleShardingConfigVersion,
str::stream() << "need to upgrade current cluster version to v"
<< CURRENT_CONFIG_VERSION << "; currently at v"
<< versionInfo.getCurrentVersion()};
}
return Status::OK();
}
return {ErrorCodes::IncompatibleShardingConfigVersion,
str::stream() << "unable to create new config version document after "
<< kMaxConfigVersionInitRetry << " retries"};
}
/**
* Stores a newer { version, minVersion, currentVersion, clusterId } config server entry
*/
void storeConfigVersion(const VersionType& versionInfo) {
DBDirectClient client(&_txn);
client.insert(VersionType::ConfigNS, versionInfo.toBSON());
}
/**
* Stores a newer { version, minVersion, currentVersion, clusterId } config server entry
*/
void storeConfigVersion(const VersionType& versionInfo) {
client().insert(VersionType::ConfigNS, versionInfo.toBSON());
}
