IndexCatalogEntry* IndexCatalogEntryContainer::find( const string& name ) {
for ( iterator i = begin(); i != end(); ++i ) {
IndexCatalogEntry* e = *i;
if ( e->descriptor()->indexName() == name )
return e;
}
return NULL;
}
IndexCatalogEntry* IndexCatalogEntryContainer::find( const IndexDescriptor* desc ) {
if ( desc->_cachedEntry )
return desc->_cachedEntry;
for ( iterator i = begin(); i != end(); ++i ) {
IndexCatalogEntry* e = *i;
if ( e->descriptor() == desc )
return e;
}
return NULL;
}
IndexCatalogEntry* IndexCatalogEntryContainer::release( const IndexDescriptor* desc ) {
for ( std::vector<IndexCatalogEntry*>::iterator i = _entries.mutableVector().begin();
i != _entries.mutableVector().end();
++i ) {
IndexCatalogEntry* e = *i;
if ( e->descriptor() != desc )
continue;
_entries.mutableVector().erase( i );
return e;
}
return NULL;
}
StatusWith<RecordId> Collection::updateDocument(OperationContext* txn,
const RecordId& oldLocation,
const Snapshotted<BSONObj>& oldDoc,
const BSONObj& newDoc,
bool enforceQuota,
bool indexesAffected,
OpDebug* debug,
oplogUpdateEntryArgs& args) {
{
auto status = checkValidation(txn, newDoc);
if (!status.isOK()) {
if (_validationLevel == STRICT_V) {
return status;
}
// moderate means we have to check the old doc
auto oldDocStatus = checkValidation(txn, oldDoc.value());
if (oldDocStatus.isOK()) {
// transitioning from good -> bad is not ok
return status;
}
// bad -> bad is ok in moderate mode
}
}
dassert(txn->lockState()->isCollectionLockedForMode(ns().toString(), MODE_IX));
invariant(oldDoc.snapshotId() == txn->recoveryUnit()->getSnapshotId());
if (_needCappedLock) {
// X-lock the metadata resource for this capped collection until the end of the WUOW. This
// prevents the primary from executing with more concurrency than secondaries.
// See SERVER-21646.
Lock::ResourceLock{txn->lockState(), ResourceId(RESOURCE_METADATA, _ns.ns()), MODE_X};
}
SnapshotId sid = txn->recoveryUnit()->getSnapshotId();
BSONElement oldId = oldDoc.value()["_id"];
if (!oldId.eoo() && (oldId != newDoc["_id"]))
return StatusWith<RecordId>(
ErrorCodes::InternalError, "in Collection::updateDocument _id mismatch", 13596);
// The MMAPv1 storage engine implements capped collections in a way that does not allow records
// to grow beyond their original size. If MMAPv1 part of a replicaset with storage engines that
// do not have this limitation, replication could result in errors, so it is necessary to set a
// uniform rule here. Similarly, it is not sufficient to disallow growing records, because this
// happens when secondaries roll back an update shrunk a record. Exactly replicating legacy
// MMAPv1 behavior would require padding shrunk documents on all storage engines. Instead forbid
// all size changes.
const auto oldSize = oldDoc.value().objsize();
if (_recordStore->isCapped() && oldSize != newDoc.objsize())
return {ErrorCodes::CannotGrowDocumentInCappedNamespace,
str::stream() << "Cannot change the size of a document in a capped collection: "
<< oldSize << " != " << newDoc.objsize()};
// At the end of this step, we will have a map of UpdateTickets, one per index, which
// represent the index updates needed to be done, based on the changes between oldDoc and
// newDoc.
OwnedPointerMap<IndexDescriptor*, UpdateTicket> updateTickets;
if (indexesAffected) {
IndexCatalog::IndexIterator ii = _indexCatalog.getIndexIterator(txn, true);
while (ii.more()) {
IndexDescriptor* descriptor = ii.next();
IndexCatalogEntry* entry = ii.catalogEntry(descriptor);
IndexAccessMethod* iam = ii.accessMethod(descriptor);
InsertDeleteOptions options;
options.logIfError = false;
options.dupsAllowed =
!(KeyPattern::isIdKeyPattern(descriptor->keyPattern()) || descriptor->unique()) ||
repl::getGlobalReplicationCoordinator()->shouldIgnoreUniqueIndex(descriptor);
UpdateTicket* updateTicket = new UpdateTicket();
updateTickets.mutableMap()[descriptor] = updateTicket;
Status ret = iam->validateUpdate(txn,
oldDoc.value(),
newDoc,
oldLocation,
options,
updateTicket,
entry->getFilterExpression());
if (!ret.isOK()) {
return StatusWith<RecordId>(ret);
}
}
}
// This can call back into Collection::recordStoreGoingToMove. If that happens, the old
// object is removed from all indexes.
StatusWith<RecordId> newLocation = _recordStore->updateRecord(
txn, oldLocation, newDoc.objdata(), newDoc.objsize(), _enforceQuota(enforceQuota), this);
if (!newLocation.isOK()) {
return newLocation;
}
// At this point, the old object may or may not still be indexed, depending on if it was
// moved. If the object did move, we need to add the new location to all indexes.
if (newLocation.getValue() != oldLocation) {
if (debug) {
if (debug->nmoved == -1) // default of -1 rather than 0
debug->nmoved = 1;
else
debug->nmoved += 1;
}
std::vector<BsonRecord> bsonRecords;
BsonRecord bsonRecord = {newLocation.getValue(), &newDoc};
bsonRecords.push_back(bsonRecord);
Status s = _indexCatalog.indexRecords(txn, bsonRecords);
if (!s.isOK())
return StatusWith<RecordId>(s);
invariant(sid == txn->recoveryUnit()->getSnapshotId());
args.ns = ns().ns();
getGlobalServiceContext()->getOpObserver()->onUpdate(txn, args);
return newLocation;
}
// Object did not move. We update each index with each respective UpdateTicket.
if (debug)
debug->keyUpdates = 0;
if (indexesAffected) {
IndexCatalog::IndexIterator ii = _indexCatalog.getIndexIterator(txn, true);
while (ii.more()) {
IndexDescriptor* descriptor = ii.next();
IndexAccessMethod* iam = ii.accessMethod(descriptor);
int64_t updatedKeys;
Status ret = iam->update(txn, *updateTickets.mutableMap()[descriptor], &updatedKeys);
if (!ret.isOK())
return StatusWith<RecordId>(ret);
if (debug)
debug->keyUpdates += updatedKeys;
}
}
invariant(sid == txn->recoveryUnit()->getSnapshotId());
args.ns = ns().ns();
getGlobalServiceContext()->getOpObserver()->onUpdate(txn, args);
return newLocation;
}
StatusWith<RecordId> Collection::updateDocument(OperationContext* txn,
const RecordId& oldLocation,
const Snapshotted<BSONObj>& oldDoc,
const BSONObj& newDoc,
bool enforceQuota,
bool indexesAffected,
OpDebug* debug,
oplogUpdateEntryArgs& args) {
{
auto status = checkValidation(txn, newDoc);
if (!status.isOK()) {
if (_validationLevel == STRICT_V) {
return status;
}
// moderate means we have to check the old doc
auto oldDocStatus = checkValidation(txn, oldDoc.value());
if (oldDocStatus.isOK()) {
// transitioning from good -> bad is not ok
return status;
}
// bad -> bad is ok in moderate mode
}
}
dassert(txn->lockState()->isCollectionLockedForMode(ns().toString(), MODE_IX));
invariant(oldDoc.snapshotId() == txn->recoveryUnit()->getSnapshotId());
SnapshotId sid = txn->recoveryUnit()->getSnapshotId();
BSONElement oldId = oldDoc.value()["_id"];
if (!oldId.eoo() && (oldId != newDoc["_id"]))
return StatusWith<RecordId>(
ErrorCodes::InternalError, "in Collection::updateDocument _id mismatch", 13596);
// At the end of this step, we will have a map of UpdateTickets, one per index, which
// represent the index updates needed to be done, based on the changes between oldDoc and
// newDoc.
OwnedPointerMap<IndexDescriptor*, UpdateTicket> updateTickets;
if (indexesAffected) {
IndexCatalog::IndexIterator ii = _indexCatalog.getIndexIterator(txn, true);
while (ii.more()) {
IndexDescriptor* descriptor = ii.next();
IndexCatalogEntry* entry = ii.catalogEntry(descriptor);
IndexAccessMethod* iam = ii.accessMethod(descriptor);
InsertDeleteOptions options;
options.logIfError = false;
options.dupsAllowed =
!(KeyPattern::isIdKeyPattern(descriptor->keyPattern()) || descriptor->unique()) ||
repl::getGlobalReplicationCoordinator()->shouldIgnoreUniqueIndex(descriptor);
UpdateTicket* updateTicket = new UpdateTicket();
updateTickets.mutableMap()[descriptor] = updateTicket;
Status ret = iam->validateUpdate(txn,
oldDoc.value(),
newDoc,
oldLocation,
options,
updateTicket,
entry->getFilterExpression());
if (!ret.isOK()) {
return StatusWith<RecordId>(ret);
}
}
}
// This can call back into Collection::recordStoreGoingToMove. If that happens, the old
// object is removed from all indexes.
StatusWith<RecordId> newLocation = _recordStore->updateRecord(
txn, oldLocation, newDoc.objdata(), newDoc.objsize(), _enforceQuota(enforceQuota), this);
if (!newLocation.isOK()) {
return newLocation;
}
// At this point, the old object may or may not still be indexed, depending on if it was
// moved. If the object did move, we need to add the new location to all indexes.
if (newLocation.getValue() != oldLocation) {
if (debug) {
if (debug->nmoved == -1) // default of -1 rather than 0
debug->nmoved = 1;
else
debug->nmoved += 1;
}
Status s = _indexCatalog.indexRecord(txn, newDoc, newLocation.getValue());
if (!s.isOK())
return StatusWith<RecordId>(s);
invariant(sid == txn->recoveryUnit()->getSnapshotId());
args.ns = ns().ns();
getGlobalServiceContext()->getOpObserver()->onUpdate(txn, args);
return newLocation;
}
// Object did not move. We update each index with each respective UpdateTicket.
if (debug)
debug->keyUpdates = 0;
if (indexesAffected) {
IndexCatalog::IndexIterator ii = _indexCatalog.getIndexIterator(txn, true);
while (ii.more()) {
IndexDescriptor* descriptor = ii.next();
IndexAccessMethod* iam = ii.accessMethod(descriptor);
int64_t updatedKeys;
Status ret = iam->update(txn, *updateTickets.mutableMap()[descriptor], &updatedKeys);
if (!ret.isOK())
return StatusWith<RecordId>(ret);
if (debug)
debug->keyUpdates += updatedKeys;
}
}
invariant(sid == txn->recoveryUnit()->getSnapshotId());
args.ns = ns().ns();
getGlobalServiceContext()->getOpObserver()->onUpdate(txn, args);
return newLocation;
}
Status BtreeBasedBulkAccessMethod::commit(set<DiskLoc>* dupsToDrop,
CurOp* op,
bool mayInterrupt) {
DiskLoc oldHead = _real->_btreeState->head();
// XXX: do we expect the tree to be empty but have a head set? Looks like so from old code.
invariant(!oldHead.isNull());
_real->_btreeState->setHead(_txn, DiskLoc());
_real->_btreeState->recordStore()->deleteRecord(_txn, oldHead);
if (_isMultiKey) {
_real->_btreeState->setMultikey( _txn );
}
_sorter->sort(false);
Timer timer;
IndexCatalogEntry* entry = _real->_btreeState;
bool dupsAllowed = !entry->descriptor()->unique()
|| ignoreUniqueIndex(entry->descriptor());
bool dropDups = entry->descriptor()->dropDups() || inDBRepair;
scoped_ptr<BSONObjExternalSorter::Iterator> i(_sorter->iterator());
// verifies that pm and op refer to the same ProgressMeter
ProgressMeter& pm = op->setMessage("Index Bulk Build: (2/3) btree bottom up",
"Index: (2/3) BTree Bottom Up Progress",
_keysInserted,
10);
scoped_ptr<BtreeBuilderInterface> builder;
builder.reset(_interface->getBulkBuilder(_txn, dupsAllowed));
while (i->more()) {
// Get the next datum and add it to the builder.
ExternalSortDatum d = i->next();
Status status = builder->addKey(d.first, d.second);
if (!status.isOK()) {
if (ErrorCodes::DuplicateKey != status.code()) {
return status;
}
// If we're here it's a duplicate key.
if (dropDups) {
static const size_t kMaxDupsToStore = 1000000;
dupsToDrop->insert(d.second);
if (dupsToDrop->size() > kMaxDupsToStore) {
return Status(ErrorCodes::InternalError,
"Too many dups on index build with dropDups = true");
}
}
else if (!dupsAllowed) {
return status;
}
}
// If we're here either it's a dup and we're cool with it or the addKey went just
// fine.
pm.hit();
}
pm.finished();
op->setMessage("Index Bulk Build: (3/3) btree-middle",
"Index: (3/3) BTree Middle Progress");
LOG(timer.seconds() > 10 ? 0 : 1 ) << "\t done building bottom layer, going to commit";
unsigned long long keysCommit = builder->commit(mayInterrupt);
if (!dropDups && (keysCommit != _keysInserted)) {
warning() << "not all entries were added to the index, probably some "
<< "keys were too large" << endl;
}
return Status::OK();
}
StatusWith<CompactStats> compactCollection(OperationContext* opCtx,
Collection* collection,
const CompactOptions* compactOptions) {
dassert(opCtx->lockState()->isCollectionLockedForMode(collection->ns().toString(), MODE_X));
DisableDocumentValidation validationDisabler(opCtx);
auto recordStore = collection->getRecordStore();
auto indexCatalog = collection->getIndexCatalog();
if (!recordStore->compactSupported())
return StatusWith<CompactStats>(ErrorCodes::CommandNotSupported,
str::stream()
<< "cannot compact collection with record store: "
<< recordStore->name());
if (recordStore->compactsInPlace()) {
CompactStats stats;
Status status = recordStore->compact(opCtx);
if (!status.isOK())
return StatusWith<CompactStats>(status);
// Compact all indexes (not including unfinished indexes)
std::unique_ptr<IndexCatalog::IndexIterator> ii(
indexCatalog->getIndexIterator(opCtx, false));
while (ii->more()) {
IndexCatalogEntry* entry = ii->next();
IndexDescriptor* descriptor = entry->descriptor();
IndexAccessMethod* iam = entry->accessMethod();
LOG(1) << "compacting index: " << descriptor->toString();
Status status = iam->compact(opCtx);
if (!status.isOK()) {
error() << "failed to compact index: " << descriptor->toString();
return status;
}
}
return StatusWith<CompactStats>(stats);
}
if (indexCatalog->numIndexesInProgress(opCtx))
return StatusWith<CompactStats>(ErrorCodes::BadValue,
"cannot compact when indexes in progress");
std::vector<BSONObj> indexSpecs;
{
std::unique_ptr<IndexCatalog::IndexIterator> ii(
indexCatalog->getIndexIterator(opCtx, false));
while (ii->more()) {
IndexDescriptor* descriptor = ii->next()->descriptor();
// Compact always creates the new index in the foreground.
const BSONObj spec =
descriptor->infoObj().removeField(IndexDescriptor::kBackgroundFieldName);
const BSONObj key = spec.getObjectField("key");
const Status keyStatus =
index_key_validate::validateKeyPattern(key, descriptor->version());
if (!keyStatus.isOK()) {
return StatusWith<CompactStats>(
ErrorCodes::CannotCreateIndex,
str::stream() << "Cannot compact collection due to invalid index " << spec
<< ": "
<< keyStatus.reason()
<< " For more info see"
<< " http://dochub.mongodb.org/core/index-validation");
}
indexSpecs.push_back(spec);
}
}
// Give a chance to be interrupted *before* we drop all indexes.
opCtx->checkForInterrupt();
{
// note that the drop indexes call also invalidates all clientcursors for the namespace,
// which is important and wanted here
WriteUnitOfWork wunit(opCtx);
log() << "compact dropping indexes";
indexCatalog->dropAllIndexes(opCtx, true);
wunit.commit();
}
CompactStats stats;
MultiIndexBlockImpl indexer(opCtx, collection);
indexer.allowInterruption();
indexer.ignoreUniqueConstraint(); // in compact we should be doing no checking
Status status = indexer.init(indexSpecs).getStatus();
if (!status.isOK())
return StatusWith<CompactStats>(status);
status = recordStore->compact(opCtx);
if (!status.isOK())
return StatusWith<CompactStats>(status);
log() << "starting index commits";
status = indexer.dumpInsertsFromBulk();
if (!status.isOK())
return StatusWith<CompactStats>(status);
{
WriteUnitOfWork wunit(opCtx);
status = indexer.commit();
if (!status.isOK()) {
return StatusWith<CompactStats>(status);
}
wunit.commit();
}
return StatusWith<CompactStats>(stats);
}
void commit( set<DiskLoc>* dupsToDrop,
CurOp* op,
bool mayInterrupt ) {
Timer timer;
IndexCatalogEntry* entry = _real->_btreeState;
bool dupsAllowed = !entry->descriptor()->unique() ||
ignoreUniqueIndex(entry->descriptor());
bool dropDups = entry->descriptor()->dropDups() || inDBRepair;
BtreeBuilder<V> btBuilder(dupsAllowed, entry);
BSONObj keyLast;
scoped_ptr<BSONObjExternalSorter::Iterator> i( _phase1.sorter->iterator() );
// verifies that pm and op refer to the same ProgressMeter
ProgressMeter& pm = op->setMessage("Index Bulk Build: (2/3) btree bottom up",
"Index: (2/3) BTree Bottom Up Progress",
_phase1.nkeys,
10);
while( i->more() ) {
RARELY if ( mayInterrupt ) killCurrentOp.checkForInterrupt();
ExternalSortDatum d = i->next();
try {
if ( !dupsAllowed && dropDups ) {
LastError::Disabled led( lastError.get() );
btBuilder.addKey(d.first, d.second);
}
else {
btBuilder.addKey(d.first, d.second);
}
}
catch( AssertionException& e ) {
if ( dupsAllowed ) {
// unknown exception??
throw;
}
if (ErrorCodes::isInterruption(
DBException::convertExceptionCode(e.getCode()))) {
killCurrentOp.checkForInterrupt();
}
if ( ! dropDups )
throw;
/* we could queue these on disk, but normally there are very few dups,
* so instead we keep in ram and have a limit.
*/
if ( dupsToDrop ) {
dupsToDrop->insert(d.second);
uassert( 10092,
"too may dups on index build with dropDups=true",
dupsToDrop->size() < 1000000 );
}
}
pm.hit();
}
pm.finished();
op->setMessage("Index Bulk Build: (3/3) btree-middle",
"Index: (3/3) BTree Middle Progress");
LOG(timer.seconds() > 10 ? 0 : 1 ) << "\t done building bottom layer, going to commit";
btBuilder.commit( mayInterrupt );
if ( btBuilder.getn() != _phase1.nkeys && ! dropDups ) {
warning() << "not all entries were added to the index, probably some "
<< "keys were too large" << endl;
}
}
