static void insertOne(WriteBatchExecutor::ExecInsertsState* state, WriteOpResult* result) { invariant(state->currIndex < state->normalizedInserts.size()); const StatusWith<BSONObj>& normalizedInsert(state->normalizedInserts[state->currIndex]); if (!normalizedInsert.isOK()) { result->setError(toWriteError(normalizedInsert.getStatus())); return; } const BSONObj& insertDoc = normalizedInsert.getValue().isEmpty() ? state->request->getInsertRequest()->getDocumentsAt( state->currIndex ) : normalizedInsert.getValue(); try { if (state->lockAndCheck(result)) { if (!state->request->isInsertIndexRequest()) { singleInsert(state->txn, insertDoc, state->getCollection(), result); } else { singleCreateIndex(state->txn, insertDoc, state->getCollection(), result); } } } catch (const DBException& ex) { Status status(ex.toStatus()); if (ErrorCodes::isInterruption(status.code())) throw; result->setError(toWriteError(status)); } // Errors release the write lock, as a matter of policy. if (result->getError()) state->unlock(); }
/** * Perform a single index insert into a collection. Requires the index descriptor be * preprocessed and the collection already has been created. * * Might fault or error, otherwise populates the result. */ static void singleCreateIndex( OperationContext* txn, const BSONObj& indexDesc, Collection* collection, WriteOpResult* result ) { const string indexNS = collection->ns().getSystemIndexesCollection(); txn->lockState()->assertWriteLocked( indexNS ); MultiIndexBlock indexer(txn, collection); indexer.allowBackgroundBuilding(); indexer.allowInterruption(); Status status = indexer.init(indexDesc); if ( status.code() == ErrorCodes::IndexAlreadyExists ) { result->getStats().n = 0; return; // inserting an existing index is a no-op. } if (!status.isOK()) { result->setError(toWriteError(status)); return; } status = indexer.insertAllDocumentsInCollection(); if (!status.isOK()) { result->setError(toWriteError(status)); return; } WriteUnitOfWork wunit(txn); indexer.commit(); repl::logOp( txn, "i", indexNS.c_str(), indexDesc ); result->getStats().n = 1; wunit.commit(); }
/** * Perform a single insert into a collection. Requires the insert be preprocessed and the * collection already has been created. * * Might fault or error, otherwise populates the result. */ static void singleInsert( const BatchItemRef& insertItem, const BSONObj& normalInsert, Collection* collection, WriteOpResult* result ) { const string& insertNS = insertItem.getRequest()->getNS(); Lock::assertWriteLocked( insertNS ); try { // XXX - are we 100% sure that all !OK statuses do not write a document? StatusWith<DiskLoc> status = collection->insertDocument( normalInsert, true ); if ( !status.isOK() ) { result->error = toWriteError( status.getStatus() ); } else { logOp( "i", insertNS.c_str(), normalInsert ); getDur().commitIfNeeded(); result->stats.n = 1; } } catch ( const PageFaultException& ex ) { // TODO: An actual data structure that's not an exception for this result->fault = new PageFaultException( ex ); } catch ( const DBException& ex ) { result->error = toWriteError( ex.toStatus() ); } }
/** * Perform a single index insert into a collection. Requires the index descriptor be * preprocessed and the collection already has been created. * * Might fault or error, otherwise populates the result. */ static void singleCreateIndex( const BatchItemRef& insertItem, const BSONObj& normalIndexDesc, Collection* collection, WriteOpResult* result ) { const string& indexNS = insertItem.getRequest()->getNS(); Lock::assertWriteLocked( indexNS ); try { Status status = collection->getIndexCatalog()->createIndex( normalIndexDesc, true ); if ( status.code() == ErrorCodes::IndexAlreadyExists ) { result->stats.n = 0; } else if ( !status.isOK() ) { result->error = toWriteError( status ); } else { logOp( "i", indexNS.c_str(), normalIndexDesc ); result->stats.n = 1; } } catch ( const PageFaultException& ex ) { // TODO: An actual data structure that's not an exception for this result->fault = new PageFaultException( ex ); } catch ( const DBException& ex ) { result->error = toWriteError( ex.toStatus() ); } }
static void multiUpdate( OperationContext* txn, const BatchItemRef& updateItem, WriteOpResult* result ) { const NamespaceString nsString(updateItem.getRequest()->getNS()); UpdateRequest request(nsString); request.setQuery(updateItem.getUpdate()->getQuery()); request.setUpdates(updateItem.getUpdate()->getUpdateExpr()); request.setMulti(updateItem.getUpdate()->getMulti()); request.setUpsert(updateItem.getUpdate()->getUpsert()); request.setUpdateOpLog(true); UpdateLifecycleImpl updateLifecycle(true, request.getNamespaceString()); request.setLifecycle(&updateLifecycle); UpdateExecutor executor(&request, &txn->getCurOp()->debug()); Status status = executor.prepare(); if (!status.isOK()) { result->setError(toWriteError(status)); return; } /////////////////////////////////////////// Lock::DBWrite writeLock(txn->lockState(), nsString.ns()); /////////////////////////////////////////// if (!checkShardVersion(txn, &shardingState, *updateItem.getRequest(), result)) return; Client::Context ctx( nsString.ns(), storageGlobalParams.dbpath, false /* don't check version */ ); try { UpdateResult res = executor.execute(txn, ctx.db()); const long long numDocsModified = res.numDocsModified; const long long numMatched = res.numMatched; const BSONObj resUpsertedID = res.upserted; // We have an _id from an insert const bool didInsert = !resUpsertedID.isEmpty(); result->getStats().nModified = didInsert ? 0 : numDocsModified; result->getStats().n = didInsert ? 1 : numMatched; result->getStats().upsertedID = resUpsertedID; } catch (const DBException& ex) { status = ex.toStatus(); if (ErrorCodes::isInterruption(status.code())) { throw; } result->setError(toWriteError(status)); } }
static void insertOne(WriteBatchExecutor::ExecInsertsState* state, WriteOpResult* result) { invariant(state->currIndex < state->normalizedInserts.size()); const StatusWith<BSONObj>& normalizedInsert(state->normalizedInserts[state->currIndex]); if (!normalizedInsert.isOK()) { result->setError(toWriteError(normalizedInsert.getStatus())); return; } const BSONObj& insertDoc = normalizedInsert.getValue().isEmpty() ? state->request->getInsertRequest()->getDocumentsAt( state->currIndex ) : normalizedInsert.getValue(); cc().clearHasWrittenThisOperation(); { PageFaultRetryableSection pageFaultSection; while (true) { try { if (!state->lockAndCheck(result)) { break; } if (!state->request->isInsertIndexRequest()) { const PregeneratedKeys* pregen = NULL; if ( state->pregeneratedKeys.size() > state->currIndex ) pregen = &state->pregeneratedKeys[state->currIndex]; singleInsert(insertDoc, state->getCollection(), pregen, result); } else { singleCreateIndex(insertDoc, state->getCollection(), result); } break; } catch (const DBException& ex) { Status status(ex.toStatus()); if (ErrorCodes::isInterruption(status.code())) throw; result->setError(toWriteError(status)); break; } catch (PageFaultException& pfe) { state->unlock(); pfe.touch(); continue; // Try the operation again. } fassertFailed(17430); } } // end PageFaultRetryableSection // Errors release the write lock, as a matter of policy. if (result->getError()) state->unlock(); }
/** * Perform a remove operation, which might remove multiple documents. Dispatches to remove code * currently to do most of this. * * Might fault or error, otherwise populates the result. */ static void multiRemove( const BatchItemRef& removeItem, WriteOpResult* result ) { const NamespaceString nss( removeItem.getRequest()->getNS() ); DeleteRequest request( nss ); request.setQuery( removeItem.getDelete()->getQuery() ); request.setMulti( removeItem.getDelete()->getLimit() != 1 ); request.setUpdateOpLog(true); request.setGod( false ); DeleteExecutor executor( &request ); Status status = executor.prepare(); if ( !status.isOK() ) { result->error = toWriteError( status ); return; } // NOTE: Deletes will not fault outside the lock once any data has been written PageFaultRetryableSection pFaultSection; /////////////////////////////////////////// Lock::DBWrite writeLock( nss.ns() ); /////////////////////////////////////////// // Check version once we're locked if ( !checkShardVersion( &shardingState, *removeItem.getRequest(), &result->error ) ) { // Version error return; } // Context once we're locked, to set more details in currentOp() // TODO: better constructor? Client::Context writeContext( nss.ns(), storageGlobalParams.dbpath, false /* don't check version */); try { result->stats.n = executor.execute(); } catch ( const PageFaultException& ex ) { // TODO: An actual data structure that's not an exception for this result->fault = new PageFaultException( ex ); } catch ( const DBException& ex ) { status = ex.toStatus(); if (ErrorCodes::isInterruption(status.code())) { throw; } result->error = toWriteError(status); } }
bool WriteBatchExecutor::ExecInsertsState::_lockAndCheckImpl(WriteOpResult* result) { if (hasLock()) { txn->getCurOp()->enter(_context.get()); return true; } invariant(!_context.get()); _writeLock.reset(new Lock::DBWrite(txn->lockState(), request->getNS())); if (!checkIsMasterForDatabase(request->getNS(), result)) { return false; } if (!checkShardVersion(txn, &shardingState, *request, result)) { return false; } if (!checkIndexConstraints(txn, &shardingState, *request, result)) { return false; } _context.reset(new Client::Context(request->getNS(), false /* don't check version */)); Database* database = _context->db(); dassert(database); _collection = database->getCollection(txn, request->getTargetingNS()); if (!_collection) { // Implicitly create if it doesn't exist _collection = database->createCollection(txn, request->getTargetingNS()); if (!_collection) { result->setError( toWriteError(Status(ErrorCodes::InternalError, "could not create collection " + request->getTargetingNS()))); return false; } } return true; }
/** * Perform a remove operation, which might remove multiple documents. Dispatches to remove code * currently to do most of this. * * Might fault or error, otherwise populates the result. */ static void multiRemove( OperationContext* txn, const BatchItemRef& removeItem, WriteOpResult* result ) { const NamespaceString nss( removeItem.getRequest()->getNS() ); DeleteRequest request( nss ); request.setQuery( removeItem.getDelete()->getQuery() ); request.setMulti( removeItem.getDelete()->getLimit() != 1 ); request.setUpdateOpLog(true); request.setGod( false ); DeleteExecutor executor( &request ); Status status = executor.prepare(); if ( !status.isOK() ) { result->setError(toWriteError(status)); return; } /////////////////////////////////////////// Lock::DBWrite writeLock(txn->lockState(), nss.ns()); /////////////////////////////////////////// // Check version once we're locked if (!checkShardVersion(txn, &shardingState, *removeItem.getRequest(), result)) { // Version error return; } // Context once we're locked, to set more details in currentOp() // TODO: better constructor? Client::Context writeContext( nss.ns(), storageGlobalParams.dbpath, false /* don't check version */); try { result->getStats().n = executor.execute(txn, writeContext.db()); } catch ( const DBException& ex ) { status = ex.toStatus(); if (ErrorCodes::isInterruption(status.code())) { throw; } result->setError(toWriteError(status)); } }
/** * Perform an update operation, which might update multiple documents in the lock. Dispatches * to update code currently to do most of this. * * Might error, otherwise populates the result. */ static void multiUpdate( const BatchItemRef& updateItem, WriteOpResult* result ) { Lock::assertWriteLocked( updateItem.getRequest()->getNS() ); BSONObj queryObj = updateItem.getUpdate()->getQuery(); BSONObj updateObj = updateItem.getUpdate()->getUpdateExpr(); bool multi = updateItem.getUpdate()->getMulti(); bool upsert = updateItem.getUpdate()->getUpsert(); bool didInsert = false; long long numMatched = 0; long long numDocsModified = 0; BSONObj resUpsertedID; try { const NamespaceString requestNs( updateItem.getRequest()->getNS() ); UpdateRequest request( requestNs ); request.setQuery( queryObj ); request.setUpdates( updateObj ); request.setUpsert( upsert ); request.setMulti( multi ); request.setUpdateOpLog(); // TODO(greg) We need to send if we are ignoring the shard version below, // but for now yes UpdateLifecycleImpl updateLifecycle( true, requestNs ); request.setLifecycle( &updateLifecycle ); UpdateResult res = update( request, &cc().curop()->debug() ); numDocsModified = res.numDocsModified; numMatched = res.numMatched; resUpsertedID = res.upserted; // We have an _id from an insert didInsert = !resUpsertedID.isEmpty(); result->stats.nModified = didInsert ? 0 : numDocsModified; result->stats.n = didInsert ? 1 : numMatched; result->stats.upsertedID = resUpsertedID; } catch ( const DBException& ex ) { result->error = toWriteError( ex.toStatus() ); } }
bool WriteBatchExecutor::ExecInsertsState::_lockAndCheckImpl(WriteOpResult* result) { if (hasLock()) { txn->getCurOp()->enter(_context.get()); return true; } invariant(!_context.get()); _writeLock.reset(new Lock::DBLock(txn->lockState(), nsToDatabase(request->getNS()), newlm::MODE_X)); if (!checkIsMasterForDatabase(request->getNS(), result)) { return false; } if (!checkShardVersion(txn, &shardingState, *request, result)) { return false; } if (!checkIndexConstraints(txn, &shardingState, *request, result)) { return false; } _context.reset(new Client::Context(txn, request->getNS(), false)); Database* database = _context->db(); dassert(database); _collection = database->getCollection(txn, request->getTargetingNS()); if (!_collection) { WriteUnitOfWork wunit (txn); // Implicitly create if it doesn't exist _collection = database->createCollection(txn, request->getTargetingNS()); if (!_collection) { result->setError( toWriteError(Status(ErrorCodes::InternalError, "could not create collection " + request->getTargetingNS()))); return false; } repl::logOp(txn, "c", (database->name() + ".$cmd").c_str(), BSON("create" << nsToCollectionSubstring(request->getTargetingNS()))); wunit.commit(); } return true; }
/** * Perform a single insert into a collection. Requires the insert be preprocessed and the * collection already has been created. * * Might fault or error, otherwise populates the result. */ static void singleInsert( const BSONObj& docToInsert, Collection* collection, WriteOpResult* result ) { const string& insertNS = collection->ns().ns(); Lock::assertWriteLocked( insertNS ); StatusWith<DiskLoc> status = collection->insertDocument( docToInsert, true ); if ( !status.isOK() ) { result->setError(toWriteError(status.getStatus())); } else { logOp( "i", insertNS.c_str(), docToInsert ); getDur().commitIfNeeded(); result->getStats().n = 1; } }
/** * Perform a single insert into a collection. Requires the insert be preprocessed and the * collection already has been created. * * Might fault or error, otherwise populates the result. */ static void singleInsert( OperationContext* txn, const BSONObj& docToInsert, Collection* collection, WriteOpResult* result ) { const string& insertNS = collection->ns().ns(); txn->lockState()->assertWriteLocked( insertNS ); StatusWith<DiskLoc> status = collection->insertDocument( txn, docToInsert, true ); if ( !status.isOK() ) { result->setError(toWriteError(status.getStatus())); } else { repl::logOp( txn, "i", insertNS.c_str(), docToInsert ); txn->recoveryUnit()->commitIfNeeded(); result->getStats().n = 1; } }
/** * Perform a single index insert into a collection. Requires the index descriptor be * preprocessed and the collection already has been created. * * Might fault or error, otherwise populates the result. */ static void singleCreateIndex( const BSONObj& indexDesc, Collection* collection, WriteOpResult* result ) { const string indexNS = collection->ns().getSystemIndexesCollection(); Lock::assertWriteLocked( indexNS ); Status status = collection->getIndexCatalog()->createIndex( indexDesc, true ); if ( status.code() == ErrorCodes::IndexAlreadyExists ) { result->getStats().n = 0; } else if ( !status.isOK() ) { result->setError(toWriteError(status)); } else { logOp( "i", indexNS.c_str(), indexDesc ); result->getStats().n = 1; } }
/** * Perform a remove operation, which might remove multiple documents. Dispatches to remove code * currently to do most of this. * * Might fault or error, otherwise populates the result. */ static void multiRemove( const BatchItemRef& removeItem, WriteOpResult* result ) { Lock::assertWriteLocked( removeItem.getRequest()->getNS() ); try { long long n = deleteObjects( removeItem.getRequest()->getNS(), removeItem.getDelete()->getQuery(), removeItem.getDelete()->getLimit() == 1, // justOne true, // logOp false // god ); result->stats.n = n; } catch ( const PageFaultException& ex ) { // TODO: An actual data structure that's not an exception for this result->fault = new PageFaultException( ex ); } catch ( const DBException& ex ) { result->error = toWriteError( ex.toStatus() ); } }
void WriteBatchExecutor::execInserts( const BatchedCommandRequest& request, std::vector<WriteErrorDetail*>* errors ) { // Bulk insert is a bit different from other bulk operations in that multiple request docs // can be processed at once inside the write lock. const NamespaceString nss( request.getTargetingNS() ); scoped_ptr<BatchItemRef> currInsertItem( new BatchItemRef( &request, 0 ) ); // Go through our request and do some preprocessing on insert documents outside the lock to // validate and put them in a normalized form - i.e. put _id in front and fill in // timestamps. The insert document may also be invalid. // TODO: Might be more efficient to do in batches. vector<StatusWith<BSONObj> > normalInserts; normalizeInserts( request, &normalInserts ); while ( currInsertItem->getItemIndex() < static_cast<int>( request.sizeWriteOps() ) ) { WriteOpResult currResult; // Don't (re-)acquire locks and create database until it's necessary if ( !normalInserts[currInsertItem->getItemIndex()].isOK() ) { currResult.error = toWriteError( normalInserts[currInsertItem->getItemIndex()].getStatus() ); } else { PageFaultRetryableSection pFaultSection; //////////////////////////////////// Lock::DBWrite writeLock( nss.ns() ); //////////////////////////////////// // Check version inside of write lock if ( checkIsMasterForCollection( nss, &currResult.error ) && checkShardVersion( &shardingState, request, &currResult.error ) && checkIndexConstraints( &shardingState, request, &currResult.error ) ) { // // Get the collection for the insert // scoped_ptr<Client::Context> writeContext; Collection* collection = NULL; try { // Context once we're locked, to set more details in currentOp() // TODO: better constructor? writeContext.reset( new Client::Context( request.getNS(), storageGlobalParams.dbpath, false /* don't check version */) ); Database* database = writeContext->db(); dassert( database ); collection = database->getCollection( nss.ns() ); if ( !collection ) { // Implicitly create if it doesn't exist collection = database->createCollection( nss.ns() ); if ( !collection ) { currResult.error = toWriteError( Status( ErrorCodes::InternalError, "could not create collection" ) ); } } } catch ( const DBException& ex ) { Status status(ex.toStatus()); if (ErrorCodes::isInterruption(status.code())) { throw; } currResult.error = toWriteError(status); } // // Perform writes inside write lock // while ( collection && currInsertItem->getItemIndex() < static_cast<int>( request.sizeWriteOps() ) ) { // // BEGIN CURRENT OP // scoped_ptr<CurOp> currentOp( beginCurrentOp( _client, *currInsertItem ) ); incOpStats( *currInsertItem ); // Get the actual document we want to write, assuming it's valid const StatusWith<BSONObj>& normalInsert = // normalInserts[currInsertItem->getItemIndex()]; const BSONObj& normalInsertDoc = normalInsert.getValue().isEmpty() ? currInsertItem->getDocument() : normalInsert.getValue(); if ( !normalInsert.isOK() ) { // This insert failed on preprocessing currResult.error = toWriteError( normalInsert.getStatus() ); } else if ( !request.isInsertIndexRequest() ) { // Try the insert singleInsert( *currInsertItem, normalInsertDoc, collection, &currResult ); } else { // Try the create index singleCreateIndex( *currInsertItem, normalInsertDoc, collection, &currResult ); } // // END CURRENT OP // finishCurrentOp( _client, currentOp.get(), currResult.error ); // Faults release the write lock if ( currResult.fault ) break; // In general, we might have stats and errors incWriteStats( *currInsertItem, currResult.stats, currResult.error, currentOp.get() ); // Errors release the write lock if ( currResult.error ) break; // Increment in the write lock and reset the stats for next time currInsertItem.reset( new BatchItemRef( &request, currInsertItem->getItemIndex() + 1 ) ); currResult.reset(); // Destruct curop so that our parent curop is restored, so that we // record the yield count in the parent. currentOp.reset(NULL); // yield sometimes int micros = ClientCursor::suggestYieldMicros(); if (micros > 0) { ClientCursor::staticYield(micros, "", NULL); } } } } // END WRITE LOCK // // Store the current error if it exists // if ( currResult.error ) { errors->push_back( currResult.releaseError() ); errors->back()->setIndex( currInsertItem->getItemIndex() ); // Break early for ordered batches if ( request.getOrdered() ) break; } // // Fault or increment // if ( currResult.fault ) { // Check page fault out of lock currResult.fault->touch(); } else { // Increment if not a fault currInsertItem.reset( new BatchItemRef( &request, currInsertItem->getItemIndex() + 1 ) ); } } }