static void insertOne(WriteBatchExecutor::ExecInsertsState* state, WriteOpResult* result) {
invariant(state->currIndex < state->normalizedInserts.size());
const StatusWith<BSONObj>& normalizedInsert(state->normalizedInserts[state->currIndex]);
if (!normalizedInsert.isOK()) {
result->setError(toWriteError(normalizedInsert.getStatus()));
return;
}
const BSONObj& insertDoc = normalizedInsert.getValue().isEmpty() ?
state->request->getInsertRequest()->getDocumentsAt( state->currIndex ) :
normalizedInsert.getValue();
try {
if (state->lockAndCheck(result)) {
if (!state->request->isInsertIndexRequest()) {
singleInsert(state->txn, insertDoc, state->getCollection(), result);
}
else {
singleCreateIndex(state->txn, insertDoc, state->getCollection(), result);
}
}
}
catch (const DBException& ex) {
Status status(ex.toStatus());
if (ErrorCodes::isInterruption(status.code()))
throw;
result->setError(toWriteError(status));
}
// Errors release the write lock, as a matter of policy.
if (result->getError())
state->unlock();
}
/**
* Perform a single index insert into a collection. Requires the index descriptor be
* preprocessed and the collection already has been created.
*
* Might fault or error, otherwise populates the result.
*/
static void singleCreateIndex( OperationContext* txn,
const BSONObj& indexDesc,
Collection* collection,
WriteOpResult* result ) {
const string indexNS = collection->ns().getSystemIndexesCollection();
txn->lockState()->assertWriteLocked( indexNS );
MultiIndexBlock indexer(txn, collection);
indexer.allowBackgroundBuilding();
indexer.allowInterruption();
Status status = indexer.init(indexDesc);
if ( status.code() == ErrorCodes::IndexAlreadyExists ) {
result->getStats().n = 0;
return; // inserting an existing index is a no-op.
}
if (!status.isOK()) {
result->setError(toWriteError(status));
return;
}
status = indexer.insertAllDocumentsInCollection();
if (!status.isOK()) {
result->setError(toWriteError(status));
return;
}
WriteUnitOfWork wunit(txn);
indexer.commit();
repl::logOp( txn, "i", indexNS.c_str(), indexDesc );
result->getStats().n = 1;
wunit.commit();
}
/**
* Perform a single insert into a collection. Requires the insert be preprocessed and the
* collection already has been created.
*
* Might fault or error, otherwise populates the result.
*/
static void singleInsert( const BatchItemRef& insertItem,
const BSONObj& normalInsert,
Collection* collection,
WriteOpResult* result ) {
const string& insertNS = insertItem.getRequest()->getNS();
Lock::assertWriteLocked( insertNS );
try {
// XXX - are we 100% sure that all !OK statuses do not write a document?
StatusWith<DiskLoc> status = collection->insertDocument( normalInsert, true );
if ( !status.isOK() ) {
result->error = toWriteError( status.getStatus() );
}
else {
logOp( "i", insertNS.c_str(), normalInsert );
getDur().commitIfNeeded();
result->stats.n = 1;
}
}
catch ( const PageFaultException& ex ) {
// TODO: An actual data structure that's not an exception for this
result->fault = new PageFaultException( ex );
}
catch ( const DBException& ex ) {
result->error = toWriteError( ex.toStatus() );
}
}
/**
* Perform a single index insert into a collection. Requires the index descriptor be
* preprocessed and the collection already has been created.
*
* Might fault or error, otherwise populates the result.
*/
static void singleCreateIndex( const BatchItemRef& insertItem,
const BSONObj& normalIndexDesc,
Collection* collection,
WriteOpResult* result ) {
const string& indexNS = insertItem.getRequest()->getNS();
Lock::assertWriteLocked( indexNS );
try {
Status status = collection->getIndexCatalog()->createIndex( normalIndexDesc, true );
if ( status.code() == ErrorCodes::IndexAlreadyExists ) {
result->stats.n = 0;
}
else if ( !status.isOK() ) {
result->error = toWriteError( status );
}
else {
logOp( "i", indexNS.c_str(), normalIndexDesc );
result->stats.n = 1;
}
}
catch ( const PageFaultException& ex ) {
// TODO: An actual data structure that's not an exception for this
result->fault = new PageFaultException( ex );
}
catch ( const DBException& ex ) {
result->error = toWriteError( ex.toStatus() );
}
}
static void multiUpdate( OperationContext* txn,
const BatchItemRef& updateItem,
WriteOpResult* result ) {
const NamespaceString nsString(updateItem.getRequest()->getNS());
UpdateRequest request(nsString);
request.setQuery(updateItem.getUpdate()->getQuery());
request.setUpdates(updateItem.getUpdate()->getUpdateExpr());
request.setMulti(updateItem.getUpdate()->getMulti());
request.setUpsert(updateItem.getUpdate()->getUpsert());
request.setUpdateOpLog(true);
UpdateLifecycleImpl updateLifecycle(true, request.getNamespaceString());
request.setLifecycle(&updateLifecycle);
UpdateExecutor executor(&request, &txn->getCurOp()->debug());
Status status = executor.prepare();
if (!status.isOK()) {
result->setError(toWriteError(status));
return;
}
///////////////////////////////////////////
Lock::DBWrite writeLock(txn->lockState(), nsString.ns());
///////////////////////////////////////////
if (!checkShardVersion(txn, &shardingState, *updateItem.getRequest(), result))
return;
Client::Context ctx( nsString.ns(),
storageGlobalParams.dbpath,
false /* don't check version */ );
try {
UpdateResult res = executor.execute(txn, ctx.db());
const long long numDocsModified = res.numDocsModified;
const long long numMatched = res.numMatched;
const BSONObj resUpsertedID = res.upserted;
// We have an _id from an insert
const bool didInsert = !resUpsertedID.isEmpty();
result->getStats().nModified = didInsert ? 0 : numDocsModified;
result->getStats().n = didInsert ? 1 : numMatched;
result->getStats().upsertedID = resUpsertedID;
}
catch (const DBException& ex) {
status = ex.toStatus();
if (ErrorCodes::isInterruption(status.code())) {
throw;
}
result->setError(toWriteError(status));
}
}
static void insertOne(WriteBatchExecutor::ExecInsertsState* state, WriteOpResult* result) {
invariant(state->currIndex < state->normalizedInserts.size());
const StatusWith<BSONObj>& normalizedInsert(state->normalizedInserts[state->currIndex]);
if (!normalizedInsert.isOK()) {
result->setError(toWriteError(normalizedInsert.getStatus()));
return;
}
const BSONObj& insertDoc = normalizedInsert.getValue().isEmpty() ?
state->request->getInsertRequest()->getDocumentsAt( state->currIndex ) :
normalizedInsert.getValue();
cc().clearHasWrittenThisOperation();
{
PageFaultRetryableSection pageFaultSection;
while (true) {
try {
if (!state->lockAndCheck(result)) {
break;
}
if (!state->request->isInsertIndexRequest()) {
const PregeneratedKeys* pregen = NULL;
if ( state->pregeneratedKeys.size() > state->currIndex )
pregen = &state->pregeneratedKeys[state->currIndex];
singleInsert(insertDoc, state->getCollection(), pregen, result);
}
else {
singleCreateIndex(insertDoc, state->getCollection(), result);
}
break;
}
catch (const DBException& ex) {
Status status(ex.toStatus());
if (ErrorCodes::isInterruption(status.code()))
throw;
result->setError(toWriteError(status));
break;
}
catch (PageFaultException& pfe) {
state->unlock();
pfe.touch();
continue; // Try the operation again.
}
fassertFailed(17430);
}
} // end PageFaultRetryableSection
// Errors release the write lock, as a matter of policy.
if (result->getError())
state->unlock();
}
/**
* Perform a remove operation, which might remove multiple documents. Dispatches to remove code
* currently to do most of this.
*
* Might fault or error, otherwise populates the result.
*/
static void multiRemove( const BatchItemRef& removeItem,
WriteOpResult* result ) {
const NamespaceString nss( removeItem.getRequest()->getNS() );
DeleteRequest request( nss );
request.setQuery( removeItem.getDelete()->getQuery() );
request.setMulti( removeItem.getDelete()->getLimit() != 1 );
request.setUpdateOpLog(true);
request.setGod( false );
DeleteExecutor executor( &request );
Status status = executor.prepare();
if ( !status.isOK() ) {
result->error = toWriteError( status );
return;
}
// NOTE: Deletes will not fault outside the lock once any data has been written
PageFaultRetryableSection pFaultSection;
///////////////////////////////////////////
Lock::DBWrite writeLock( nss.ns() );
///////////////////////////////////////////
// Check version once we're locked
if ( !checkShardVersion( &shardingState, *removeItem.getRequest(), &result->error ) ) {
// Version error
return;
}
// Context once we're locked, to set more details in currentOp()
// TODO: better constructor?
Client::Context writeContext( nss.ns(),
storageGlobalParams.dbpath,
false /* don't check version */);
try {
result->stats.n = executor.execute();
}
catch ( const PageFaultException& ex ) {
// TODO: An actual data structure that's not an exception for this
result->fault = new PageFaultException( ex );
}
catch ( const DBException& ex ) {
status = ex.toStatus();
if (ErrorCodes::isInterruption(status.code())) {
throw;
}
result->error = toWriteError(status);
}
}
bool WriteBatchExecutor::ExecInsertsState::_lockAndCheckImpl(WriteOpResult* result) {
if (hasLock()) {
txn->getCurOp()->enter(_context.get());
return true;
}
invariant(!_context.get());
_writeLock.reset(new Lock::DBWrite(txn->lockState(), request->getNS()));
if (!checkIsMasterForDatabase(request->getNS(), result)) {
return false;
}
if (!checkShardVersion(txn, &shardingState, *request, result)) {
return false;
}
if (!checkIndexConstraints(txn, &shardingState, *request, result)) {
return false;
}
_context.reset(new Client::Context(request->getNS(),
false /* don't check version */));
Database* database = _context->db();
dassert(database);
_collection = database->getCollection(txn, request->getTargetingNS());
if (!_collection) {
// Implicitly create if it doesn't exist
_collection = database->createCollection(txn, request->getTargetingNS());
if (!_collection) {
result->setError(
toWriteError(Status(ErrorCodes::InternalError,
"could not create collection " +
request->getTargetingNS())));
return false;
}
}
return true;
}
/**
* Perform a remove operation, which might remove multiple documents. Dispatches to remove code
* currently to do most of this.
*
* Might fault or error, otherwise populates the result.
*/
static void multiRemove( OperationContext* txn,
const BatchItemRef& removeItem,
WriteOpResult* result ) {
const NamespaceString nss( removeItem.getRequest()->getNS() );
DeleteRequest request( nss );
request.setQuery( removeItem.getDelete()->getQuery() );
request.setMulti( removeItem.getDelete()->getLimit() != 1 );
request.setUpdateOpLog(true);
request.setGod( false );
DeleteExecutor executor( &request );
Status status = executor.prepare();
if ( !status.isOK() ) {
result->setError(toWriteError(status));
return;
}
///////////////////////////////////////////
Lock::DBWrite writeLock(txn->lockState(), nss.ns());
///////////////////////////////////////////
// Check version once we're locked
if (!checkShardVersion(txn, &shardingState, *removeItem.getRequest(), result)) {
// Version error
return;
}
// Context once we're locked, to set more details in currentOp()
// TODO: better constructor?
Client::Context writeContext( nss.ns(),
storageGlobalParams.dbpath,
false /* don't check version */);
try {
result->getStats().n = executor.execute(txn, writeContext.db());
}
catch ( const DBException& ex ) {
status = ex.toStatus();
if (ErrorCodes::isInterruption(status.code())) {
throw;
}
result->setError(toWriteError(status));
}
}
/**
* Perform an update operation, which might update multiple documents in the lock. Dispatches
* to update code currently to do most of this.
*
* Might error, otherwise populates the result.
*/
static void multiUpdate( const BatchItemRef& updateItem,
WriteOpResult* result ) {
Lock::assertWriteLocked( updateItem.getRequest()->getNS() );
BSONObj queryObj = updateItem.getUpdate()->getQuery();
BSONObj updateObj = updateItem.getUpdate()->getUpdateExpr();
bool multi = updateItem.getUpdate()->getMulti();
bool upsert = updateItem.getUpdate()->getUpsert();
bool didInsert = false;
long long numMatched = 0;
long long numDocsModified = 0;
BSONObj resUpsertedID;
try {
const NamespaceString requestNs( updateItem.getRequest()->getNS() );
UpdateRequest request( requestNs );
request.setQuery( queryObj );
request.setUpdates( updateObj );
request.setUpsert( upsert );
request.setMulti( multi );
request.setUpdateOpLog();
// TODO(greg) We need to send if we are ignoring the shard version below,
// but for now yes
UpdateLifecycleImpl updateLifecycle( true, requestNs );
request.setLifecycle( &updateLifecycle );
UpdateResult res = update( request, &cc().curop()->debug() );
numDocsModified = res.numDocsModified;
numMatched = res.numMatched;
resUpsertedID = res.upserted;
// We have an _id from an insert
didInsert = !resUpsertedID.isEmpty();
result->stats.nModified = didInsert ? 0 : numDocsModified;
result->stats.n = didInsert ? 1 : numMatched;
result->stats.upsertedID = resUpsertedID;
}
catch ( const DBException& ex ) {
result->error = toWriteError( ex.toStatus() );
}
}
bool WriteBatchExecutor::ExecInsertsState::_lockAndCheckImpl(WriteOpResult* result) {
if (hasLock()) {
txn->getCurOp()->enter(_context.get());
return true;
}
invariant(!_context.get());
_writeLock.reset(new Lock::DBLock(txn->lockState(),
nsToDatabase(request->getNS()),
newlm::MODE_X));
if (!checkIsMasterForDatabase(request->getNS(), result)) {
return false;
}
if (!checkShardVersion(txn, &shardingState, *request, result)) {
return false;
}
if (!checkIndexConstraints(txn, &shardingState, *request, result)) {
return false;
}
_context.reset(new Client::Context(txn, request->getNS(), false));
Database* database = _context->db();
dassert(database);
_collection = database->getCollection(txn, request->getTargetingNS());
if (!_collection) {
WriteUnitOfWork wunit (txn);
// Implicitly create if it doesn't exist
_collection = database->createCollection(txn, request->getTargetingNS());
if (!_collection) {
result->setError(
toWriteError(Status(ErrorCodes::InternalError,
"could not create collection " +
request->getTargetingNS())));
return false;
}
repl::logOp(txn,
"c",
(database->name() + ".$cmd").c_str(),
BSON("create" << nsToCollectionSubstring(request->getTargetingNS())));
wunit.commit();
}
return true;
}
/**
* Perform a single insert into a collection. Requires the insert be preprocessed and the
* collection already has been created.
*
* Might fault or error, otherwise populates the result.
*/
static void singleInsert( const BSONObj& docToInsert,
Collection* collection,
WriteOpResult* result ) {
const string& insertNS = collection->ns().ns();
Lock::assertWriteLocked( insertNS );
StatusWith<DiskLoc> status = collection->insertDocument( docToInsert, true );
if ( !status.isOK() ) {
result->setError(toWriteError(status.getStatus()));
}
else {
logOp( "i", insertNS.c_str(), docToInsert );
getDur().commitIfNeeded();
result->getStats().n = 1;
}
}
/**
* Perform a single insert into a collection. Requires the insert be preprocessed and the
* collection already has been created.
*
* Might fault or error, otherwise populates the result.
*/
static void singleInsert( OperationContext* txn,
const BSONObj& docToInsert,
Collection* collection,
WriteOpResult* result ) {
const string& insertNS = collection->ns().ns();
txn->lockState()->assertWriteLocked( insertNS );
StatusWith<DiskLoc> status = collection->insertDocument( txn, docToInsert, true );
if ( !status.isOK() ) {
result->setError(toWriteError(status.getStatus()));
}
else {
repl::logOp( txn, "i", insertNS.c_str(), docToInsert );
txn->recoveryUnit()->commitIfNeeded();
result->getStats().n = 1;
}
}
/**
* Perform a single index insert into a collection. Requires the index descriptor be
* preprocessed and the collection already has been created.
*
* Might fault or error, otherwise populates the result.
*/
static void singleCreateIndex( const BSONObj& indexDesc,
Collection* collection,
WriteOpResult* result ) {
const string indexNS = collection->ns().getSystemIndexesCollection();
Lock::assertWriteLocked( indexNS );
Status status = collection->getIndexCatalog()->createIndex( indexDesc, true );
if ( status.code() == ErrorCodes::IndexAlreadyExists ) {
result->getStats().n = 0;
}
else if ( !status.isOK() ) {
result->setError(toWriteError(status));
}
else {
logOp( "i", indexNS.c_str(), indexDesc );
result->getStats().n = 1;
}
}
/**
* Perform a remove operation, which might remove multiple documents. Dispatches to remove code
* currently to do most of this.
*
* Might fault or error, otherwise populates the result.
*/
static void multiRemove( const BatchItemRef& removeItem,
WriteOpResult* result ) {
Lock::assertWriteLocked( removeItem.getRequest()->getNS() );
try {
long long n = deleteObjects( removeItem.getRequest()->getNS(),
removeItem.getDelete()->getQuery(),
removeItem.getDelete()->getLimit() == 1, // justOne
true, // logOp
false // god
);
result->stats.n = n;
}
catch ( const PageFaultException& ex ) {
// TODO: An actual data structure that's not an exception for this
result->fault = new PageFaultException( ex );
}
catch ( const DBException& ex ) {
result->error = toWriteError( ex.toStatus() );
}
}
void WriteBatchExecutor::execInserts( const BatchedCommandRequest& request,
std::vector<WriteErrorDetail*>* errors ) {
// Bulk insert is a bit different from other bulk operations in that multiple request docs
// can be processed at once inside the write lock.
const NamespaceString nss( request.getTargetingNS() );
scoped_ptr<BatchItemRef> currInsertItem( new BatchItemRef( &request, 0 ) );
// Go through our request and do some preprocessing on insert documents outside the lock to
// validate and put them in a normalized form - i.e. put _id in front and fill in
// timestamps. The insert document may also be invalid.
// TODO: Might be more efficient to do in batches.
vector<StatusWith<BSONObj> > normalInserts;
normalizeInserts( request, &normalInserts );
while ( currInsertItem->getItemIndex() < static_cast<int>( request.sizeWriteOps() ) ) {
WriteOpResult currResult;
// Don't (re-)acquire locks and create database until it's necessary
if ( !normalInserts[currInsertItem->getItemIndex()].isOK() ) {
currResult.error =
toWriteError( normalInserts[currInsertItem->getItemIndex()].getStatus() );
}
else {
PageFaultRetryableSection pFaultSection;
////////////////////////////////////
Lock::DBWrite writeLock( nss.ns() );
////////////////////////////////////
// Check version inside of write lock
if ( checkIsMasterForCollection( nss, &currResult.error )
&& checkShardVersion( &shardingState, request, &currResult.error )
&& checkIndexConstraints( &shardingState, request, &currResult.error ) ) {
//
// Get the collection for the insert
//
scoped_ptr<Client::Context> writeContext;
Collection* collection = NULL;
try {
// Context once we're locked, to set more details in currentOp()
// TODO: better constructor?
writeContext.reset( new Client::Context( request.getNS(),
storageGlobalParams.dbpath,
false /* don't check version */) );
Database* database = writeContext->db();
dassert( database );
collection = database->getCollection( nss.ns() );
if ( !collection ) {
// Implicitly create if it doesn't exist
collection = database->createCollection( nss.ns() );
if ( !collection ) {
currResult.error =
toWriteError( Status( ErrorCodes::InternalError,
"could not create collection" ) );
}
}
}
catch ( const DBException& ex ) {
Status status(ex.toStatus());
if (ErrorCodes::isInterruption(status.code())) {
throw;
}
currResult.error = toWriteError(status);
}
//
// Perform writes inside write lock
//
while ( collection
&& currInsertItem->getItemIndex()
< static_cast<int>( request.sizeWriteOps() ) ) {
//
// BEGIN CURRENT OP
//
scoped_ptr<CurOp> currentOp( beginCurrentOp( _client, *currInsertItem ) );
incOpStats( *currInsertItem );
// Get the actual document we want to write, assuming it's valid
const StatusWith<BSONObj>& normalInsert = //
normalInserts[currInsertItem->getItemIndex()];
const BSONObj& normalInsertDoc =
normalInsert.getValue().isEmpty() ?
currInsertItem->getDocument() : normalInsert.getValue();
if ( !normalInsert.isOK() ) {
// This insert failed on preprocessing
currResult.error = toWriteError( normalInsert.getStatus() );
}
else if ( !request.isInsertIndexRequest() ) {
// Try the insert
singleInsert( *currInsertItem,
normalInsertDoc,
collection,
&currResult );
}
else {
// Try the create index
singleCreateIndex( *currInsertItem,
normalInsertDoc,
collection,
&currResult );
}
//
// END CURRENT OP
//
finishCurrentOp( _client, currentOp.get(), currResult.error );
// Faults release the write lock
if ( currResult.fault )
break;
// In general, we might have stats and errors
incWriteStats( *currInsertItem,
currResult.stats,
currResult.error,
currentOp.get() );
// Errors release the write lock
if ( currResult.error )
break;
// Increment in the write lock and reset the stats for next time
currInsertItem.reset( new BatchItemRef( &request,
currInsertItem->getItemIndex()
+ 1 ) );
currResult.reset();
// Destruct curop so that our parent curop is restored, so that we
// record the yield count in the parent.
currentOp.reset(NULL);
// yield sometimes
int micros = ClientCursor::suggestYieldMicros();
if (micros > 0) {
ClientCursor::staticYield(micros, "", NULL);
}
}
}
} // END WRITE LOCK
//
// Store the current error if it exists
//
if ( currResult.error ) {
errors->push_back( currResult.releaseError() );
errors->back()->setIndex( currInsertItem->getItemIndex() );
// Break early for ordered batches
if ( request.getOrdered() )
break;
}
//
// Fault or increment
//
if ( currResult.fault ) {
// Check page fault out of lock
currResult.fault->touch();
}
else {
// Increment if not a fault
currInsertItem.reset( new BatchItemRef( &request,
currInsertItem->getItemIndex() + 1 ) );
}
}
}
