// Helper to determine whether a number of targeted writes require a new targeted batch
static bool isNewBatchRequired(const vector<TargetedWrite*>& writes,
const TargetedBatchMap& batchMap) {
for (vector<TargetedWrite*>::const_iterator it = writes.begin(); it != writes.end(); ++it) {
TargetedWrite* write = *it;
if (batchMap.find(&write->endpoint) == batchMap.end()) {
return true;
}
}
return false;
}
Status BatchWriteOp::targetBatch(OperationContext* txn,
const NSTargeter& targeter,
bool recordTargetErrors,
vector<TargetedWriteBatch*>* targetedBatches) {
//
// Targeting of unordered batches is fairly simple - each remaining write op is targeted,
// and each of those targeted writes are grouped into a batch for a particular shard
// endpoint.
//
// Targeting of ordered batches is a bit more complex - to respect the ordering of the
// batch, we can only send:
// A) a single targeted batch to one shard endpoint
// B) multiple targeted batches, but only containing targeted writes for a single write op
//
// This means that any multi-shard write operation must be targeted and sent one-by-one.
// Subsequent single-shard write operations can be batched together if they go to the same
// place.
//
// Ex: ShardA : { skey : a->k }, ShardB : { skey : k->z }
//
// Ordered insert batch of: [{ skey : a }, { skey : b }, { skey : x }]
// broken into:
// [{ skey : a }, { skey : b }],
// [{ skey : x }]
//
// Ordered update Batch of :
// [{ skey : a }{ $push },
// { skey : b }{ $push },
// { skey : [c, x] }{ $push },
// { skey : y }{ $push },
// { skey : z }{ $push }]
// broken into:
// [{ skey : a }, { skey : b }],
// [{ skey : [c,x] }],
// [{ skey : y }, { skey : z }]
//
const bool ordered = _clientRequest->getOrdered();
TargetedBatchMap batchMap;
TargetedBatchSizeMap batchSizes;
int numTargetErrors = 0;
size_t numWriteOps = _clientRequest->sizeWriteOps();
for (size_t i = 0; i < numWriteOps; ++i) {
WriteOp& writeOp = _writeOps[i];
// Only target _Ready ops
if (writeOp.getWriteState() != WriteOpState_Ready)
continue;
//
// Get TargetedWrites from the targeter for the write operation
//
// TargetedWrites need to be owned once returned
OwnedPointerVector<TargetedWrite> writesOwned;
vector<TargetedWrite*>& writes = writesOwned.mutableVector();
Status targetStatus = writeOp.targetWrites(txn, targeter, &writes);
if (!targetStatus.isOK()) {
WriteErrorDetail targetError;
buildTargetError(targetStatus, &targetError);
if (!recordTargetErrors) {
// Cancel current batch state with an error
cancelBatches(targetError, _writeOps, &batchMap);
dassert(batchMap.empty());
return targetStatus;
} else if (!ordered || batchMap.empty()) {
// Record an error for this batch
writeOp.setOpError(targetError);
++numTargetErrors;
if (ordered)
return Status::OK();
continue;
} else {
dassert(ordered && !batchMap.empty());
// Send out what we have, but don't record an error yet, since there may be an
// error in the writes before this point.
writeOp.cancelWrites(&targetError);
break;
}
}
//
// If ordered and we have a previous endpoint, make sure we don't need to send these
// targeted writes to any other endpoints.
//
if (ordered && !batchMap.empty()) {
dassert(batchMap.size() == 1u);
if (isNewBatchRequired(writes, batchMap)) {
writeOp.cancelWrites(NULL);
break;
}
}
//
// If this write will push us over some sort of size limit, stop targeting
//
int writeSizeBytes = getWriteSizeBytes(writeOp);
if (wouldMakeBatchesTooBig(writes, writeSizeBytes, batchSizes)) {
invariant(!batchMap.empty());
writeOp.cancelWrites(NULL);
break;
}
//
// Targeting went ok, add to appropriate TargetedBatch
//
for (vector<TargetedWrite*>::iterator it = writes.begin(); it != writes.end(); ++it) {
TargetedWrite* write = *it;
TargetedBatchMap::iterator batchIt = batchMap.find(&write->endpoint);
TargetedBatchSizeMap::iterator batchSizeIt = batchSizes.find(&write->endpoint);
if (batchIt == batchMap.end()) {
TargetedWriteBatch* newBatch = new TargetedWriteBatch(write->endpoint);
batchIt = batchMap.insert(make_pair(&newBatch->getEndpoint(), newBatch)).first;
batchSizeIt =
batchSizes.insert(make_pair(&newBatch->getEndpoint(), BatchSize())).first;
}
TargetedWriteBatch* batch = batchIt->second;
BatchSize& batchSize = batchSizeIt->second;
++batchSize.numOps;
batchSize.sizeBytes += writeSizeBytes;
batch->addWrite(write);
}
// Relinquish ownership of TargetedWrites, now the TargetedBatches own them
writesOwned.mutableVector().clear();
//
// Break if we're ordered and we have more than one endpoint - later writes cannot be
// enforced as ordered across multiple shard endpoints.
//
if (ordered && batchMap.size() > 1u)
break;
}
//
// Send back our targeted batches
//
for (TargetedBatchMap::iterator it = batchMap.begin(); it != batchMap.end(); ++it) {
TargetedWriteBatch* batch = it->second;
if (batch->getWrites().empty())
continue;
// Remember targeted batch for reporting
_targeted.insert(batch);
// Send the handle back to caller
targetedBatches->push_back(batch);
}
return Status::OK();
}
Status BatchWriteOp::targetBatch( const NSTargeter& targeter,
bool recordTargetErrors,
vector<TargetedWriteBatch*>* targetedBatches ) {
TargetedBatchMap batchMap;
size_t numWriteOps = _clientRequest->sizeWriteOps();
for ( size_t i = 0; i < numWriteOps; ++i ) {
// Only do one-at-a-time ops if COE is false
if ( !_clientRequest->getContinueOnError() && !batchMap.empty() ) break;
WriteOp& writeOp = _writeOps[i];
// Only target _Ready ops
if ( writeOp.getWriteState() != WriteOpState_Ready ) continue;
//
// Get TargetedWrites from the targeter for the write operation
//
// TargetedWrites need to be owned once returned
OwnedPointerVector<TargetedWrite> writesOwned;
vector<TargetedWrite*>& writes = writesOwned.mutableVector();
Status targetStatus = writeOp.targetWrites( targeter, &writes );
if ( !targetStatus.isOK() ) {
//
// We're not sure how to target here, so either record the error or cancel the
// current batches.
//
BatchedErrorDetail targetError;
buildTargetError( targetStatus, &targetError );
if ( recordTargetErrors ) {
writeOp.setOpError( targetError );
continue;
}
else {
// Cancel current batch state with an error
cancelBatches( targetError, _writeOps, &batchMap );
dassert( batchMap.empty() );
return targetStatus;
}
}
//
// Targeting went ok, add to appropriate TargetedBatch
//
for ( vector<TargetedWrite*>::iterator it = writes.begin(); it != writes.end(); ++it ) {
TargetedWrite* write = *it;
TargetedBatchMap::iterator seenIt = batchMap.find( &write->endpoint );
if ( seenIt == batchMap.end() ) {
TargetedWriteBatch* newBatch = new TargetedWriteBatch( write->endpoint );
seenIt = batchMap.insert( make_pair( &newBatch->getEndpoint(), //
newBatch ) ).first;
}
TargetedWriteBatch* batch = seenIt->second;
batch->addWrite( write );
}
// Relinquish ownership of TargetedWrites, now the TargetedBatches own them
writesOwned.mutableVector().clear();
}
//
// Send back our targeted batches
//
for ( TargetedBatchMap::iterator it = batchMap.begin(); it != batchMap.end(); ++it ) {
TargetedWriteBatch* batch = it->second;
// Remember targeted batch for reporting
_targeted.insert( batch );
// Send the handle back to caller
targetedBatches->push_back( batch );
}
return Status::OK();
}
