void RepositoryXL::clear() {
objectMap_.clear();
errorMessageMap_.clear();
callingRanges_.clear();
}
void clear()
{
_ranges.clear();
}
void run() {
int numShards = 10;
int numInitialChunks = 5;
int maxChunks = 100000; // Needed to not overflow the BSONArray's max bytes
int keySize = 2;
BSONArrayBuilder chunksB;
BSONObj lastSplitPt;
ShardChunkVersion version( 1, 0, OID() );
//
// Generate numChunks with a given key size over numShards
// All chunks have double key values, so we can split them a bunch
//
for( int i = -1; i < numInitialChunks; i++ ){
BSONObjBuilder splitPtB;
for( int k = 0; k < keySize; k++ ){
string field = string( "k" ) + string( 1, (char)('0' + k) );
if( i < 0 )
splitPtB.appendMinKey( field );
else if( i < numInitialChunks - 1 )
splitPtB.append( field, (double)i );
else
splitPtB.appendMaxKey( field );
}
BSONObj splitPt = splitPtB.obj();
if( i >= 0 ){
BSONObjBuilder chunkB;
chunkB.append( "min", lastSplitPt );
chunkB.append( "max", splitPt );
int shardNum = rand( numShards );
chunkB.append( "shard", "shard" + string( 1, (char)('A' + shardNum) ) );
rand( 2 ) ? version.incMajor() : version.incMinor();
version.addToBSON( chunkB, "lastmod" );
chunksB.append( chunkB.obj() );
}
lastSplitPt = splitPt;
}
BSONArray chunks = chunksB.arr();
// log() << "Chunks generated : " << chunks << endl;
DBClientMockCursor chunksCursor( chunks );
// Setup the empty ranges and versions first
RangeMap ranges;
ShardChunkVersion maxVersion = ShardChunkVersion( 0, 0, OID() );
VersionMap maxShardVersions;
// Create a differ which will track our progress
boost::shared_ptr< DefaultDiffAdapter > differ( _inverse ? new InverseDiffAdapter() : new DefaultDiffAdapter() );
differ->attach( "test", ranges, maxVersion, maxShardVersions );
// Validate initial load
differ->calculateConfigDiff( chunksCursor );
validate( chunks, ranges, maxVersion, maxShardVersions );
// Generate a lot of diffs, and keep validating that updating from the diffs always
// gives us the right ranges and versions
int numDiffs = 135; // Makes about 100000 chunks overall
int numChunks = numInitialChunks;
for( int i = 0; i < numDiffs; i++ ){
// log() << "Generating new diff... " << i << endl;
BSONArrayBuilder diffsB;
BSONArrayBuilder newChunksB;
BSONObjIterator chunksIt( chunks );
while( chunksIt.more() ){
BSONObj chunk = chunksIt.next().Obj();
int randChoice = rand( 10 );
if( randChoice < 2 && numChunks < maxChunks ){
// Simulate a split
// log() << " ...starting a split with chunk " << chunk << endl;
BSONObjBuilder leftB;
BSONObjBuilder rightB;
BSONObjBuilder midB;
for( int k = 0; k < keySize; k++ ){
string field = string( "k" ) + string( 1, (char)('0' + k) );
BSONType maxType = chunk["max"].Obj()[field].type();
double max = maxType == NumberDouble ? chunk["max"].Obj()[field].Number() : 0.0;
BSONType minType = chunk["min"].Obj()[field].type();
double min = minType == NumberDouble ? chunk["min"].Obj()[field].Number() : 0.0;
if( minType == MinKey ){
midB.append( field, max - 1.0 );
}
else if( maxType == MaxKey ){
midB.append( field, min + 1.0 );
}
else {
midB.append( field, ( max + min ) / 2.0 );
}
}
BSONObj midPt = midB.obj();
// Only happens if we can't split the min chunk
if( midPt.isEmpty() ) continue;
leftB.append( chunk["min"] );
leftB.append( "max", midPt );
rightB.append( "min", midPt );
rightB.append( chunk["max"] );
leftB.append( chunk["shard"] );
rightB.append( chunk["shard"] );
version.incMajor();
version._minor = 0;
version.addToBSON( leftB, "lastmod" );
version.incMinor();
version.addToBSON( rightB, "lastmod" );
BSONObj left = leftB.obj();
BSONObj right = rightB.obj();
// log() << " ... split into " << left << " and " << right << endl;
newChunksB.append( left );
newChunksB.append( right );
diffsB.append( right );
diffsB.append( left );
numChunks++;
}
else if( randChoice < 4 && chunksIt.more() ){
// Simulate a migrate
// log() << " ...starting a migrate with chunk " << chunk << endl;
BSONObj prevShardChunk;
while( chunksIt.more() ){
prevShardChunk = chunksIt.next().Obj();
if( prevShardChunk["shard"].String() == chunk["shard"].String() ) break;
// log() << "... appending chunk from diff shard: " << prevShardChunk << endl;
newChunksB.append( prevShardChunk );
prevShardChunk = BSONObj();
}
// We need to move between different shards, hence the weirdness in logic here
if( ! prevShardChunk.isEmpty() ){
BSONObjBuilder newShardB;
BSONObjBuilder prevShardB;
newShardB.append( chunk["min"] );
newShardB.append( chunk["max"] );
prevShardB.append( prevShardChunk["min"] );
prevShardB.append( prevShardChunk["max"] );
int shardNum = rand( numShards );
newShardB.append( "shard", "shard" + string( 1, (char)('A' + shardNum) ) );
prevShardB.append( prevShardChunk["shard"] );
version.incMajor();
version._minor = 0;
version.addToBSON( newShardB, "lastmod" );
version.incMinor();
version.addToBSON( prevShardB, "lastmod" );
BSONObj newShard = newShardB.obj();
BSONObj prevShard = prevShardB.obj();
// log() << " ... migrated to " << newShard << " and updated " << prevShard << endl;
newChunksB.append( newShard );
newChunksB.append( prevShard );
diffsB.append( newShard );
diffsB.append( prevShard );
}
else{
// log() << "... appending chunk, no more left: " << chunk << endl;
newChunksB.append( chunk );
}
}
else{
// log() << "Appending chunk : " << chunk << endl;
newChunksB.append( chunk );
}
}
BSONArray diffs = diffsB.arr();
chunks = newChunksB.arr();
// log() << "Diffs generated : " << diffs << endl;
// log() << "All chunks : " << chunks << endl;
// Rarely entirely clear out our data
if( rand( 10 ) < 1 ){
diffs = chunks;
ranges.clear();
maxVersion = ShardChunkVersion( 0, 0, OID() );
maxShardVersions.clear();
}
// log() << "Total number of chunks : " << numChunks << " iteration " << i << endl;
DBClientMockCursor diffCursor( diffs );
differ->calculateConfigDiff( diffCursor );
validate( chunks, ranges, maxVersion, maxShardVersions );
}
}
