// Validates that the ranges and versions are valid given the chunks
void validate( const BSONArray& chunks, RangeMap* ranges, ShardChunkVersion maxVersion, const VersionMap& maxShardVersions ){
BSONObjIterator it( chunks );
int chunkCount = 0;
ShardChunkVersion foundMaxVersion;
VersionMap foundMaxShardVersions;
//
// Validate that all the chunks are there and collect versions
//
while( it.more() ){
BSONObj chunkDoc = it.next().Obj();
chunkCount++;
if( ranges != NULL ){
// log() << "Validating chunk " << chunkDoc << " size : " << ranges->size() << " vs " << chunkCount << endl;
RangeMap::iterator chunkRange = ranges->find( _inverse ? chunkDoc["max"].Obj() : chunkDoc["min"].Obj() );
ASSERT( chunkRange != ranges->end() );
ASSERT( chunkRange->second.woCompare( _inverse ? chunkDoc["min"].Obj() : chunkDoc["max"].Obj() ) == 0 );
}
ShardChunkVersion version = ShardChunkVersion::fromBSON( chunkDoc["lastmod"] );
if( version > foundMaxVersion ) foundMaxVersion = version;
ShardChunkVersion shardMaxVersion = foundMaxShardVersions[ chunkDoc["shard"].String() ];
if( version > shardMaxVersion ) foundMaxShardVersions[ chunkDoc["shard"].String() ] = version;
}
// Make sure all chunks are accounted for
if( ranges != NULL ) ASSERT( chunkCount == (int) ranges->size() );
// log() << "Validating that all shard versions are up to date..." << endl;
// Validate that all the versions are the same
ASSERT( foundMaxVersion.isEquivalentTo( maxVersion ) );
for( VersionMap::iterator it = foundMaxShardVersions.begin(); it != foundMaxShardVersions.end(); it++ ){
ShardChunkVersion foundVersion = it->second;
VersionMap::const_iterator maxIt = maxShardVersions.find( it->first );
ASSERT( maxIt != maxShardVersions.end() );
ASSERT( foundVersion.isEquivalentTo( maxIt->second ) );
}
// Make sure all shards are accounted for
ASSERT( foundMaxShardVersions.size() == maxShardVersions.size() );
}
/**
* Create the simple API
* @param nrhs :: The number of parameters in the Matlab function call
* @param prhs :: The data from the Matlab function call
* @returns An integer indicating success/failure
*/
int CreateSimpleAPI(int, mxArray **, int nrhs, const mxArray *prhs[]) {
// Ensure all libraries are loaded
FrameworkManager::Instance();
// Create directory to store mfiles
std::string mpath("");
if (nrhs == 0) {
mpath = "MantidSimpleAPI";
} else if (nrhs == 1) {
char buffer[256];
mxGetString(prhs[0], buffer, sizeof(buffer));
mpath = std::string(buffer) + "/MantidSimpleAPI";
} else {
mexErrMsgTxt("SimpleAPI_Create takes either 0 or 1 arguments.");
}
Poco::File simpleAPI(mpath);
if (simpleAPI.exists()) {
simpleAPI.remove(true);
}
try {
simpleAPI.createDirectory();
} catch (std::exception &) {
mexErrMsgTxt(
"An error occurred while creating the directory for the simple API.");
}
std::vector<std::string> algKeys = AlgorithmFactory::Instance().getKeys();
std::vector<std::string>::const_iterator sIter = algKeys.begin();
typedef std::map<std::string, unsigned int> VersionMap;
VersionMap vMap;
for (; sIter != algKeys.end(); ++sIter) {
std::string key = (*sIter);
std::string name = key.substr(0, key.find("|"));
VersionMap::iterator vIter = vMap.find(name);
if (vIter == vMap.end())
vMap.emplace(name, 1);
else
++(vIter->second);
}
std::string contents_path = simpleAPI.path() + "/Contents.m";
std::ofstream contents(contents_path.c_str());
contents << "%A simpler API for Mantid\n%\n%The algorithms available are:\n";
VersionMap::const_iterator vIter = vMap.begin();
for (; vIter != vMap.end(); ++vIter) {
contents << "% " << vIter->first << "\n";
CreateSimpleAPIHelper(vIter->first, mpath + std::string("/"));
}
contents
<< "% For help with an individual command type \"help algorithm_name\"\n";
contents.close();
return 0;
}
void VersionInfo::AddVersion(VersionMap& versions, const SceneVersion& version)
{
DVASSERT(versions.find(version.version) == versions.end());
versions.insert(VersionMap::value_type(version.version, version));
}
void run() {
int numShards = 10;
int numInitialChunks = 5;
int maxChunks = 100000; // Needed to not overflow the BSONArray's max bytes
int keySize = 2;
BSONArrayBuilder chunksB;
BSONObj lastSplitPt;
ShardChunkVersion version( 1, 0, OID() );
//
// Generate numChunks with a given key size over numShards
// All chunks have double key values, so we can split them a bunch
//
for( int i = -1; i < numInitialChunks; i++ ){
BSONObjBuilder splitPtB;
for( int k = 0; k < keySize; k++ ){
string field = string( "k" ) + string( 1, (char)('0' + k) );
if( i < 0 )
splitPtB.appendMinKey( field );
else if( i < numInitialChunks - 1 )
splitPtB.append( field, (double)i );
else
splitPtB.appendMaxKey( field );
}
BSONObj splitPt = splitPtB.obj();
if( i >= 0 ){
BSONObjBuilder chunkB;
chunkB.append( "min", lastSplitPt );
chunkB.append( "max", splitPt );
int shardNum = rand( numShards );
chunkB.append( "shard", "shard" + string( 1, (char)('A' + shardNum) ) );
rand( 2 ) ? version.incMajor() : version.incMinor();
version.addToBSON( chunkB, "lastmod" );
chunksB.append( chunkB.obj() );
}
lastSplitPt = splitPt;
}
BSONArray chunks = chunksB.arr();
// log() << "Chunks generated : " << chunks << endl;
DBClientMockCursor chunksCursor( chunks );
// Setup the empty ranges and versions first
RangeMap ranges;
ShardChunkVersion maxVersion = ShardChunkVersion( 0, 0, OID() );
VersionMap maxShardVersions;
// Create a differ which will track our progress
boost::shared_ptr< DefaultDiffAdapter > differ( _inverse ? new InverseDiffAdapter() : new DefaultDiffAdapter() );
differ->attach( "test", ranges, maxVersion, maxShardVersions );
// Validate initial load
differ->calculateConfigDiff( chunksCursor );
validate( chunks, ranges, maxVersion, maxShardVersions );
// Generate a lot of diffs, and keep validating that updating from the diffs always
// gives us the right ranges and versions
int numDiffs = 135; // Makes about 100000 chunks overall
int numChunks = numInitialChunks;
for( int i = 0; i < numDiffs; i++ ){
// log() << "Generating new diff... " << i << endl;
BSONArrayBuilder diffsB;
BSONArrayBuilder newChunksB;
BSONObjIterator chunksIt( chunks );
while( chunksIt.more() ){
BSONObj chunk = chunksIt.next().Obj();
int randChoice = rand( 10 );
if( randChoice < 2 && numChunks < maxChunks ){
// Simulate a split
// log() << " ...starting a split with chunk " << chunk << endl;
BSONObjBuilder leftB;
BSONObjBuilder rightB;
BSONObjBuilder midB;
for( int k = 0; k < keySize; k++ ){
string field = string( "k" ) + string( 1, (char)('0' + k) );
BSONType maxType = chunk["max"].Obj()[field].type();
double max = maxType == NumberDouble ? chunk["max"].Obj()[field].Number() : 0.0;
BSONType minType = chunk["min"].Obj()[field].type();
double min = minType == NumberDouble ? chunk["min"].Obj()[field].Number() : 0.0;
if( minType == MinKey ){
midB.append( field, max - 1.0 );
}
else if( maxType == MaxKey ){
midB.append( field, min + 1.0 );
}
else {
midB.append( field, ( max + min ) / 2.0 );
}
}
BSONObj midPt = midB.obj();
// Only happens if we can't split the min chunk
if( midPt.isEmpty() ) continue;
leftB.append( chunk["min"] );
leftB.append( "max", midPt );
rightB.append( "min", midPt );
rightB.append( chunk["max"] );
leftB.append( chunk["shard"] );
rightB.append( chunk["shard"] );
version.incMajor();
version._minor = 0;
version.addToBSON( leftB, "lastmod" );
version.incMinor();
version.addToBSON( rightB, "lastmod" );
BSONObj left = leftB.obj();
BSONObj right = rightB.obj();
// log() << " ... split into " << left << " and " << right << endl;
newChunksB.append( left );
newChunksB.append( right );
diffsB.append( right );
diffsB.append( left );
numChunks++;
}
else if( randChoice < 4 && chunksIt.more() ){
// Simulate a migrate
// log() << " ...starting a migrate with chunk " << chunk << endl;
BSONObj prevShardChunk;
while( chunksIt.more() ){
prevShardChunk = chunksIt.next().Obj();
if( prevShardChunk["shard"].String() == chunk["shard"].String() ) break;
// log() << "... appending chunk from diff shard: " << prevShardChunk << endl;
newChunksB.append( prevShardChunk );
prevShardChunk = BSONObj();
}
// We need to move between different shards, hence the weirdness in logic here
if( ! prevShardChunk.isEmpty() ){
BSONObjBuilder newShardB;
BSONObjBuilder prevShardB;
newShardB.append( chunk["min"] );
newShardB.append( chunk["max"] );
prevShardB.append( prevShardChunk["min"] );
prevShardB.append( prevShardChunk["max"] );
int shardNum = rand( numShards );
newShardB.append( "shard", "shard" + string( 1, (char)('A' + shardNum) ) );
prevShardB.append( prevShardChunk["shard"] );
version.incMajor();
version._minor = 0;
version.addToBSON( newShardB, "lastmod" );
version.incMinor();
version.addToBSON( prevShardB, "lastmod" );
BSONObj newShard = newShardB.obj();
BSONObj prevShard = prevShardB.obj();
// log() << " ... migrated to " << newShard << " and updated " << prevShard << endl;
newChunksB.append( newShard );
newChunksB.append( prevShard );
diffsB.append( newShard );
diffsB.append( prevShard );
}
else{
// log() << "... appending chunk, no more left: " << chunk << endl;
newChunksB.append( chunk );
}
}
else{
// log() << "Appending chunk : " << chunk << endl;
newChunksB.append( chunk );
}
}
BSONArray diffs = diffsB.arr();
chunks = newChunksB.arr();
// log() << "Diffs generated : " << diffs << endl;
// log() << "All chunks : " << chunks << endl;
// Rarely entirely clear out our data
if( rand( 10 ) < 1 ){
diffs = chunks;
ranges.clear();
maxVersion = ShardChunkVersion( 0, 0, OID() );
maxShardVersions.clear();
}
// log() << "Total number of chunks : " << numChunks << " iteration " << i << endl;
DBClientMockCursor diffCursor( diffs );
differ->calculateConfigDiff( diffCursor );
validate( chunks, ranges, maxVersion, maxShardVersions );
}
}
