Exemple #1
0
        // Validates that the ranges and versions are valid given the chunks
        void validate( const BSONArray& chunks, RangeMap* ranges, ShardChunkVersion maxVersion, const VersionMap& maxShardVersions ){

            BSONObjIterator it( chunks );
            int chunkCount = 0;
            ShardChunkVersion foundMaxVersion;
            VersionMap foundMaxShardVersions;

            //
            // Validate that all the chunks are there and collect versions
            //

            while( it.more() ){

                BSONObj chunkDoc = it.next().Obj();
                chunkCount++;

                if( ranges != NULL ){

                    // log() << "Validating chunk " << chunkDoc << " size : " << ranges->size() << " vs " << chunkCount << endl;

                    RangeMap::iterator chunkRange = ranges->find( _inverse ? chunkDoc["max"].Obj() : chunkDoc["min"].Obj() );

                    ASSERT( chunkRange != ranges->end() );
                    ASSERT( chunkRange->second.woCompare( _inverse ? chunkDoc["min"].Obj() : chunkDoc["max"].Obj() ) == 0 );
                }

                ShardChunkVersion version = ShardChunkVersion::fromBSON( chunkDoc["lastmod"] );
                if( version > foundMaxVersion ) foundMaxVersion = version;

                ShardChunkVersion shardMaxVersion = foundMaxShardVersions[ chunkDoc["shard"].String() ];
                if( version > shardMaxVersion ) foundMaxShardVersions[ chunkDoc["shard"].String() ] = version;
            }
            // Make sure all chunks are accounted for
            if( ranges != NULL ) ASSERT( chunkCount == (int) ranges->size() );

            // log() << "Validating that all shard versions are up to date..." << endl;

            // Validate that all the versions are the same
            ASSERT( foundMaxVersion.isEquivalentTo( maxVersion ) );

            for( VersionMap::iterator it = foundMaxShardVersions.begin(); it != foundMaxShardVersions.end(); it++ ){

                ShardChunkVersion foundVersion = it->second;
                VersionMap::const_iterator maxIt = maxShardVersions.find( it->first );

                ASSERT( maxIt != maxShardVersions.end() );
                ASSERT( foundVersion.isEquivalentTo( maxIt->second ) );
            }
            // Make sure all shards are accounted for
            ASSERT( foundMaxShardVersions.size() == maxShardVersions.size() );
        }
Exemple #2
0
/**
  * Create the simple API
  * @param nrhs :: The number of parameters in the Matlab function call
  * @param prhs :: The data from the Matlab function call
  * @returns An integer indicating success/failure
  */
int CreateSimpleAPI(int, mxArray **, int nrhs, const mxArray *prhs[]) {

  // Ensure all libraries are loaded
  FrameworkManager::Instance();

  // Create directory to store mfiles
  std::string mpath("");
  if (nrhs == 0) {
    mpath = "MantidSimpleAPI";
  } else if (nrhs == 1) {
    char buffer[256];
    mxGetString(prhs[0], buffer, sizeof(buffer));
    mpath = std::string(buffer) + "/MantidSimpleAPI";
  } else {
    mexErrMsgTxt("SimpleAPI_Create takes either 0 or 1 arguments.");
  }

  Poco::File simpleAPI(mpath);
  if (simpleAPI.exists()) {
    simpleAPI.remove(true);
  }
  try {
    simpleAPI.createDirectory();
  } catch (std::exception &) {
    mexErrMsgTxt(
        "An error occurred while creating the directory for the simple API.");
  }

  std::vector<std::string> algKeys = AlgorithmFactory::Instance().getKeys();
  std::vector<std::string>::const_iterator sIter = algKeys.begin();
  typedef std::map<std::string, unsigned int> VersionMap;
  VersionMap vMap;
  for (; sIter != algKeys.end(); ++sIter) {
    std::string key = (*sIter);
    std::string name = key.substr(0, key.find("|"));
    VersionMap::iterator vIter = vMap.find(name);
    if (vIter == vMap.end())
      vMap.emplace(name, 1);
    else
      ++(vIter->second);
  }

  std::string contents_path = simpleAPI.path() + "/Contents.m";
  std::ofstream contents(contents_path.c_str());
  contents << "%A simpler API for Mantid\n%\n%The algorithms available are:\n";
  VersionMap::const_iterator vIter = vMap.begin();
  for (; vIter != vMap.end(); ++vIter) {
    contents << "% " << vIter->first << "\n";
    CreateSimpleAPIHelper(vIter->first, mpath + std::string("/"));
  }
  contents
      << "% For help with an individual command type \"help algorithm_name\"\n";
  contents.close();
  return 0;
}
Exemple #3
0
void VersionInfo::AddVersion(VersionMap& versions, const SceneVersion& version)
{
    DVASSERT(versions.find(version.version) == versions.end());
    versions.insert(VersionMap::value_type(version.version, version));
}
Exemple #4
0
        void run() {

            int numShards = 10;
            int numInitialChunks = 5;
            int maxChunks = 100000; // Needed to not overflow the BSONArray's max bytes
            int keySize = 2;

            BSONArrayBuilder chunksB;

            BSONObj lastSplitPt;
            ShardChunkVersion version( 1, 0, OID() );

            //
            // Generate numChunks with a given key size over numShards
            // All chunks have double key values, so we can split them a bunch
            //

            for( int i = -1; i < numInitialChunks; i++ ){

                BSONObjBuilder splitPtB;
                for( int k = 0; k < keySize; k++ ){
                    string field = string( "k" ) + string( 1, (char)('0' + k) );
                    if( i < 0 )
                        splitPtB.appendMinKey( field );
                    else if( i < numInitialChunks - 1 )
                        splitPtB.append( field, (double)i );
                    else
                        splitPtB.appendMaxKey( field );
                }
                BSONObj splitPt = splitPtB.obj();

                if( i >= 0 ){
                    BSONObjBuilder chunkB;

                    chunkB.append( "min", lastSplitPt );
                    chunkB.append( "max", splitPt );

                    int shardNum = rand( numShards );
                    chunkB.append( "shard", "shard" + string( 1, (char)('A' + shardNum) ) );

                    rand( 2 ) ? version.incMajor() : version.incMinor();
                    version.addToBSON( chunkB, "lastmod" );

                    chunksB.append( chunkB.obj() );
                }

                lastSplitPt = splitPt;
            }

            BSONArray chunks = chunksB.arr();

            // log() << "Chunks generated : " << chunks << endl;

            DBClientMockCursor chunksCursor( chunks );

            // Setup the empty ranges and versions first
            RangeMap ranges;
            ShardChunkVersion maxVersion = ShardChunkVersion( 0, 0, OID() );
            VersionMap maxShardVersions;

            // Create a differ which will track our progress
            boost::shared_ptr< DefaultDiffAdapter > differ( _inverse ? new InverseDiffAdapter() : new DefaultDiffAdapter() );
            differ->attach( "test", ranges, maxVersion, maxShardVersions );

            // Validate initial load
            differ->calculateConfigDiff( chunksCursor );
            validate( chunks, ranges, maxVersion, maxShardVersions );

            // Generate a lot of diffs, and keep validating that updating from the diffs always
            // gives us the right ranges and versions

            int numDiffs = 135; // Makes about 100000 chunks overall
            int numChunks = numInitialChunks;
            for( int i = 0; i < numDiffs; i++ ){

                // log() << "Generating new diff... " << i << endl;

                BSONArrayBuilder diffsB;
                BSONArrayBuilder newChunksB;
                BSONObjIterator chunksIt( chunks );

                while( chunksIt.more() ){

                    BSONObj chunk = chunksIt.next().Obj();

                    int randChoice = rand( 10 );

                    if( randChoice < 2 && numChunks < maxChunks ){
                        // Simulate a split

                        // log() << " ...starting a split with chunk " << chunk << endl;

                        BSONObjBuilder leftB;
                        BSONObjBuilder rightB;
                        BSONObjBuilder midB;

                        for( int k = 0; k < keySize; k++ ){
                            string field = string( "k" ) + string( 1, (char)('0' + k) );

                            BSONType maxType = chunk["max"].Obj()[field].type();
                            double max = maxType == NumberDouble ? chunk["max"].Obj()[field].Number() : 0.0;
                            BSONType minType = chunk["min"].Obj()[field].type();
                            double min = minType == NumberDouble ? chunk["min"].Obj()[field].Number() : 0.0;

                            if( minType == MinKey ){
                                midB.append( field, max - 1.0 );
                            }
                            else if( maxType == MaxKey ){
                                midB.append( field, min + 1.0 );
                            }
                            else {
                                midB.append( field, ( max + min ) / 2.0 );
                            }
                        }

                        BSONObj midPt = midB.obj();
                        // Only happens if we can't split the min chunk
                        if( midPt.isEmpty() ) continue;

                        leftB.append( chunk["min"] );
                        leftB.append( "max", midPt );
                        rightB.append( "min", midPt );
                        rightB.append( chunk["max"] );

                        leftB.append( chunk["shard"] );
                        rightB.append( chunk["shard"] );

                        version.incMajor();
                        version._minor = 0;
                        version.addToBSON( leftB, "lastmod" );
                        version.incMinor();
                        version.addToBSON( rightB, "lastmod" );

                        BSONObj left = leftB.obj();
                        BSONObj right = rightB.obj();

                        // log() << " ... split into " << left << " and " << right << endl;

                        newChunksB.append( left );
                        newChunksB.append( right );

                        diffsB.append( right );
                        diffsB.append( left );

                        numChunks++;
                    }
                    else if( randChoice < 4 && chunksIt.more() ){
                        // Simulate a migrate

                        // log() << " ...starting a migrate with chunk " << chunk << endl;

                        BSONObj prevShardChunk;
                        while( chunksIt.more() ){
                            prevShardChunk = chunksIt.next().Obj();
                            if( prevShardChunk["shard"].String() == chunk["shard"].String() ) break;

                            // log() << "... appending chunk from diff shard: " << prevShardChunk << endl;
                            newChunksB.append( prevShardChunk );

                            prevShardChunk = BSONObj();
                        }

                        // We need to move between different shards, hence the weirdness in logic here
                        if( ! prevShardChunk.isEmpty() ){

                            BSONObjBuilder newShardB;
                            BSONObjBuilder prevShardB;

                            newShardB.append( chunk["min"] );
                            newShardB.append( chunk["max"] );
                            prevShardB.append( prevShardChunk["min"] );
                            prevShardB.append( prevShardChunk["max"] );

                            int shardNum = rand( numShards );
                            newShardB.append( "shard", "shard" + string( 1, (char)('A' + shardNum) ) );
                            prevShardB.append( prevShardChunk["shard"] );

                            version.incMajor();
                            version._minor = 0;
                            version.addToBSON( newShardB, "lastmod" );
                            version.incMinor();
                            version.addToBSON( prevShardB, "lastmod" );

                            BSONObj newShard = newShardB.obj();
                            BSONObj prevShard = prevShardB.obj();

                            // log() << " ... migrated to " << newShard << " and updated " << prevShard << endl;

                            newChunksB.append( newShard );
                            newChunksB.append( prevShard );

                            diffsB.append( newShard );
                            diffsB.append( prevShard );

                        }
                        else{
                            // log() << "... appending chunk, no more left: " << chunk << endl;
                            newChunksB.append( chunk );
                        }
                    }
                    else{
                        // log() << "Appending chunk : " << chunk << endl;
                        newChunksB.append( chunk );
                    }

                }

                BSONArray diffs = diffsB.arr();
                chunks = newChunksB.arr();

                // log() << "Diffs generated : " << diffs << endl;
                // log() << "All chunks : " << chunks << endl;

                // Rarely entirely clear out our data
                if( rand( 10 ) < 1 ){
                    diffs = chunks;
                    ranges.clear();
                    maxVersion = ShardChunkVersion( 0, 0, OID() );
                    maxShardVersions.clear();
                }

                // log() << "Total number of chunks : " << numChunks << " iteration " << i << endl;

                DBClientMockCursor diffCursor( diffs );

                differ->calculateConfigDiff( diffCursor );

                validate( chunks, ranges, maxVersion, maxShardVersions );

            }

        }