예제 #1
0
파일: dump.cpp 프로젝트: renzhewk/mongo
    DiskLoc _repairExtent( Database* db , string ns, bool forward , DiskLoc eLoc , Writer& w ) {
        LogIndentLevel lil;

        if ( eLoc.getOfs() <= 0 ) {
            error() << "invalid extent ofs: " << eLoc.getOfs() << endl;
            return DiskLoc();
        }


        MongoDataFile * mdf = db->getFile( eLoc.a() );

        Extent * e = mdf->debug_getExtent( eLoc );
        if ( ! e->isOk() ) {
            warning() << "Extent not ok magic: " << e->magic << " going to try to continue" << endl;
        }

        log() << "length:" << e->length << endl;

        LogIndentLevel lil2;

        set<DiskLoc> seen;

        DiskLoc loc = forward ? e->firstRecord : e->lastRecord;
        while ( ! loc.isNull() ) {

            if ( ! seen.insert( loc ).second ) {
                error() << "infinite loop in extend, seen: " << loc << " before" << endl;
                break;
            }

            if ( loc.getOfs() <= 0 ) {
                error() << "offset is 0 for record which should be impossible" << endl;
                break;
            }
            log(1) << loc << endl;
            Record* rec = loc.rec();
            BSONObj obj;
            try {
                obj = loc.obj();
                assert( obj.valid() );
                LOG(1) << obj << endl;
                w( obj );
            }
            catch ( std::exception& e ) {
                log() << "found invalid document @ " << loc << " " << e.what() << endl;
                if ( ! obj.isEmpty() ) {
                    try {
                        BSONElement e = obj.firstElement();
                        stringstream ss;
                        ss << "first element: " << e;
                        log() << ss.str();
                    }
                    catch ( std::exception& ) {
                    }
                }
            }
            loc = forward ? rec->getNext( loc ) : rec->getPrev( loc );
        }
        return forward ? e->xnext : e->xprev;

    }
예제 #2
0
파일: mr.cpp 프로젝트: DumaGit/mongo
        /**
         * actually applies a reduce, to a list of tuples (key, value).
         * After the call, tuples will hold a single tuple {"0": key, "1": value}
         */
        void JSReducer::_reduce( const BSONList& tuples , BSONObj& key , int& endSizeEstimate ) {
            uassert( 10074 ,  "need values" , tuples.size() );

            int sizeEstimate = ( tuples.size() * tuples.begin()->getField( "value" ).size() ) + 128;

            // need to build the reduce args: ( key, [values] )
            BSONObjBuilder reduceArgs( sizeEstimate );
            boost::scoped_ptr<BSONArrayBuilder>  valueBuilder;
            int sizeSoFar = 0;
            unsigned n = 0;
            for ( ; n<tuples.size(); n++ ) {
                BSONObjIterator j(tuples[n]);
                BSONElement keyE = j.next();
                if ( n == 0 ) {
                    reduceArgs.append( keyE );
                    key = keyE.wrap();
                    sizeSoFar = 5 + keyE.size();
                    valueBuilder.reset(new BSONArrayBuilder( reduceArgs.subarrayStart( "tuples" ) ));
                }

                BSONElement ee = j.next();

                uassert( 13070 , "value too large to reduce" , ee.size() < ( BSONObjMaxUserSize / 2 ) );

                if ( sizeSoFar + ee.size() > BSONObjMaxUserSize ) {
                    assert( n > 1 ); // if not, inf. loop
                    break;
                }

                valueBuilder->append( ee );
                sizeSoFar += ee.size();
            }
            assert(valueBuilder);
            valueBuilder->done();
            BSONObj args = reduceArgs.obj();

            Scope * s = _func.scope();

            s->invokeSafe( _func.func() , &args, 0 );
            ++numReduces;

            if ( s->type( "return" ) == Array ) {
                uasserted( 10075 , "reduce -> multiple not supported yet");
                return;
            }

            endSizeEstimate = key.objsize() + ( args.objsize() / tuples.size() );

            if ( n == tuples.size() )
                return;

            // the input list was too large, add the rest of elmts to new tuples and reduce again
            // note: would be better to use loop instead of recursion to avoid stack overflow
            BSONList x;
            for ( ; n < tuples.size(); n++ ) {
                x.push_back( tuples[n] );
            }
            BSONObjBuilder temp( endSizeEstimate );
            temp.append( key.firstElement() );
            s->append( temp , "1" , "return" );
            x.push_back( temp.obj() );
            _reduce( x , key , endSizeEstimate );
        }
예제 #3
0
        bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) {
            Timer t;
            string ns = dbname + '.' + cmdObj.firstElement().valuestr();

            string key = cmdObj["key"].valuestrsafe();
            BSONObj keyPattern = BSON( key << 1 );

            BSONObj query = getQuery( cmdObj );

            int bufSize = BSONObjMaxUserSize - 4096;
            BufBuilder bb( bufSize );
            char * start = bb.buf();

            BSONArrayBuilder arr( bb );
            BSONElementSet values;

            long long nscanned = 0; // locations looked at
            long long nscannedObjects = 0; // full objects looked at
            long long n = 0; // matches
            MatchDetails md;

            NamespaceDetails * d = nsdetails( ns.c_str() );

            if ( ! d ) {
                result.appendArray( "values" , BSONObj() );
                result.append( "stats" , BSON( "n" << 0 << "nscanned" << 0 << "nscannedObjects" << 0 ) );
                return true;
            }

            shared_ptr<Cursor> cursor;
            if ( ! query.isEmpty() ) {
                cursor = NamespaceDetailsTransient::getCursor(ns.c_str() , query , BSONObj() );
            }
            else {

                // query is empty, so lets see if we can find an index
                // with the key so we don't have to hit the raw data
                NamespaceDetails::IndexIterator ii = d->ii();
                while ( ii.more() ) {
                    IndexDetails& idx = ii.next();

                    if ( d->isMultikey( ii.pos() - 1 ) )
                        continue;

                    if ( idx.inKeyPattern( key ) ) {
                        cursor = NamespaceDetailsTransient::bestGuessCursor( ns.c_str() ,
                                                                            BSONObj() ,
                                                                            idx.keyPattern() );
                        if( cursor.get() ) break;
                    }

                }

                if ( ! cursor.get() )
                    cursor = NamespaceDetailsTransient::getCursor(ns.c_str() , query , BSONObj() );

            }

            
            verify( cursor );
            string cursorName = cursor->toString();
            
            auto_ptr<ClientCursor> cc (new ClientCursor(QueryOption_NoCursorTimeout, cursor, ns));

            while ( cursor->ok() ) {
                nscanned++;
                bool loadedRecord = false;

                if ( cursor->currentMatches( &md ) && !cursor->getsetdup( cursor->currLoc() ) ) {
                    n++;

                    BSONObj holder;
                    BSONElementSet temp;
                    loadedRecord = ! cc->getFieldsDotted( key , temp, holder );

                    for ( BSONElementSet::iterator i=temp.begin(); i!=temp.end(); ++i ) {
                        BSONElement e = *i;
                        if ( values.count( e ) )
                            continue;

                        int now = bb.len();

                        uassert(10044,  "distinct too big, 16mb cap", ( now + e.size() + 1024 ) < bufSize );

                        arr.append( e );
                        BSONElement x( start + now );

                        values.insert( x );
                    }
                }

                if ( loadedRecord || md.hasLoadedRecord() )
                    nscannedObjects++;

                cursor->advance();

                if (!cc->yieldSometimes( ClientCursor::MaybeCovered )) {
                    cc.release();
                    break;
                }

                RARELY killCurrentOp.checkForInterrupt();
            }

            verify( start == bb.buf() );

            result.appendArray( "values" , arr.done() );

            {
                BSONObjBuilder b;
                b.appendNumber( "n" , n );
                b.appendNumber( "nscanned" , nscanned );
                b.appendNumber( "nscannedObjects" , nscannedObjects );
                b.appendNumber( "timems" , t.millis() );
                b.append( "cursor" , cursorName );
                result.append( "stats" , b.obj() );
            }

            return true;
        }
예제 #4
0
    /* ns:      namespace, e.g. <database>.<collection>
       pattern: the "where" clause / criteria
       justOne: stop after 1 match
       god:     allow access to system namespaces, and don't yield
    */
    long long deleteObjects(const char *ns, BSONObj pattern, bool justOne, bool logop, bool god, RemoveSaver * rs ) {
        if( !god ) {
            if ( strstr(ns, ".system.") ) {
                /* note a delete from system.indexes would corrupt the db
                if done here, as there are pointers into those objects in
                NamespaceDetails.
                */
                uassert(12050, "cannot delete from system namespace", legalClientSystemNS( ns , true ) );
            }
            if ( strchr( ns , '$' ) ) {
                log() << "cannot delete from collection with reserved $ in name: " << ns << endl;
                uassert( 10100 ,  "cannot delete from collection with reserved $ in name", strchr(ns, '$') == 0 );
            }
        }

        {
            NamespaceDetails *d = nsdetails( ns );
            if ( ! d )
                return 0;
            uassert( 10101 ,  "can't remove from a capped collection" , ! d->isCapped() );
        }

        long long nDeleted = 0;

        shared_ptr< Cursor > creal = NamespaceDetailsTransient::getCursor( ns, pattern );

        if( !creal->ok() )
            return nDeleted;

        shared_ptr< Cursor > cPtr = creal;
        auto_ptr<ClientCursor> cc( new ClientCursor( QueryOption_NoCursorTimeout, cPtr, ns) );
        cc->setDoingDeletes( true );

        CursorId id = cc->cursorid();

        bool canYield = !god && !(creal->matcher() && creal->matcher()->docMatcher().atomic());

        do {
            // TODO: we can generalize this I believe
            //       
            bool willNeedRecord = (creal->matcher() && creal->matcher()->needRecord()) || pattern.isEmpty() || isSimpleIdQuery( pattern );
            if ( ! willNeedRecord ) {
                // TODO: this is a total hack right now
                // check if the index full encompasses query
                
                if ( pattern.nFields() == 1 && 
                     str::equals( pattern.firstElement().fieldName() , creal->indexKeyPattern().firstElement().fieldName() ) )
                    willNeedRecord = true;
            }
            
            if ( canYield && ! cc->yieldSometimes( willNeedRecord ? ClientCursor::WillNeed : ClientCursor::MaybeCovered ) ) {
                cc.release(); // has already been deleted elsewhere
                // TODO should we assert or something?
                break;
            }
            if ( !cc->ok() ) {
                break; // if we yielded, could have hit the end
            }

            // this way we can avoid calling prepareToYield() every time (expensive)
            // as well as some other nuances handled
            cc->setDoingDeletes( true );

            DiskLoc rloc = cc->currLoc();
            BSONObj key = cc->currKey();

            bool match = creal->currentMatches();

            cc->advance();
            
            if ( ! match )
                continue;

            // SERVER-5198 Advance past the document to be modified, but see SERVER-5725.
            while( cc->ok() && rloc == cc->currLoc() ) {
                cc->advance();
            }
            
            bool foundAllResults = ( justOne || !cc->ok() );

            if ( !foundAllResults ) {
                // NOTE: Saving and restoring a btree cursor's position was historically described
                // as slow here.
                cc->c()->prepareToTouchEarlierIterate();
            }

            {
                BSONElement e;
                if( BSONObj::make( rloc.rec() ).getObjectID( e ) ) {
                    BSONObjBuilder b;
                    b.append( e );
                    bool replJustOne = true;
		    if(logop)
			    logOp( "d", ns, b.done(), 0, &replJustOne );
		    postNotification("d",ns,b.done(),0);
                }
                else {
                    problem() << "deleted object without id, not logging" << endl;
                }
            }

            theDataFileMgr.deleteRecord(ns, rloc.rec(), rloc);
            nDeleted++;
            if ( foundAllResults ) {
                break;
            }
            cc->c()->recoverFromTouchingEarlierIterate();
         
            if( !god ) 
                getDur().commitIfNeeded();

            if( debug && god && nDeleted == 100 ) 
                log() << "warning high number of deletes with god=true which could use significant memory" << endl;
        }
        while ( cc->ok() );

        if ( cc.get() && ClientCursor::find( id , false ) == 0 ) {
            // TODO: remove this and the id declaration above if this doesn't trigger
            //       if it does, then i'm very confused (ERH 06/2011)
            error() << "this should be impossible" << endl;
            printStackTrace();
            cc.release();
        }

        return nDeleted;
    }
예제 #5
0
        void run(){
            Scope * s = globalScriptEngine->newScope();
            
            { // date
                BSONObj o;
                { 
                    BSONObjBuilder b;
                    b.appendDate( "d" , 123456789 );
                    o = b.obj();
                }
                s->setObject( "x" , o );
                
                s->invoke( "return x.d.getTime() != 12;" , BSONObj() );
                ASSERT_EQUALS( true, s->getBoolean( "return" ) );
                
                s->invoke( "z = x.d.getTime();" , BSONObj() );
                ASSERT_EQUALS( 123456789 , s->getNumber( "z" ) );
                
                s->invoke( "z = { z : x.d }" , BSONObj() );
                BSONObj out = s->getObject( "z" );
                ASSERT( out["z"].type() == Date );
            }

            { // regex
                BSONObj o;
                { 
                    BSONObjBuilder b;
                    b.appendRegex( "r" , "^a" , "i" );
                    o = b.obj();
                }
                s->setObject( "x" , o );
                
                s->invoke( "z = x.r.test( 'b' );" , BSONObj() );
                ASSERT_EQUALS( false , s->getBoolean( "z" ) );

                s->invoke( "z = x.r.test( 'a' );" , BSONObj() );
                ASSERT_EQUALS( true , s->getBoolean( "z" ) );

                s->invoke( "z = x.r.test( 'ba' );" , BSONObj() );
                ASSERT_EQUALS( false , s->getBoolean( "z" ) );

                s->invoke( "z = { a : x.r };" , BSONObj() );

                BSONObj out = s->getObject("z");
                ASSERT_EQUALS( (string)"^a" , out["a"].regex() );
                ASSERT_EQUALS( (string)"i" , out["a"].regexFlags() );

            }
            
            // array
            {
                BSONObj o = fromjson( "{r:[1,2,3]}" );
                s->setObject( "x", o, false );                
                BSONObj out = s->getObject( "x" );
                ASSERT_EQUALS( Array, out.firstElement().type() );

                s->setObject( "x", o, true );                
                out = s->getObject( "x" );
                ASSERT_EQUALS( Array, out.firstElement().type() );
            }
            
            delete s;
        }
예제 #6
0
    bool wrappedRun(OperationContext* txn,
                    const string& dbname,
                    BSONObj& jsobj,
                    string& errmsg,
                    BSONObjBuilder& anObjBuilder) {
        BSONElement e = jsobj.firstElement();
        const string toDeleteNs = dbname + '.' + e.valuestr();
        if (!serverGlobalParams.quiet) {
            LOG(0) << "CMD: dropIndexes " << toDeleteNs << endl;
        }

        Client::Context ctx(toDeleteNs);
        Database* db = ctx.db();

        Collection* collection = db->getCollection( txn, toDeleteNs );
        if ( ! collection ) {
            errmsg = "ns not found";
            return false;
        }

        stopIndexBuilds(txn, db, jsobj);

        IndexCatalog* indexCatalog = collection->getIndexCatalog();
        anObjBuilder.appendNumber("nIndexesWas", indexCatalog->numIndexesTotal() );


        BSONElement f = jsobj.getField("index");
        if ( f.type() == String ) {

            string indexToDelete = f.valuestr();

            if ( indexToDelete == "*" ) {
                Status s = indexCatalog->dropAllIndexes(txn, false);
                if ( !s.isOK() ) {
                    appendCommandStatus( anObjBuilder, s );
                    return false;
                }
                anObjBuilder.append("msg", "non-_id indexes dropped for collection");
                return true;
            }

            IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByName( indexToDelete );
            if ( desc == NULL ) {
                errmsg = str::stream() << "index not found with name [" << indexToDelete << "]";
                return false;
            }

            if ( desc->isIdIndex() ) {
                errmsg = "cannot drop _id index";
                return false;
            }

            Status s = indexCatalog->dropIndex(txn, desc);
            if ( !s.isOK() ) {
                appendCommandStatus( anObjBuilder, s );
                return false;
            }

            return true;
        }

        if ( f.type() == Object ) {
            IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByKeyPattern( f.embeddedObject() );
            if ( desc == NULL ) {
                errmsg = "can't find index with key:";
                errmsg += f.embeddedObject().toString();
                return false;
            }

            if ( desc->isIdIndex() ) {
                errmsg = "cannot drop _id index";
                return false;
            }

            Status s = indexCatalog->dropIndex(txn, desc);
            if ( !s.isOK() ) {
                appendCommandStatus( anObjBuilder, s );
                return false;
            }

            return true;
        }

        errmsg = "invalid index name spec";
        return false;
    }
예제 #7
0
      // PD_TRACE_DECLARE_FUNCTION ( SDB__IXMKEYGEN__GENKEYSWITHARRELE, "_ixmKeyGenerator::_genKeyWithArrayEle" )
      INT32 _genKeyWithArrayEle( BSONElement *keyEles,
                                 UINT32 eleNum,
                                 const BSONElement *arrElement,
                                 const CHAR *arrEleName,
                                 UINT32 arrElePos,
                                 BSONObjSet &keys ) const
      {
         PD_TRACE_ENTRY ( SDB__IXMKEYGEN__GENKEYSWITHARRELE );
         INT32 rc = SDB_OK ;
         BSONObj arrObj = arrElement->embeddedObject() ;
         if ( arrObj.firstElement().eoo() )
         {
            keyEles[arrElePos] = *arrElement ;
            rc = _genKeyWithNormalEle( keyEles, eleNum, keys ) ;
            if ( SDB_OK != rc )
            {
               goto error ;
            }
         }

         if ( '\0' == *arrEleName )
         {
            BSONObjIterator itr( arrObj ) ;
            BSONElement &e = keyEles[arrElePos] ;
            while ( itr.more() )
            {
               e = itr.next() ;
               rc = _genKeyWithNormalEle( keyEles, eleNum, keys ) ;
               if ( SDB_OK != rc )
               {
                  goto error ;
               }
            }
         }
         else
         {
            BSONObjIterator itr( arrObj ) ;
            while ( itr.more() )
            {
               const CHAR *dottedName = arrEleName ;
               BSONElement next = itr.next() ;
               if ( Object == next.type() )
               {
                  BSONElement e =
                     next.embeddedObject()
                     .getFieldDottedOrArray( dottedName ) ;
                  if ( Array == e.type() )
                  {
                     rc = _genKeyWithArrayEle(keyEles, eleNum,
                                              &e,
                                              dottedName, arrElePos,
                                              keys) ;
                     if ( SDB_OK != rc )
                     {
                        goto error ;
                     }
                     else
                     {
                        continue ;
                     }
                  }
                  else
                  {
                     keyEles[arrElePos] = e ;
                  }
               }
               else
               {
                  keyEles[arrElePos] = BSONElement() ;
               }

               rc = _genKeyWithNormalEle( keyEles, eleNum, keys ) ;
               if ( SDB_OK != rc )
               {
                  goto error ;
               }
            } 
         }
      done:
         PD_TRACE_EXITRC( SDB__IXMKEYGEN__GENKEYSWITHARRELE, rc ) ;
         return rc ;
      error:
         goto done ;
      }
TEST(MatchExpressionParserTest, ParseIntegerElementToLongRejectsLargestDecimal) {
    BSONObj query = BSON("" << Decimal128(Decimal128::kLargestPositive));
    ASSERT_NOT_OK(MatchExpressionParser::parseIntegerElementToLong(query.firstElement()));
}
TEST(MatchExpressionParserTest, ParseIntegerElementToNonNegativeLongRejectsNegative) {
    BSONObj query = BSON("" << -2LL);
    ASSERT_NOT_OK(
        MatchExpressionParser::parseIntegerElementToNonNegativeLong(query.firstElement()));
}
TEST(MatchExpressionParserTest, ParseIntegerElementToLongRejectsString) {
    BSONObj query = BSON(""
                         << "1");
    ASSERT_NOT_OK(MatchExpressionParser::parseIntegerElementToLong(query.firstElement()));
}
TEST(MatchExpressionParserTest, ParseIntegerElementToLongRejectsNonIntegralDecimal) {
    BSONObj query = BSON("" << Decimal128("2.5"));
    ASSERT_NOT_OK(MatchExpressionParser::parseIntegerElementToLong(query.firstElement()));
}
TEST(MatchExpressionParserTest, ParseIntegerElementToLongRejectsTooLargeNegativeDouble) {
    BSONObj query = BSON("" << std::numeric_limits<double>::min());
    ASSERT_NOT_OK(MatchExpressionParser::parseIntegerElementToLong(query.firstElement()));
}
TEST(MatchExpressionParserTest, ParseIntegerElementToLongRejectsTooLargeDouble) {
    BSONObj query = BSON("" << MatchExpressionParser::kLongLongMaxPlusOneAsDouble);
    ASSERT_NOT_OK(MatchExpressionParser::parseIntegerElementToLong(query.firstElement()));
}
예제 #14
0
파일: apply_ops.cpp 프로젝트: ambroff/mongo
        virtual bool run(OperationContext* txn, const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) {

            if ( cmdObj.firstElement().type() != Array ) {
                errmsg = "ops has to be an array";
                return false;
            }

            BSONObj ops = cmdObj.firstElement().Obj();

            {
                // check input
                BSONObjIterator i( ops );
                while ( i.more() ) {
                    BSONElement e = i.next();
                    if (!_checkOperation(e, errmsg)) {
                        return false;
                    }
                }
            }

            // SERVER-4328 todo : is global ok or does this take a long time? i believe multiple 
            // ns used so locking individually requires more analysis
            ScopedTransaction scopedXact(txn, MODE_X);
            Lock::GlobalWrite globalWriteLock(txn->lockState());

            if (!fromRepl &&
                !repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase(dbname)) {
                return appendCommandStatus(result, Status(ErrorCodes::NotMaster, str::stream()
                    << "Not primary while applying ops to database " << dbname));
            }

            // Preconditions check reads the database state, so needs to be done locked
            if ( cmdObj["preCondition"].type() == Array ) {
                BSONObjIterator i( cmdObj["preCondition"].Obj() );
                while ( i.more() ) {
                    BSONObj f = i.next().Obj();

                    DBDirectClient db( txn );
                    BSONObj realres = db.findOne( f["ns"].String() , f["q"].Obj() );

                    // Apply-ops would never have a $where matcher, so use the default callback,
                    // which will throw an error if $where is found.
                    Matcher m(f["res"].Obj());
                    if ( ! m.matches( realres ) ) {
                        result.append( "got" , realres );
                        result.append( "whatFailed" , f );
                        errmsg = "pre-condition failed";
                        return false;
                    }
                }
            }

            // apply
            int num = 0;
            int errors = 0;
            
            BSONObjIterator i( ops );
            BSONArrayBuilder ab;
            const bool alwaysUpsert = cmdObj.hasField("alwaysUpsert") ?
                    cmdObj["alwaysUpsert"].trueValue() : true;
            
            while ( i.more() ) {
                BSONElement e = i.next();
                const BSONObj& temp = e.Obj();

                // Ignore 'n' operations.
                const char *opType = temp["op"].valuestrsafe();
                if (*opType == 'n') continue;

                const string ns = temp["ns"].String();

                // Run operations under a nested lock as a hack to prevent yielding.
                //
                // The list of operations is supposed to be applied atomically; yielding
                // would break atomicity by allowing an interruption or a shutdown to occur
                // after only some operations are applied.  We are already locked globally
                // at this point, so taking a DBLock on the namespace creates a nested lock,
                // and yields are disallowed for operations that hold a nested lock.
                //
                // We do not have a wrapping WriteUnitOfWork so it is possible for a journal
                // commit to happen with a subset of ops applied.
                // TODO figure out what to do about this.
                Lock::GlobalWrite globalWriteLockDisallowTempRelease(txn->lockState());

                // Ensures that yielding will not happen (see the comment above).
                DEV {
                    Locker::LockSnapshot lockSnapshot;
                    invariant(!txn->lockState()->saveLockStateAndUnlock(&lockSnapshot));
                };

                OldClientContext ctx(txn, ns);

                Status status(ErrorCodes::InternalError, "");
                while (true) {
                    try {
                        // We assume that in the WriteConflict retry case, either the op rolls back
                        // any changes it makes or is otherwise safe to rerun.
                        status =
                            repl::applyOperation_inlock(txn, ctx.db(), temp, false, alwaysUpsert);
                        break;
                    }
                    catch (const WriteConflictException& wce) {
                        LOG(2) << "WriteConflictException in applyOps command, retrying.";
                        txn->recoveryUnit()->commitAndRestart();
                        continue;
                    }
                }

                ab.append(status.isOK());
                if (!status.isOK()) {
                    errors++;
                }

                num++;

                WriteUnitOfWork wuow(txn);
                logOpForDbHash(txn, ns.c_str());
                wuow.commit();
            }

            result.append( "applied" , num );
            result.append( "results" , ab.arr() );

            if ( ! fromRepl ) {
                // We want this applied atomically on slaves
                // so we re-wrap without the pre-condition for speed

                string tempNS = str::stream() << dbname << ".$cmd";

                // TODO: possibly use mutable BSON to remove preCondition field
                // once it is available
                BSONObjIterator iter(cmdObj);
                BSONObjBuilder cmdBuilder;

                while (iter.more()) {
                    BSONElement elem(iter.next());
                    if (strcmp(elem.fieldName(), "preCondition") != 0) {
                        cmdBuilder.append(elem);
                    }
                }

                const BSONObj cmdRewritten = cmdBuilder.done();

                // We currently always logOp the command regardless of whether the individial ops
                // succeeded and rely on any failures to also happen on secondaries. This isn't
                // perfect, but it's what the command has always done and is part of its "correct"
                // behavior.
                while (true) {
                    try {
                        WriteUnitOfWork wunit(txn);
                        getGlobalEnvironment()->getOpObserver()->onApplyOps(txn,
                                                                            tempNS,
                                                                            cmdRewritten);
                        wunit.commit();
                        break;
                    }
                    catch (const WriteConflictException& wce) {
                        LOG(2) <<
                            "WriteConflictException while logging applyOps command, retrying.";
                        txn->recoveryUnit()->commitAndRestart();
                        continue;
                    }
                }
            }

            if (errors != 0) {
                return false;
            }

            return true;
        }
예제 #15
0
    ProjectionExec::ProjectionExec(const BSONObj& spec, const MatchExpression* queryExpression)
        : _include(true),
          _special(false),
          _source(spec),
          _includeID(true),
          _skip(0),
          _limit(-1),
          _arrayOpType(ARRAY_OP_NORMAL),
          _hasNonSimple(false),
          _hasDottedField(false),
          _queryExpression(queryExpression),
          _hasReturnKey(false) {

        // Are we including or excluding fields?
        // -1 when we haven't initialized it.
        // 1 when we're including
        // 0 when we're excluding.
        int include_exclude = -1;

        BSONObjIterator it(_source);
        while (it.more()) {
            BSONElement e = it.next();

            if (!e.isNumber() && !e.isBoolean()) {
                _hasNonSimple = true;
            }

            if (Object == e.type()) {
                BSONObj obj = e.embeddedObject();
                verify(1 == obj.nFields());

                BSONElement e2 = obj.firstElement();
                if (mongoutils::str::equals(e2.fieldName(), "$slice")) {
                    if (e2.isNumber()) {
                        int i = e2.numberInt();
                        if (i < 0) {
                            add(e.fieldName(), i, -i); // limit is now positive
                        }
                        else {
                            add(e.fieldName(), 0, i);
                        }
                    }
                    else {
                        verify(e2.type() == Array);
                        BSONObj arr = e2.embeddedObject();
                        verify(2 == arr.nFields());

                        BSONObjIterator it(arr);
                        int skip = it.next().numberInt();
                        int limit = it.next().numberInt();

                        verify(limit > 0);

                        add(e.fieldName(), skip, limit);
                    }
                }
                else if (mongoutils::str::equals(e2.fieldName(), "$elemMatch")) {
                    _arrayOpType = ARRAY_OP_ELEM_MATCH;

                    // Create a MatchExpression for the elemMatch.
                    BSONObj elemMatchObj = e.wrap();
                    verify(elemMatchObj.isOwned());
                    _elemMatchObjs.push_back(elemMatchObj);
                    StatusWithMatchExpression swme = MatchExpressionParser::parse(elemMatchObj);
                    verify(swme.isOK());
                    // And store it in _matchers.
                    _matchers[mongoutils::str::before(e.fieldName(), '.').c_str()]
                        = swme.getValue();

                    add(e.fieldName(), true);
                }
                else if (mongoutils::str::equals(e2.fieldName(), "$meta")) {
                    verify(String == e2.type());
                    if (mongoutils::str::equals(e2.valuestr(), "text")) {
                        _meta[e.fieldName()] = META_TEXT;
                    }
                    else if (mongoutils::str::equals(e2.valuestr(), "diskloc")) {
                        _meta[e.fieldName()] = META_DISKLOC;
                    }
                    else if (mongoutils::str::equals(e2.valuestr(), "indexKey")) {
                        _hasReturnKey = true;
                        // The index key clobbers everything so just stop parsing here.
                        return;
                    }
                    else {
                        // This shouldn't happen, should be caught by parsing.
                        verify(0);
                    }
                }
                else {
                    verify(0);
                }
            }
            else if (mongoutils::str::equals(e.fieldName(), "_id") && !e.trueValue()) {
                _includeID = false;
            }
            else {
                add(e.fieldName(), e.trueValue());

                // Projections of dotted fields aren't covered.
                if (mongoutils::str::contains(e.fieldName(), '.')) {
                    _hasDottedField = true;
                }

                // Validate input.
                if (include_exclude == -1) {
                    // If we haven't specified an include/exclude, initialize include_exclude.
                    // We expect further include/excludes to match it.
                    include_exclude = e.trueValue();
                    _include = !e.trueValue();
                }
            }

            if (mongoutils::str::contains(e.fieldName(), ".$")) {
                _arrayOpType = ARRAY_OP_POSITIONAL;
            }
        }
    }
TEST(MatchExpressionParserTest, ParseIntegerElementToLongAcceptsNegative) {
    BSONObj query = BSON("" << -2LL);
    auto result = MatchExpressionParser::parseIntegerElementToLong(query.firstElement());
    ASSERT_OK(result.getStatus());
    ASSERT_EQ(-2LL, result.getValue());
}
예제 #17
0
파일: insert.cpp 프로젝트: AlanLiu-AI/mongo
    StatusWith<BSONObj> fixDocumentForInsert( const BSONObj& doc ) {
        if ( doc.objsize() > BSONObjMaxUserSize )
            return StatusWith<BSONObj>( ErrorCodes::BadValue,
                                        str::stream()
                                        << "object to insert too large"
                                        << doc.objsize() );

        bool firstElementIsId = doc.firstElement().fieldNameStringData() == "_id";
        bool hasTimestampToFix = false;
        {
            BSONObjIterator i( doc );
            while ( i.more() ) {
                BSONElement e = i.next();

                if ( e.type() == Timestamp && e.timestampValue() == 0 ) {
                    // we replace Timestamp(0,0) at the top level with a correct value
                    // in the fast pass, we just mark that we want to swap
                    hasTimestampToFix = true;
                    break;
                }

                const char* fieldName = e.fieldName();

                if ( fieldName[0] == '$' ) {
                    return StatusWith<BSONObj>( ErrorCodes::BadValue,
                                                str::stream()
                                                << "Document can't have $ prefixed field names: "
                                                << e.fieldName() );
                }

                // check no regexp for _id (SERVER-9502)
                // also, disallow undefined and arrays
                if ( str::equals( fieldName, "_id") ) {
                    if ( e.type() == RegEx ) {
                        return StatusWith<BSONObj>( ErrorCodes::BadValue,
                                                    "can't use a regex for _id" );
                    }
                    if ( e.type() == Undefined ) {
                        return StatusWith<BSONObj>( ErrorCodes::BadValue,
                                                    "can't use a undefined for _id" );
                    }
                    if ( e.type() == Array ) {
                        return StatusWith<BSONObj>( ErrorCodes::BadValue,
                                                    "can't use an array for _id" );
                    }
                    if ( e.type() == Object ) {
                        BSONObj o = e.Obj();
                        Status s = o.storageValidEmbedded();
                        if ( !s.isOK() )
                            return StatusWith<BSONObj>( s );
                    }
                }

            }
        }

        if ( firstElementIsId && !hasTimestampToFix )
            return StatusWith<BSONObj>( BSONObj() );

        bool hadId = firstElementIsId;

        BSONObjIterator i( doc );

        BSONObjBuilder b( doc.objsize() + 16 );
        if ( firstElementIsId ) {
            b.append( doc.firstElement() );
            i.next();
        }
        else {
            BSONElement e = doc["_id"];
            if ( e.type() ) {
                b.append( e );
                hadId = true;
            }
            else {
                b.appendOID( "_id", NULL, true );
            }
        }

        while ( i.more() ) {
            BSONElement e = i.next();
            if ( hadId && e.fieldNameStringData() == "_id" ) {
                // no-op
            }
            else if ( e.type() == Timestamp && e.timestampValue() == 0 ) {
                mutex::scoped_lock lk(OpTime::m);
                b.append( e.fieldName(), OpTime::now(lk) );
            }
            else {
                b.append( e );
            }
        }
        return StatusWith<BSONObj>( b.obj() );
    }
예제 #18
0
파일: cloner.cpp 프로젝트: Ry4an/mongo
        void operator()( DBClientCursorBatchIterator &i ) {
            mongolock l( true );
            if ( context ) {
                context->relocked();
            }

            while( i.moreInCurrentBatch() ) {
                if ( n % 128 == 127 /*yield some*/ ) {
                    time_t now = time(0);
                    if( now - lastLog >= 60 ) { 
                        // report progress
                        if( lastLog )
                            log() << "clone " << to_collection << ' ' << n << endl;
                        lastLog = now;
                    }
                    mayInterrupt( _mayBeInterrupted );
                    dbtempreleaseif t( _mayYield );
                }

                BSONObj tmp = i.nextSafe();

                /* assure object is valid.  note this will slow us down a little. */
                if ( !tmp.valid() ) {
                    stringstream ss;
                    ss << "Cloner: skipping corrupt object from " << from_collection;
                    BSONElement e = tmp.firstElement();
                    try {
                        e.validate();
                        ss << " firstElement: " << e;
                    }
                    catch( ... ) {
                        ss << " firstElement corrupt";
                    }
                    out() << ss.str() << endl;
                    continue;
                }

                ++n;

                BSONObj js = tmp;
                if ( isindex ) {
                    assert( strstr(from_collection, "system.indexes") );
                    js = fixindex(tmp);
                    storedForLater->push_back( js.getOwned() );
                    continue;
                }

                try {
                    theDataFileMgr.insertWithObjMod(to_collection, js);
                    if ( logForRepl )
                        logOp("i", to_collection, js);

                    getDur().commitIfNeeded();
                }
                catch( UserException& e ) {
                    log() << "warning: exception cloning object in " << from_collection << ' ' << e.what() << " obj:" << js.toString() << '\n';
                }

                RARELY if ( time( 0 ) - saveLast > 60 ) {
                    log() << n << " objects cloned so far from collection " << from_collection << endl;
                    saveLast = time( 0 );
                }
            }
        }
예제 #19
0
    bool run(OperationContext* txn, const string& dbname , BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) {
        DBDirectClient db;

        BSONElement e = jsobj.firstElement();
        string toDeleteNs = dbname + '.' + e.valuestr();

        LOG(0) << "CMD: reIndex " << toDeleteNs << endl;

        Lock::DBWrite dbXLock(txn->lockState(), dbname);
        Client::Context ctx(toDeleteNs);

        Collection* collection = ctx.db()->getCollection( txn, toDeleteNs );

        if ( !collection ) {
            errmsg = "ns not found";
            return false;
        }

        BackgroundOperation::assertNoBgOpInProgForNs( toDeleteNs );

        std::vector<BSONObj> indexesInProg = stopIndexBuilds(txn, ctx.db(), jsobj);

        list<BSONObj> all;
        auto_ptr<DBClientCursor> i = db.query( dbname + ".system.indexes" , BSON( "ns" << toDeleteNs ) , 0 , 0 , 0 , QueryOption_SlaveOk );
        BSONObjBuilder b;
        while ( i->more() ) {
            const BSONObj spec = i->next().removeField("v").getOwned();
            const BSONObj key = spec.getObjectField("key");
            const Status keyStatus = validateKeyPattern(key);
            if (!keyStatus.isOK()) {
                errmsg = str::stream()
                         << "Cannot rebuild index " << spec << ": " << keyStatus.reason()
                         << " For more info see http://dochub.mongodb.org/core/index-validation";
                return false;
            }

            b.append( BSONObjBuilder::numStr( all.size() ) , spec );
            all.push_back( spec );
        }
        result.appendNumber( "nIndexesWas", collection->getIndexCatalog()->numIndexesTotal() );

        Status s = collection->getIndexCatalog()->dropAllIndexes(txn, true);
        if ( !s.isOK() ) {
            errmsg = "dropIndexes failed";
            return appendCommandStatus( result, s );
        }

        for ( list<BSONObj>::iterator i=all.begin(); i!=all.end(); i++ ) {
            BSONObj o = *i;
            LOG(1) << "reIndex ns: " << toDeleteNs << " index: " << o << endl;
            Status s = collection->getIndexCatalog()->createIndex(txn, o, false);
            if ( !s.isOK() )
                return appendCommandStatus( result, s );
        }

        result.append( "nIndexes" , (int)all.size() );
        result.appendArray( "indexes" , b.obj() );

        IndexBuilder::restoreIndexes(indexesInProg);
        return true;
    }
예제 #20
0
            bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) {

                if ( ! okForConfigChanges( errmsg ) )
                    return false;

                ShardConnection::sync();

                string ns = cmdObj.firstElement().valuestrsafe();
                if ( ns.size() == 0 ) {
                    errmsg = "no ns";
                    return false;
                }

                DBConfigPtr config = grid.getDBConfig( ns );
                if ( ! config->isSharded( ns ) ) {
                    config->reload();
                    if ( ! config->isSharded( ns ) ) {
                        errmsg = "ns not sharded.  have to shard before can split";
                        return false;
                    }
                }

                BSONObj find = cmdObj.getObjectField( "find" );
                if ( find.isEmpty() ) {
                    find = cmdObj.getObjectField( "middle" );

                    if ( find.isEmpty() ) {
                        errmsg = "need to specify find or middle";
                        return false;
                    }
                }

                ChunkManagerPtr info = config->getChunkManager( ns );
                ChunkPtr chunk = info->findChunk( find );
                BSONObj middle = cmdObj.getObjectField( "middle" );

                assert( chunk.get() );
                log() << "splitting: " << ns << "  shard: " << chunk << endl;

                BSONObj res;
                bool worked;
                if ( middle.isEmpty() ) {
                    BSONObj ret = chunk->singleSplit( true /* force a split even if not enough data */ , res );
                    worked = !ret.isEmpty();
                }
                else {
                    // sanity check if the key provided is a valid split point
                    if ( ( middle == chunk->getMin() ) || ( middle == chunk->getMax() ) ) {
                        errmsg = "cannot split on initial or final chunk's key";
                        return false;
                    }

                    if (!fieldsMatch(middle, info->getShardKey().key())){
                        errmsg = "middle has different fields (or different order) than shard key";
                        return false;
                    }

                    vector<BSONObj> splitPoints;
                    splitPoints.push_back( middle );
                    worked = chunk->multiSplit( splitPoints , res );
                }

                if ( !worked ) {
                    errmsg = "split failed";
                    result.append( "cause" , res );
                    return false;
                }
                config->getChunkManager( ns , true );
                return true;
            }
예제 #21
0
namespace engine
{

   #define IXM_MAX_PREALLOCATED_UNDEFKEY        ( 10 )
   /*
      IXM Tool functions
   */
   BSONObj ixmGetUndefineKeyObj( INT32 fieldNum )
   {
      static BSONObj s_undefineKeys[ IXM_MAX_PREALLOCATED_UNDEFKEY ] ;
      static BOOLEAN s_init = FALSE ;
      static ossSpinXLatch s_latch ;

      if ( FALSE == s_init )
      {
         s_latch.get() ;
         if ( FALSE == s_init )
         {
            for ( SINT32 i = 0; i < IXM_MAX_PREALLOCATED_UNDEFKEY ; ++i )
            {
               BSONObjBuilder b ;
               for ( SINT32 j = 0; j <= i; ++j )
               {
                  b.appendUndefined("") ;
               }
               s_undefineKeys[i] = b.obj() ;
            }
            s_init = TRUE ;
         }
         s_latch.release() ;
      }

      if ( fieldNum > 0 && fieldNum <= IXM_MAX_PREALLOCATED_UNDEFKEY )
      {
         return s_undefineKeys[ fieldNum - 1 ] ;
      }
      else
      {
         BSONObjBuilder b ;
         for ( INT32 i = 0; i < fieldNum; ++i )
         {
            b.appendUndefined("") ;
         }
         return b.obj() ;
      }
   }

   /*
      IXM Global opt var
   */
   const static BSONObj gUndefinedObj =
          BSONObjBuilder().appendUndefined("").obj() ;
   const static BSONElement gUndefinedElt = gUndefinedObj.firstElement() ;

   class _ixmKeyGenerator
   {
   protected:
      const _ixmIndexKeyGen *_keygen ;
      mutable vector<BSONObj *> _objs ;
   public:
      _ixmKeyGenerator ( const _ixmIndexKeyGen *keygen )
      {
         _keygen = keygen ;
      }
      ~_ixmKeyGenerator()
      {
         vector<BSONObj *>::iterator itr = _objs.begin() ;
         for ( ; itr != _objs.end(); itr++ )
         {
            SDB_OSS_DEL *itr ;
         }
      }
      // PD_TRACE_DECLARE_FUNCTION ( SDB__IXMKEYGEN_GETKEYS, "_ixmKeyGenerator::getKeys" )
      INT32 getKeys ( const BSONObj &obj, BSONObjSet &keys,
                      BSONElement *pArrEle ) const
      {
         INT32 rc = SDB_OK ;
         PD_TRACE_ENTRY ( SDB__IXMKEYGEN_GETKEYS );
         SDB_ASSERT( _keygen, "spec can't be NULL" ) ;
         SDB_ASSERT( !_keygen->_fieldNames.empty(), "can not be empty" ) ;
         vector<const CHAR*> fieldNames ( _keygen->_fieldNames ) ;
         BSONElement arrEle ;
         try
         {
            rc = _getKeys( fieldNames, obj, keys, &arrEle ) ;
         }
         catch ( std::exception &e )
         {
            PD_LOG( PDERROR, "unexpected err:%s", e.what() ) ;
            rc = SDB_INVALIDARG ;
            goto error ;
         }

         if ( SDB_OK != rc )
         {
            PD_LOG ( PDERROR, "Failed to generate key from object: %s",
                     obj.toString().c_str() ) ;
            goto error ;
         }

         if ( keys.empty() )
         {
            keys.insert ( _keygen->_undefinedKey ) ;
         }

         if ( NULL != pArrEle && !arrEle.eoo() )
         {
            *pArrEle = arrEle ;
         }
      done :
         PD_TRACE_EXITRC ( SDB__IXMKEYGEN_GETKEYS, rc );
         return rc ;
      error :
         goto done ;
      }
   protected:
      // PD_TRACE_DECLARE_FUNCTION ( SDB__IXMKEYGEN__GETKEYS, "_ixmKeyGenerator::_getKeys" )
      INT32 _getKeys( vector<const CHAR *> &fieldNames,
                      const BSONObj &obj,
                      BSONObjSet &keys,
                      BSONElement *arrEle ) const
      {
         INT32 rc = SDB_OK ;
         PD_TRACE_ENTRY ( SDB__IXMKEYGEN__GETKEYS );
#define IXM_DEFAULT_FIELD_NUM 3
         BSONElement eleOnStack[IXM_DEFAULT_FIELD_NUM] ;
         BSONElement *keyEles = NULL ;
         const CHAR *arrEleName = NULL ;
         UINT32 arrElePos = 0 ;
         UINT32 eooNum = 0 ;

         if ( IXM_DEFAULT_FIELD_NUM < fieldNames.size() )
         {
            keyEles = new(std::nothrow) BSONElement[fieldNames.size()] ;
            if ( NULL == keyEles )
            {
               PD_LOG( PDERROR, "failed to allocalte mem." ) ;
               rc = SDB_OOM ;
               goto error ;
            }
         }
         else
         {
            keyEles = ( BSONElement* )eleOnStack ;
         }

         for ( UINT32 i = 0; i < fieldNames.size(); i++ )
         {
            const CHAR *name = fieldNames.at( i ) ;
            SDB_ASSERT( '\0' != name[0], "can not be empty" ) ;
            BSONElement &e = keyEles[i] ;
            e = obj.getFieldDottedOrArray( name ) ;
            if ( e.eoo() )
            {
               ++eooNum ;
               continue ;
            }
            else if ( Array == e.type() )
            {
               if ( !arrEle->eoo() )
               {
                  PD_LOG( PDERROR, "At most one array can be in the key:",
                          arrEle->fieldName(), e.fieldName() ) ;
                  rc = SDB_IXM_MULTIPLE_ARRAY ;
                  goto error ;
               }
               else
               {
                  *arrEle = e ;
                  arrEleName = name ;
                  arrElePos = i ;
               }
            }
            else
            {
               continue ;
            }
         }

         if ( fieldNames.size() == eooNum )
         {
            rc = SDB_OK ;
            goto done ;
         }
         else if ( !arrEle->eoo() )
         {
            rc = _genKeyWithArrayEle( keyEles, fieldNames.size(),
                                      arrEle,
                                      arrEleName, arrElePos,
                                      keys ) ;
            if ( SDB_OK != rc )
            {
               PD_LOG( PDERROR, "failed to gen keys with array element:%d", rc ) ;
               goto error ;
            }
         }
         else
         {
            rc = _genKeyWithNormalEle( keyEles, fieldNames.size(), keys ) ;
            if ( SDB_OK != rc )
            {
               PD_LOG( PDERROR, "failed to gen keys with normal element:%d", rc ) ;
               goto error ;
            }
         }
      done:
         if ( IXM_DEFAULT_FIELD_NUM < fieldNames.size() &&
              NULL != keyEles )
         {
            delete []keyEles ;
         }
         PD_TRACE_EXITRC ( SDB__IXMKEYGEN__GETKEYS, rc );
         return rc ;
      error:
         goto done ;
      }

      // PD_TRACE_DECLARE_FUNCTION ( SDB__IXMKEYGEN__GENKEYSWITHARRELE, "_ixmKeyGenerator::_genKeyWithArrayEle" )
      INT32 _genKeyWithArrayEle( BSONElement *keyEles,
                                 UINT32 eleNum,
                                 const BSONElement *arrElement,
                                 const CHAR *arrEleName,
                                 UINT32 arrElePos,
                                 BSONObjSet &keys ) const
      {
         PD_TRACE_ENTRY ( SDB__IXMKEYGEN__GENKEYSWITHARRELE );
         INT32 rc = SDB_OK ;
         BSONObj arrObj = arrElement->embeddedObject() ;
         if ( arrObj.firstElement().eoo() )
         {
            keyEles[arrElePos] = *arrElement ;
            rc = _genKeyWithNormalEle( keyEles, eleNum, keys ) ;
            if ( SDB_OK != rc )
            {
               goto error ;
            }
         }

         if ( '\0' == *arrEleName )
         {
            BSONObjIterator itr( arrObj ) ;
            BSONElement &e = keyEles[arrElePos] ;
            while ( itr.more() )
            {
               e = itr.next() ;
               rc = _genKeyWithNormalEle( keyEles, eleNum, keys ) ;
               if ( SDB_OK != rc )
               {
                  goto error ;
               }
            }
         }
         else
         {
            BSONObjIterator itr( arrObj ) ;
            while ( itr.more() )
            {
               const CHAR *dottedName = arrEleName ;
               BSONElement next = itr.next() ;
               if ( Object == next.type() )
               {
                  BSONElement e =
                     next.embeddedObject()
                     .getFieldDottedOrArray( dottedName ) ;
                  if ( Array == e.type() )
                  {
                     rc = _genKeyWithArrayEle(keyEles, eleNum,
                                              &e,
                                              dottedName, arrElePos,
                                              keys) ;
                     if ( SDB_OK != rc )
                     {
                        goto error ;
                     }
                     else
                     {
                        continue ;
                     }
                  }
                  else
                  {
                     keyEles[arrElePos] = e ;
                  }
               }
               else
               {
                  keyEles[arrElePos] = BSONElement() ;
               }

               rc = _genKeyWithNormalEle( keyEles, eleNum, keys ) ;
               if ( SDB_OK != rc )
               {
                  goto error ;
               }
            } 
         }
      done:
         PD_TRACE_EXITRC( SDB__IXMKEYGEN__GENKEYSWITHARRELE, rc ) ;
         return rc ;
      error:
         goto done ;
      }

      // PD_TRACE_DECLARE_FUNCTION ( SDB__IXMKEYGEN__GENKEYSWITHNORMALELE, "_ixmKeyGenerator::_genKeyWithNormalEle" )
      INT32 _genKeyWithNormalEle( BSONElement *keyELes,
                                  UINT32 eleNum,
                                  BSONObjSet &keys ) const
      {
         PD_TRACE_ENTRY ( SDB__IXMKEYGEN__GENKEYSWITHNORMALELE );
         INT32 rc = SDB_OK ;
         BSONObjBuilder builder ;
         for ( UINT32 i = 0; i < eleNum; i++ )
         {
            BSONElement &e = keyELes[i] ;
            if ( e.eoo() )
            {
               builder.appendAs( gUndefinedElt, "" ) ;
            }
            else
            {
               builder.appendAs( e, "" ) ;
            }
         }

         keys.insert( builder.obj() ) ;
         PD_TRACE_EXITRC ( SDB__IXMKEYGEN__GENKEYSWITHNORMALELE, rc );
         return rc ;
      }
   } ;
   typedef class _ixmKeyGenerator ixmKeyGenerator ;
   _ixmIndexKeyGen::_ixmIndexKeyGen ( const _ixmIndexCB *indexCB,
                                      IXM_KEYGEN_TYPE genType )
   {
      SDB_ASSERT ( indexCB, "details can't be NULL" ) ;
      _keyPattern = indexCB->keyPattern() ;
      _info = indexCB->_infoObj ;
      _type = indexCB->getIndexType() ;
      _keyGenType = genType ;
      _init() ;
   }
   _ixmIndexKeyGen::_ixmIndexKeyGen ( const BSONObj &keyDef,
                                      IXM_KEYGEN_TYPE genType )
   {
      _keyPattern = keyDef.copy () ;
      _type = IXM_EXTENT_TYPE_NONE ;
      _keyGenType = genType ;
      _init () ;
   }

   // PD_TRACE_DECLARE_FUNCTION ( SDB__IXMINXKEYGEN__INIT, "_ixmIndexKeyGen::_init" )
   void _ixmIndexKeyGen::_init()
   {
      PD_TRACE_ENTRY ( SDB__IXMINXKEYGEN__INIT );
      _nFields = _keyPattern.nFields () ;
      INT32 fieldNum = 0 ;
      {
         BSONObjIterator i(_keyPattern) ;
         while ( i.more())
         {
            BSONElement e = i.next() ;
            _fieldNames.push_back(e.fieldName()) ;
            _fixedElements.push_back(BSONElement()) ;
            ++fieldNum ;
         }
         _undefinedKey = ixmGetUndefineKeyObj( fieldNum ) ;
      }
      PD_TRACE_EXIT ( SDB__IXMINXKEYGEN__INIT );
   }

   INT32 _ixmIndexKeyGen::getKeys ( const BSONObj &obj, BSONObjSet &keys,
                                    BSONElement *pArrEle ) const
   {
      ixmKeyGenerator g (this) ;
      if ( pArrEle )
      {
         *pArrEle = BSONElement() ;
      }
      return g.getKeys ( obj, keys, pArrEle ) ;
   }

   static BOOLEAN anyElementNamesMatch( const BSONObj& a , const BSONObj& b )
   {
      BSONObjIterator x(a);
      while ( x.more() )
      {
         BSONElement e = x.next();
         BSONObjIterator y(b);
         while ( y.more() )
         {
            BSONElement f = y.next();
            FieldCompareResult res = compareDottedFieldNames( e.fieldName(),
                                                              f.fieldName()
                                                            ) ;
            if ( res == SAME || res == LEFT_SUBFIELD || res == RIGHT_SUBFIELD )
               return TRUE;
         }
      }
      return FALSE;
   }
   IndexSuitability ixmIndexKeyGen::suitability( const BSONObj &query ,
                                                 const BSONObj &order ) const
   {
      return _suitability( query , order );
   }

   IndexSuitability ixmIndexKeyGen::_suitability( const BSONObj& query ,
                                                  const BSONObj& order ) const
   {
       if ( anyElementNamesMatch( _keyPattern , query ) == 0 &&
            anyElementNamesMatch( _keyPattern , order ) == 0 )
          return USELESS;
       return HELPFUL;
   }
   
   // PD_TRACE_DECLARE_FUNCTION ( SDB_IXMINXKEYGEN, "ixmIndexKeyGen::reset" )
   INT32 ixmIndexKeyGen::reset ( const BSONObj & info )
   {
      INT32 rc = SDB_OK ;
      PD_TRACE_ENTRY ( SDB_IXMINXKEYGEN );
      _info = info ;
      try
      {
         _keyPattern = _info[IXM_KEY_FIELD].embeddedObjectUserCheck() ;
      }
      catch ( std::exception &e )
      {
         PD_LOG ( PDERROR, "Unable to locate valid key in index: %s",
                  e.what() ) ;
         rc = SDB_INVALIDARG ;
         goto error ;
      }
      if ( _keyPattern.objsize() == 0 )
      {
         PD_LOG ( PDERROR, "Empty key" ) ;
         rc = SDB_INVALIDARG ;
         goto error ;
      }
      _init() ;
   done :
      PD_TRACE_EXITRC ( SDB_IXMINXKEYGEN, rc );
      return rc ;
   error :
      goto done ;
   }
   INT32 ixmIndexKeyGen::reset ( const _ixmIndexCB *indexCB )
   {
      SDB_ASSERT ( indexCB, "details can't be NULL" ) ;
      return reset ( indexCB->_infoObj ) ;
   }
   BSONElement ixmIndexKeyGen::missingField() const
   {
      return gUndefinedElt ;
   }

   BOOLEAN _ixmIndexKeyGen::validateKeyDef ( const BSONObj &keyDef )
   {
      BSONObjIterator i ( keyDef ) ;
      INT32 count = 0 ;
      while ( i.more () )
      {
         ++count ;
         BSONElement ie = i.next () ;
         if ( ie.type() != NumberInt ||
              ( ie.numberInt() != -1 &&
                ie.numberInt() != 1 ) )
         {
            return FALSE ;
         }
      }
      return 0 != count ;
   }
}
예제 #22
0
            bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) {

                if ( ! okForConfigChanges( errmsg ) )
                    return false;

                ShardConnection::sync();

                Timer t;
                string ns = cmdObj.firstElement().valuestrsafe();
                if ( ns.size() == 0 ) {
                    errmsg = "no ns";
                    return false;
                }

                DBConfigPtr config = grid.getDBConfig( ns );
                if ( ! config->isSharded( ns ) ) {
                    config->reload();
                    if ( ! config->isSharded( ns ) ) {
                        errmsg = "ns not sharded.  have to shard before we can move a chunk";
                        return false;
                    }
                }

                BSONObj find = cmdObj.getObjectField( "find" );
                if ( find.isEmpty() ) {
                    errmsg = "need to specify find.  see help";
                    return false;
                }

                string toString = cmdObj["to"].valuestrsafe();
                if ( ! toString.size()  ) {
                    errmsg = "you have to specify where you want to move the chunk";
                    return false;
                }

                Shard to = Shard::make( toString );

                // so far, chunk size serves test purposes; it may or may not become a supported parameter
                long long maxChunkSizeBytes = cmdObj["maxChunkSizeBytes"].numberLong();
                if ( maxChunkSizeBytes == 0 ) {
                    maxChunkSizeBytes = Chunk::MaxChunkSize;
                }

                tlog() << "CMD: movechunk: " << cmdObj << endl;

                ChunkManagerPtr info = config->getChunkManager( ns );
                ChunkPtr c = info->findChunk( find );
                const Shard& from = c->getShard();

                if ( from == to ) {
                    errmsg = "that chunk is already on that shard";
                    return false;
                }

                BSONObj res;
                if ( ! c->moveAndCommit( to , maxChunkSizeBytes , res ) ) {
                    errmsg = "move failed";
                    result.append( "cause" , res );
                    return false;
                }
                
                // preemptively reload the config to get new version info
                config->getChunkManager( ns , true );

                result.append( "millis" , t.millis() );
                return true;
            }
    // static
    void ExpressionKeysPrivate::get2DKeys(const BSONObj &obj,
                                         const TwoDIndexingParams& params,
                                         BSONObjSet* keys,
                                         std::vector<BSONObj>* locs) {
        BSONElementMSet bSet;

        // Get all the nested location fields, but don't return individual elements from
        // the last array, if it exists.
        obj.getFieldsDotted(params.geo.c_str(), bSet, false);

        if (bSet.empty())
            return;

        for (BSONElementMSet::iterator setI = bSet.begin(); setI != bSet.end(); ++setI) {
            BSONElement geo = *setI;

            if (geo.eoo() || !geo.isABSONObj())
                continue;

            //
            // Grammar for location lookup:
            // locs ::= [loc,loc,...,loc]|{<k>:loc,<k>:loc,...,<k>:loc}|loc
            // loc  ::= { <k1> : #, <k2> : # }|[#, #]|{}
            //
            // Empty locations are ignored, preserving single-location semantics
            //

            BSONObj embed = geo.embeddedObject();
            if (embed.isEmpty())
                continue;

            // Differentiate between location arrays and locations
            // by seeing if the first element value is a number
            bool singleElement = embed.firstElement().isNumber();

            BSONObjIterator oi(embed);

            while (oi.more()) {
                BSONObj locObj;

                if (singleElement) {
                    locObj = embed;
                } else {
                    BSONElement locElement = oi.next();

                    uassert(16804, mongoutils::str::stream() <<
                            "location object expected, location array not in correct format",
                            locElement.isABSONObj());

                    locObj = locElement.embeddedObject();
                    if(locObj.isEmpty())
                        continue;
                }

                BSONObjBuilder b(64);

                // Remember the actual location object if needed
                if (locs)
                    locs->push_back(locObj);

                // Stop if we don't need to get anything but location objects
                if (!keys) {
                    if (singleElement) break;
                    else continue;
                }

                params.geoHashConverter->hash(locObj, &obj).appendHashMin(&b, "");

                // Go through all the other index keys
                for (vector<pair<string, int> >::const_iterator i = params.other.begin();
                     i != params.other.end(); ++i) {
                    // Get *all* fields for the index key
                    BSONElementSet eSet;
                    obj.getFieldsDotted(i->first, eSet);

                    if (eSet.size() == 0)
                        b.appendNull("");
                    else if (eSet.size() == 1)
                        b.appendAs(*(eSet.begin()), "");
                    else {
                        // If we have more than one key, store as an array of the objects
                        BSONArrayBuilder aBuilder;

                        for (BSONElementSet::iterator ei = eSet.begin(); ei != eSet.end();
                             ++ei) {
                            aBuilder.append(*ei);
                        }

                        b.append("", aBuilder.arr());
                    }
                }
                keys->insert(b.obj());
                if(singleElement) break;
            }
        }
    }
예제 #24
0
            bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) {
                string target = cmdObj.firstElement().valuestrsafe();
                Shard s = Shard::make( target );
                if ( ! grid.knowAboutShard( s.getConnString() ) ) {
                    errmsg = "unknown shard";
                    return false;
                }

                ScopedDbConnection conn( configServer.getPrimary() );

                // If the server is not yet draining chunks, put it in draining mode.
                BSONObj searchDoc = BSON( "_id" << s.getName() );
                BSONObj drainingDoc = BSON( "_id" << s.getName() << ShardFields::draining(true) );
                BSONObj shardDoc = conn->findOne( "config.shards", drainingDoc );
                if ( shardDoc.isEmpty() ) {

                    // TODO prevent move chunks to this shard.

                    log() << "going to start draining shard: " << s.getName() << endl;
                    BSONObj newStatus = BSON( "$set" << BSON( ShardFields::draining(true) ) );
                    conn->update( "config.shards" , searchDoc , newStatus, false /* do no upsert */);

                    errmsg = conn->getLastError();
                    if ( errmsg.size() ) {
                        log() << "error starting remove shard: " << s.getName() << " err: " << errmsg << endl;
                        return false;
                    }

                    Shard::reloadShardInfo();

                    result.append( "msg"   , "draining started successfully" );
                    result.append( "state" , "started" );
                    result.append( "shard" , s.getName() );
                    conn.done();
                    return true;
                }

                // If the server has been completely drained, remove it from the ConfigDB.
                // Check not only for chunks but also databases.
                BSONObj shardIDDoc = BSON( "shard" << shardDoc[ "_id" ].str() );
                long long chunkCount = conn->count( "config.chunks" , shardIDDoc );
                BSONObj primaryDoc = BSON( "primary" << shardDoc[ "_id" ].str() );
                long long dbCount = conn->count( "config.databases" , primaryDoc );
                if ( ( chunkCount == 0 ) && ( dbCount == 0 ) ) {
                    log() << "going to remove shard: " << s.getName() << endl;
                    conn->remove( "config.shards" , searchDoc );

                    errmsg = conn->getLastError();
                    if ( errmsg.size() ) {
                        log() << "error concluding remove shard: " << s.getName() << " err: " << errmsg << endl;
                        return false;
                    }

                    Shard::removeShard( shardDoc[ "_id" ].str() );
                    Shard::reloadShardInfo();

                    result.append( "msg"   , "removeshard completed successfully" );
                    result.append( "state" , "completed" );
                    result.append( "shard" , s.getName() );
                    conn.done();
                    return true;
                }

                // If the server is already in draining mode, just report on its progress.
                // Report on databases (not just chunks) that are left too.
                result.append( "msg"  , "draining ongoing" );
                result.append( "state" , "ongoing" );
                BSONObjBuilder inner;
                inner.append( "chunks" , chunkCount );
                inner.append( "dbs" , dbCount );
                result.append( "remaining" , inner.obj() );

                conn.done();
                return true;
            }
예제 #25
0
        void run() {
            Scope * s = globalScriptEngine->newScope();
            s->localConnect( "blah" );
            BSONObjBuilder b;
            long long val = (long long)( 0xbabadeadbeefbaddULL );
            b.append( "a", val );
            BSONObj in = b.obj();
            s->setObject( "a", in );
            BSONObj out = s->getObject( "a" );
            ASSERT_EQUALS( mongo::NumberLong, out.firstElement().type() );
            
            ASSERT( s->exec( "printjson( a ); b = {b:a.a}", "foo", false, true, false ) );
            out = s->getObject( "b" );
            ASSERT_EQUALS( mongo::NumberLong, out.firstElement().type() );
            if( val != out.firstElement().numberLong() ) {
                cout << val << endl;
                cout << out.firstElement().numberLong() << endl;
                cout << out.toString() << endl;
                ASSERT_EQUALS( val, out.firstElement().numberLong() );
            }
            
            ASSERT( s->exec( "c = {c:a.a.toString()}", "foo", false, true, false ) );
            out = s->getObject( "c" );
            stringstream ss;
            ss << "NumberLong(\"" << val << "\")";
            ASSERT_EQUALS( ss.str(), out.firstElement().valuestr() );

            ASSERT( s->exec( "d = {d:a.a.toNumber()}", "foo", false, true, false ) );
            out = s->getObject( "d" );
            ASSERT_EQUALS( NumberDouble, out.firstElement().type() );
            ASSERT_EQUALS( double( val ), out.firstElement().number() );

            ASSERT( s->exec( "e = {e:a.a.floatApprox}", "foo", false, true, false ) );
            out = s->getObject( "e" );
            ASSERT_EQUALS( NumberDouble, out.firstElement().type() );
            ASSERT_EQUALS( double( val ), out.firstElement().number() );     

            ASSERT( s->exec( "f = {f:a.a.top}", "foo", false, true, false ) );
            out = s->getObject( "f" );
            ASSERT( NumberDouble == out.firstElement().type() || NumberInt == out.firstElement().type() );
            
            s->setObject( "z", BSON( "z" << (long long)( 4 ) ) );
            ASSERT( s->exec( "y = {y:z.z.top}", "foo", false, true, false ) );
            out = s->getObject( "y" );
            ASSERT_EQUALS( Undefined, out.firstElement().type() );

            ASSERT( s->exec( "x = {x:z.z.floatApprox}", "foo", false, true, false ) );
            out = s->getObject( "x" );
            ASSERT( NumberDouble == out.firstElement().type() || NumberInt == out.firstElement().type() );
            ASSERT_EQUALS( double( 4 ), out.firstElement().number() );     

            ASSERT( s->exec( "w = {w:z.z}", "foo", false, true, false ) );
            out = s->getObject( "w" );
            ASSERT_EQUALS( mongo::NumberLong, out.firstElement().type() );
            ASSERT_EQUALS( 4, out.firstElement().numberLong() );     
            
        }
예제 #26
0
        ProgramRunner::ProgramRunner( const BSONObj &args ) {
            verify( !args.isEmpty() );

            string program( args.firstElement().valuestrsafe() );
            verify( !program.empty() );
            boost::filesystem::path programPath = findProgram(program);

            string prefix( "mongod-" );
            bool isMongodProgram =
                    string("mongod") == program ||
                    program.compare( 0, prefix.size(), prefix ) == 0;

            prefix = "mongos-";
            bool isMongosProgram =
                    string("mongos") == program ||
                    program.compare( 0, prefix.size(), prefix ) == 0;

#if 0
            if (isMongosProgram == "mongos") {
                _argv.push_back("valgrind");
                _argv.push_back("--log-file=/tmp/mongos-%p.valgrind");
                _argv.push_back("--leak-check=yes");
                _argv.push_back("--suppressions=valgrind.suppressions");
                //_argv.push_back("--error-exitcode=1");
                _argv.push_back("--");
            }
#endif

            _argv.push_back( programPath.string() );

            _port = -1;

            BSONObjIterator j( args );
            j.next(); // skip program name (handled above)
            while(j.more()) {
                BSONElement e = j.next();
                string str;
                if ( e.isNumber() ) {
                    stringstream ss;
                    ss << e.number();
                    str = ss.str();
                }
                else {
                    verify( e.type() == mongo::String );
                    str = e.valuestr();
                }
                if ( str == "--port" )
                    _port = -2;
                else if ( _port == -2 )
                    _port = strtol( str.c_str(), 0, 10 );
                _argv.push_back(str);
            }

            if ( ! isMongodProgram && ! isMongosProgram && program != "mongobridge" )
                _port = 0;
            else {
                if ( _port <= 0 )
                    log() << "error: a port number is expected when running " << program << " from the shell" << endl;
                verify( _port > 0 );
            }
            if ( _port > 0 ) {
                bool haveDbForPort = registry.isPortRegistered( _port );
                if ( haveDbForPort ) {
                    log() << "already have db for port: " << _port << endl;
                    verify( !haveDbForPort );
                }
            }
        }
예제 #27
0
파일: mr.cpp 프로젝트: DumaGit/mongo
        Config::Config( const string& _dbname , const BSONObj& cmdObj ) {

            dbname = _dbname;
            ns = dbname + "." + cmdObj.firstElement().valuestr();

            verbose = cmdObj["verbose"].trueValue();
            jsMode = cmdObj["jsMode"].trueValue();

            jsMaxKeys = 500000;
            reduceTriggerRatio = 2.0;
            maxInMemSize = 5 * 1024 * 1024;

            uassert( 13602 , "outType is no longer a valid option" , cmdObj["outType"].eoo() );

            if ( cmdObj["out"].type() == String ) {
                finalShort = cmdObj["out"].String();
                outType = REPLACE;
            }
            else if ( cmdObj["out"].type() == Object ) {
                BSONObj o = cmdObj["out"].embeddedObject();

                BSONElement e = o.firstElement();
                string t = e.fieldName();

                if ( t == "normal" || t == "replace" ) {
                    outType = REPLACE;
                    finalShort = e.String();
                }
                else if ( t == "merge" ) {
                    outType = MERGE;
                    finalShort = e.String();
                }
                else if ( t == "reduce" ) {
                    outType = REDUCE;
                    finalShort = e.String();
                }
                else if ( t == "inline" ) {
                    outType = INMEMORY;
                }
                else {
                    uasserted( 13522 , str::stream() << "unknown out specifier [" << t << "]" );
                }

                if (o.hasElement("db")) {
                    outDB = o["db"].String();
                }
            }
            else {
                uasserted( 13606 , "'out' has to be a string or an object" );
            }

            if ( outType != INMEMORY ) { // setup names
                tempLong = str::stream() << (outDB.empty() ? dbname : outDB) << ".tmp.mr." << cmdObj.firstElement().String() << "_" << JOB_NUMBER++;

                incLong = tempLong + "_inc";

                finalLong = str::stream() << (outDB.empty() ? dbname : outDB) << "." << finalShort;
            }

            {
                // scope and code

                if ( cmdObj["scope"].type() == Object )
                    scopeSetup = cmdObj["scope"].embeddedObjectUserCheck();

                mapper.reset( new JSMapper( cmdObj["map"] ) );
                reducer.reset( new JSReducer( cmdObj["reduce"] ) );
                if ( cmdObj["finalize"].type() && cmdObj["finalize"].trueValue() )
                    finalizer.reset( new JSFinalizer( cmdObj["finalize"] ) );

                if ( cmdObj["mapparams"].type() == Array ) {
                    mapParams = cmdObj["mapparams"].embeddedObjectUserCheck();
                }

            }

            {
                // query options
                BSONElement q = cmdObj["query"];
                if ( q.type() == Object )
                    filter = q.embeddedObjectUserCheck();
                else
                    uassert( 13608 , "query has to be blank or an Object" , ! q.trueValue() );


                BSONElement s = cmdObj["sort"];
                if ( s.type() == Object )
                    sort = s.embeddedObjectUserCheck();
                else
                    uassert( 13609 , "sort has to be blank or an Object" , ! s.trueValue() );

                if ( cmdObj["limit"].isNumber() )
                    limit = cmdObj["limit"].numberLong();
                else
                    limit = 0;
            }
        }
예제 #28
0
Status UpdateDriver::parse(const BSONObj& updateExpr, const bool multi) {
    clear();

    // Check if the update expression is a full object replacement.
    if (*updateExpr.firstElementFieldName() != '$') {
        if (multi) {
            return Status(ErrorCodes::FailedToParse, "multi update only works with $ operators");
        }

        // Modifiers expect BSONElements as input. But the input to object replace is, by
        // definition, an object. We wrap the 'updateExpr' as the mod is expecting. Note
        // that the wrapper is temporary so the object replace mod should make a copy of
        // the object.
        unique_ptr<ModifierObjectReplace> mod(new ModifierObjectReplace);
        BSONObj wrapper = BSON("dummy" << updateExpr);
        Status status = mod->init(wrapper.firstElement(), _modOptions);
        if (!status.isOK()) {
            return status;
        }

        _mods.push_back(mod.release());

        // Register the fact that this driver will only do full object replacements.
        _replacementMode = true;

        return Status::OK();
    }

    // The update expression is made of mod operators, that is
    // { <$mod>: {...}, <$mod>: {...}, ...  }
    BSONObjIterator outerIter(updateExpr);
    while (outerIter.more()) {
        BSONElement outerModElem = outerIter.next();

        // Check whether this is a valid mod type.
        modifiertable::ModifierType modType = modifiertable::getType(outerModElem.fieldName());
        if (modType == modifiertable::MOD_UNKNOWN) {
            return Status(ErrorCodes::FailedToParse,
                          str::stream() << "Unknown modifier: " << outerModElem.fieldName());
        }

        // Check whether there is indeed a list of mods under this modifier.
        if (outerModElem.type() != Object) {
            return Status(ErrorCodes::FailedToParse,
                          str::stream() << "Modifiers operate on fields but we found type "
                                        << typeName(outerModElem.type())
                                        << " instead. For example: {$mod: {<field>: ...}}"
                                        << " not {"
                                        << outerModElem.toString()
                                        << "}");
        }

        // Check whether there are indeed mods under this modifier.
        if (outerModElem.embeddedObject().isEmpty()) {
            return Status(ErrorCodes::FailedToParse,
                          str::stream() << "'" << outerModElem.fieldName()
                                        << "' is empty. You must specify a field like so: "
                                           "{"
                                        << outerModElem.fieldName()
                                        << ": {<field>: ...}}");
        }

        BSONObjIterator innerIter(outerModElem.embeddedObject());
        while (innerIter.more()) {
            BSONElement innerModElem = innerIter.next();

            Status status = addAndParse(modType, innerModElem);
            if (!status.isOK()) {
                return status;
            }
        }
    }

    // Register the fact that there will be only $mod's in this driver -- no object
    // replacement.
    _replacementMode = false;

    return Status::OK();
}
예제 #29
0
파일: mr.cpp 프로젝트: mapleoin/mongo
            bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){
                string shardedOutputCollection = cmdObj["shardedOutputCollection"].valuestrsafe();

                MRSetup mr( dbname , cmdObj.firstElement().embeddedObjectUserCheck() , false );
                
                set<ServerAndQuery> servers;
                
                BSONObjBuilder shardCounts;
                map<string,long long> counts;
                
                BSONObj shards = cmdObj["shards"].embeddedObjectUserCheck();
                vector< auto_ptr<DBClientCursor> > shardCursors;

                { // parse per shard results 
                    BSONObjIterator i( shards );
                    while ( i.more() ){
                        BSONElement e = i.next();
                        string shard = e.fieldName();
                        
                        BSONObj res = e.embeddedObjectUserCheck();
                        
                        uassert( 10078 ,  "something bad happened" , shardedOutputCollection == res["result"].valuestrsafe() );
                        servers.insert( shard );
                        shardCounts.appendAs( res["counts"] , shard );
                        
                        BSONObjIterator j( res["counts"].embeddedObjectUserCheck() );
                        while ( j.more() ){
                            BSONElement temp = j.next();
                            counts[temp.fieldName()] += temp.numberLong();
                        }
                        
                    }
                    
                }
                
                DBDirectClient db;
                    
                { // reduce from each stream
                    
                    BSONObj sortKey = BSON( "_id" << 1 );
                    
                    ParallelSortClusteredCursor cursor( servers , dbname + "." + shardedOutputCollection ,
                                                        Query().sort( sortKey ) );
                    cursor.init();
                    
                    auto_ptr<Scope> s = globalScriptEngine->getPooledScope( dbname );
                    s->localConnect( dbname.c_str() );
                    ScriptingFunction reduceFunction = s->createFunction( mr.reduceCode.c_str() );
                    ScriptingFunction finalizeFunction = 0;
                    if ( mr.finalizeCode.size() )
                        finalizeFunction = s->createFunction( mr.finalizeCode.c_str() );
                    
                    BSONList values;
                    
                    result.append( "result" , mr.finalShort );
                    
                    while ( cursor.more() ){
                        BSONObj t = cursor.next().getOwned();
                        
                        if ( values.size() == 0 ){
                            values.push_back( t );
                            continue;
                        }
                        
                        if ( t.woSortOrder( *(values.begin()) , sortKey ) == 0 ){
                            values.push_back( t );
                            continue;
                        }
                        
                        
                        db.insert( mr.tempLong , reduceValues( values , s.get() , reduceFunction , 1 , finalizeFunction ) );
                        values.clear();
                        values.push_back( t );
                    }
                    
                    if ( values.size() )
                        db.insert( mr.tempLong , reduceValues( values , s.get() , reduceFunction , 1 , finalizeFunction ) );
                }
                
                long long finalCount = mr.renameIfNeeded( db );
                log(0) << " mapreducefinishcommand " << mr.finalLong << " " << finalCount << endl;

                for ( set<ServerAndQuery>::iterator i=servers.begin(); i!=servers.end(); i++ ){
                    ScopedDbConnection conn( i->_server );
                    conn->dropCollection( dbname + "." + shardedOutputCollection );
                    conn.done();
                }
                
                result.append( "shardCounts" , shardCounts.obj() );
                
                {
                    BSONObjBuilder c;
                    for ( map<string,long long>::iterator i=counts.begin(); i!=counts.end(); i++ ){
                        c.append( i->first , i->second );
                    }
                    result.append( "counts" , c.obj() );
                }

                return 1;
            }
예제 #30
0
파일: cloner.cpp 프로젝트: ahopedog/mongo
        void operator()( DBClientCursorBatchIterator &i ) {
            Lock::GlobalWrite lk;
            if ( context ) {
                context->relocked();
            }

            while( i.moreInCurrentBatch() ) {
                if ( n % 128 == 127 /*yield some*/ ) {
                    time_t now = time(0);
                    if( now - lastLog >= 60 ) { 
                        // report progress
                        if( lastLog )
                            log() << "clone " << to_collection << ' ' << n << endl;
                        lastLog = now;
                    }
                    mayInterrupt( _mayBeInterrupted );
                    dbtempreleaseif t( _mayYield );
                }

                BSONObj tmp = i.nextSafe();

                /* assure object is valid.  note this will slow us down a little. */
                if ( !tmp.valid() ) {
                    stringstream ss;
                    ss << "Cloner: skipping corrupt object from " << from_collection;
                    BSONElement e = tmp.firstElement();
                    try {
                        e.validate();
                        ss << " firstElement: " << e;
                    }
                    catch( ... ) {
                        ss << " firstElement corrupt";
                    }
                    out() << ss.str() << endl;
                    continue;
                }

                ++n;

                BSONObj js = tmp;
                if ( isindex ) {
                    verify( strstr(from_collection, "system.indexes") );
                    js = fixindex(tmp);
                    storedForLater->push_back( js.getOwned() );
                    continue;
                }

                try {
                    // add keys for presorting
                    DiskLoc loc = theDataFileMgr.insertWithObjMod(to_collection, js);
                    loc.assertOk();
                    if (_sortersForIndex != NULL) {
                        // add key to SortersForNS
                        for (SortersForIndex::iterator iSorter = _sortersForIndex->begin();
                             iSorter != _sortersForIndex->end();
                             ++iSorter) {
                            iSorter->second.preSortPhase.addKeys(iSorter->second.spec, js,
                                                                 loc, false);
                        }
                    }
                    if ( logForRepl )
                        logOp("i", to_collection, js);

                    getDur().commitIfNeeded();
                }
                catch( UserException& e ) {
                    error() << "error: exception cloning object in " << from_collection << ' ' << e.what() << " obj:" << js.toString() << '\n';
                    throw;
                }

                RARELY if ( time( 0 ) - saveLast > 60 ) {
                    log() << n << " objects cloned so far from collection " << from_collection << endl;
                    saveLast = time( 0 );
                }
            }
        }