Example #1
0
        BSONObj stats() {
            if ( _http ) {
                HttpClient c;
                HttpClient::Result r;

                string url;
                {
                    stringstream ss;
                    ss << "http://" << _host;
                    if ( _host.find( ":" ) == string::npos )
                        ss << ":28017";
                    ss << "/_status";
                    url = ss.str();
                }

                if ( c.get( url , &r ) != 200 ) {
                    cout << "error (http): " << r.getEntireResponse() << endl;
                    return BSONObj();
                }

                BSONObj x = fromjson( r.getBody() );
                BSONElement e = x["serverStatus"];
                if ( e.type() != Object ) {
                    cout << "BROKEN: " << x << endl;
                    return BSONObj();
                }
                return e.embeddedObjectUserCheck();
            }
            BSONObj out;
            if ( ! conn().simpleCommand( _db , &out , "serverStatus" ) ) {
                cout << "error: " << out << endl;
                return BSONObj();
            }
            return out.getOwned();
        }
Example #2
0
        void run() {
            if ( _token.size() == 0  && _name.size() == 0 ) {
                log(1) << "mms not configured" << endl;
                return;
            }

            if ( _token.size() == 0 ) {
                log() << "no token for mms - not running" << endl;
                return;
            }

            if ( _name.size() == 0 ) {
                log() << "no name for mms - not running" << endl;
                return;
            }

            log() << "mms monitor staring...  token:" << _token << " name:" << _name << " interval: " << _secsToSleep << endl;
            Client::initThread( "mms" );
            Client& c = cc();


            // TODO: using direct client is bad, but easy for now

            while ( ! inShutdown() ) {
                sleepsecs( _secsToSleep );

                try {
                    stringstream url;
                    url << _baseurl << "?"
                        << "token=" << _token << "&"
                        << "name=" << _name << "&"
                        << "ts=" << time(0)
                        ;

                    BSONObjBuilder bb;
                    // duplicated so the post has everything
                    bb.append( "token" , _token );
                    bb.append( "name" , _name );
                    bb.appendDate( "ts" , jsTime()  );

                    // any commands
                    _add( bb , "buildinfo" );
                    _add( bb , "serverStatus" );

                    BSONObj postData = bb.obj();

                    log(1) << "mms url: " << url.str() << "\n\t post: " << postData << endl;;

                    HttpClient c;
                    HttpClient::Result r;
                    int rc = c.post( url.str() , postData.jsonString() , &r );
                    log(1) << "\t response code: " << rc << endl;
                    if ( rc != 200 ) {
                        log() << "mms error response code:" << rc << endl;
                        log(1) << "mms error body:" << r.getEntireResponse() << endl;
                    }
                }
                catch ( std::exception& e ) {
                    log() << "mms exception: " << e.what() << endl;
                }
            }

            c.shutdown();
        }
Example #3
0
    int run(){
        string filename = getParam( "file" );
		string url = getParam( "url" );
        long long fileSize = -1;

        istream * in = &cin;

		if ( url.size() > 0 && filename.size() > 0 ){
			cerr << "both url and file cannot be specified" << endl;
			return -1;
		}
		
		_root = getParam( "jsonRoot" );
		
		stringstream iss;
		ifstream file( filename.c_str() , ios_base::in );
		
		if ( url.size() > 0 ){
			HttpClient c;
			HttpClient::Result r;
			int rc = c.get( url , &r );
			log(1) << "url fetch response code: " << rc << endl;
			if ( rc != 200 ){
				log() << "url fetch error response code:" << rc << endl;
				log(1) << "url fetch error body:" << r.getEntireResponse() << endl;
				return -1;
			}
			iss << r.getBody();
			in = &iss;
			fileSize = r.getBody().size();
		}
		else if ( filename.size() > 0 && filename != "-" ){
            if ( ! exists( filename ) ){
                cerr << "file doesn't exist: " << filename << endl;
                return -1;
            }
            in = &file;
            fileSize = file_size( filename );
        }

        string ns;

        try {
            ns = getNS();
        } catch (...) {
            printHelp(cerr);
            return -1;
        }
        
        log(1) << "ns: " << ns << endl;
        
        auth();

        if ( hasParam( "drop" ) ){
            cout << "dropping: " << ns << endl;
            conn().dropCollection( ns.c_str() );
        }

        if ( hasParam( "ignoreBlanks" ) ){
            _ignoreBlanks = true;
        }

        if ( hasParam( "upsert" ) ){
            _upsert = true;

            string uf = getParam("upsertFields");
            if (uf.empty()){
                _upsertFields.push_back("_id");
            } else {
                StringSplitter(uf.c_str(), ",").split(_upsertFields);
            }
        }

        if ( hasParam( "noimport" ) ){
            _doimport = false;
        }

        if ( hasParam( "type" ) ){
            string type = getParam( "type" );
            if ( type == "json" )
                _type = JSON;
            else if ( type == "csv" ){
                _type = CSV;
                _sep = ",";
            }
            else if ( type == "tsv" ){
                _type = TSV;
                _sep = "\t";
            }
            else {
                cerr << "don't know what type [" << type << "] is" << endl;
                return -1;
            }
        }
        
        if ( _type == CSV || _type == TSV ){
            _headerLine = hasParam( "headerline" );
            if ( ! _headerLine )
                needFields();
        }

        if (_type == JSON && hasParam("jsonArray")){
            _jsonArray = true;
        }

        int errors = 0;
        
        int num = 0;
        
        time_t start = time(0);

        log(1) << "filesize: " << fileSize << endl;
        ProgressMeter pm( fileSize );
        const int BUF_SIZE = 1024 * 1024 * 4;
        boost::scoped_array<char> line(new char[BUF_SIZE+2]);
        char * buf = line.get();
        while ( _jsonArray || in->rdstate() == 0 ){
            if (_jsonArray){
                if (buf == line.get()){ //first pass
                    in->read(buf, BUF_SIZE);
                    uassert(13295, "JSONArray file too large", (in->rdstate() & ios_base::eofbit));
                    buf[ in->gcount() ] = '\0';
                }
            } else {
                buf = line.get();
                in->getline( buf , BUF_SIZE );
                log(1) << "got line:" << buf << endl;
            }
            uassert( 10263 ,  "unknown error reading file" ,
                    (!(in->rdstate() & ios_base::badbit)) &&
                    (!(in->rdstate() & ios_base::failbit) || (in->rdstate() & ios_base::eofbit)) );

            int len = 0;
            if (strncmp("\xEF\xBB\xBF", buf, 3) == 0){ // UTF-8 BOM (notepad is stupid)
                buf += 3;
                len += 3;
            }

            if (_jsonArray){
                while (buf[0] != '{' && buf[0] != '\0') {
                    len++;
                    buf++;
                }
                if (buf[0] == '\0')
                    break;
            } else {
                while (isspace( buf[0] )){
                    len++;
                    buf++;
                }
                if (buf[0] == '\0')
                    continue;
                len += strlen( buf );
            }

            try {
				vector<BSONObj> objects;
				
                if (_jsonArray){
                    int jslen;
					objects.push_back( fromjson(buf, &jslen) );
                    len += jslen;
                    buf += jslen;
                } else {
                    parseLine( buf, &objects );
                }

                if ( _headerLine ){
                    _headerLine = false;
                } else if (_doimport) {
					for (vector<BSONObj>::const_iterator b_it=objects.begin(), end=objects.end(); b_it!=end; ++b_it){
						bool doUpsert = _upsert;
						BSONObjBuilder b;
						if (_upsert){
							for (vector<string>::const_iterator it=_upsertFields.begin(), end=_upsertFields.end(); it!=end; ++it){
								BSONElement e = b_it->getFieldDotted(it->c_str());
								if (e.eoo()){
									doUpsert = false;
									break;
								}
								b.appendAs(e, *it);
							}
						}

						if (doUpsert){
							conn().update(ns, Query(b.obj()), *b_it, true);
						} else {
							conn().insert( ns.c_str() , *b_it );
						}
						++num;
					}
                }
            }
            catch ( std::exception& e ){
                cout << "exception:" << e.what() << endl;
                cout << buf << endl;
                errors++;
                
                if (hasParam("stopOnError") || _jsonArray)
                    break;
            }

            if ( pm.hit( len + 1 ) ){
                cout << "\t\t\t" << num << "\t" << ( num / ( time(0) - start ) ) << "/second" << endl;
            }
        }

        cout << "imported " << num << " objects" << endl;

        conn().getLastError();
        
        if ( errors == 0 )
            return 0;
        
        cerr << "encountered " << errors << " error" << ( errors == 1 ? "" : "s" ) << endl;
        return -1;
    }