BSONObj stats() {
if ( _http ) {
HttpClient c;
HttpClient::Result r;
string url;
{
stringstream ss;
ss << "http://" << _host;
if ( _host.find( ":" ) == string::npos )
ss << ":28017";
ss << "/_status";
url = ss.str();
}
if ( c.get( url , &r ) != 200 ) {
cout << "error (http): " << r.getEntireResponse() << endl;
return BSONObj();
}
BSONObj x = fromjson( r.getBody() );
BSONElement e = x["serverStatus"];
if ( e.type() != Object ) {
cout << "BROKEN: " << x << endl;
return BSONObj();
}
return e.embeddedObjectUserCheck();
}
BSONObj out;
if ( ! conn().simpleCommand( _db , &out , "serverStatus" ) ) {
cout << "error: " << out << endl;
return BSONObj();
}
return out.getOwned();
}
int run(){
string filename = getParam( "file" );
string url = getParam( "url" );
long long fileSize = -1;
istream * in = &cin;
if ( url.size() > 0 && filename.size() > 0 ){
cerr << "both url and file cannot be specified" << endl;
return -1;
}
_root = getParam( "jsonRoot" );
stringstream iss;
ifstream file( filename.c_str() , ios_base::in );
if ( url.size() > 0 ){
HttpClient c;
HttpClient::Result r;
int rc = c.get( url , &r );
log(1) << "url fetch response code: " << rc << endl;
if ( rc != 200 ){
log() << "url fetch error response code:" << rc << endl;
log(1) << "url fetch error body:" << r.getEntireResponse() << endl;
return -1;
}
iss << r.getBody();
in = &iss;
fileSize = r.getBody().size();
}
else if ( filename.size() > 0 && filename != "-" ){
if ( ! exists( filename ) ){
cerr << "file doesn't exist: " << filename << endl;
return -1;
}
in = &file;
fileSize = file_size( filename );
}
string ns;
try {
ns = getNS();
} catch (...) {
printHelp(cerr);
return -1;
}
log(1) << "ns: " << ns << endl;
auth();
if ( hasParam( "drop" ) ){
cout << "dropping: " << ns << endl;
conn().dropCollection( ns.c_str() );
}
if ( hasParam( "ignoreBlanks" ) ){
_ignoreBlanks = true;
}
if ( hasParam( "upsert" ) ){
_upsert = true;
string uf = getParam("upsertFields");
if (uf.empty()){
_upsertFields.push_back("_id");
} else {
StringSplitter(uf.c_str(), ",").split(_upsertFields);
}
}
if ( hasParam( "noimport" ) ){
_doimport = false;
}
if ( hasParam( "type" ) ){
string type = getParam( "type" );
if ( type == "json" )
_type = JSON;
else if ( type == "csv" ){
_type = CSV;
_sep = ",";
}
else if ( type == "tsv" ){
_type = TSV;
_sep = "\t";
}
else {
cerr << "don't know what type [" << type << "] is" << endl;
return -1;
}
}
if ( _type == CSV || _type == TSV ){
_headerLine = hasParam( "headerline" );
if ( ! _headerLine )
needFields();
}
if (_type == JSON && hasParam("jsonArray")){
_jsonArray = true;
}
int errors = 0;
int num = 0;
time_t start = time(0);
log(1) << "filesize: " << fileSize << endl;
ProgressMeter pm( fileSize );
const int BUF_SIZE = 1024 * 1024 * 4;
boost::scoped_array<char> line(new char[BUF_SIZE+2]);
char * buf = line.get();
while ( _jsonArray || in->rdstate() == 0 ){
if (_jsonArray){
if (buf == line.get()){ //first pass
in->read(buf, BUF_SIZE);
uassert(13295, "JSONArray file too large", (in->rdstate() & ios_base::eofbit));
buf[ in->gcount() ] = '\0';
}
} else {
buf = line.get();
in->getline( buf , BUF_SIZE );
log(1) << "got line:" << buf << endl;
}
uassert( 10263 , "unknown error reading file" ,
(!(in->rdstate() & ios_base::badbit)) &&
(!(in->rdstate() & ios_base::failbit) || (in->rdstate() & ios_base::eofbit)) );
int len = 0;
if (strncmp("\xEF\xBB\xBF", buf, 3) == 0){ // UTF-8 BOM (notepad is stupid)
buf += 3;
len += 3;
}
if (_jsonArray){
while (buf[0] != '{' && buf[0] != '\0') {
len++;
buf++;
}
if (buf[0] == '\0')
break;
} else {
while (isspace( buf[0] )){
len++;
buf++;
}
if (buf[0] == '\0')
continue;
len += strlen( buf );
}
try {
vector<BSONObj> objects;
if (_jsonArray){
int jslen;
objects.push_back( fromjson(buf, &jslen) );
len += jslen;
buf += jslen;
} else {
parseLine( buf, &objects );
}
if ( _headerLine ){
_headerLine = false;
} else if (_doimport) {
for (vector<BSONObj>::const_iterator b_it=objects.begin(), end=objects.end(); b_it!=end; ++b_it){
bool doUpsert = _upsert;
BSONObjBuilder b;
if (_upsert){
for (vector<string>::const_iterator it=_upsertFields.begin(), end=_upsertFields.end(); it!=end; ++it){
BSONElement e = b_it->getFieldDotted(it->c_str());
if (e.eoo()){
doUpsert = false;
break;
}
b.appendAs(e, *it);
}
}
if (doUpsert){
conn().update(ns, Query(b.obj()), *b_it, true);
} else {
conn().insert( ns.c_str() , *b_it );
}
++num;
}
}
}
catch ( std::exception& e ){
cout << "exception:" << e.what() << endl;
cout << buf << endl;
errors++;
if (hasParam("stopOnError") || _jsonArray)
break;
}
if ( pm.hit( len + 1 ) ){
cout << "\t\t\t" << num << "\t" << ( num / ( time(0) - start ) ) << "/second" << endl;
}
}
cout << "imported " << num << " objects" << endl;
conn().getLastError();
if ( errors == 0 )
return 0;
cerr << "encountered " << errors << " error" << ( errors == 1 ? "" : "s" ) << endl;
return -1;
}