long long BSONTool::processFile( const boost::filesystem::path& root ) {
std::string fileName = root.string();
unsigned long long fileLength = file_size( root );
if ( fileLength == 0 ) {
toolInfoOutput() << "file " << fileName << " empty, skipping" << std::endl;
return 0;
}
FILE* file = fopen( fileName.c_str() , "rb" );
if ( ! file ) {
toolError() << "error opening file: " << fileName << " " << errnoWithDescription()
<< std::endl;
return 0;
}
#ifdef POSIX_FADV_SEQUENTIAL
posix_fadvise(fileno(file), 0, fileLength, POSIX_FADV_SEQUENTIAL);
#endif
if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) {
toolInfoOutput() << "\t file size: " << fileLength << std::endl;
}
unsigned long long read = 0;
unsigned long long num = 0;
unsigned long long processed = 0;
const int BUF_SIZE = BSONObjMaxUserSize + ( 1024 * 1024 );
boost::scoped_array<char> buf_holder(new char[BUF_SIZE]);
char * buf = buf_holder.get();
ProgressMeter m(fileLength);
if (!toolGlobalParams.quiet) {
m.setUnits( "bytes" );
}
while ( read < fileLength ) {
size_t amt = fread(buf, 1, 4, file);
verify( amt == 4 );
int size = ((int*)buf)[0];
uassert( 10264 , str::stream() << "invalid object size: " << size , size < BUF_SIZE );
amt = fread(buf+4, 1, size-4, file);
verify( amt == (size_t)( size - 4 ) );
BSONObj o( buf );
if (bsonToolGlobalParams.objcheck) {
const Status status = validateBSON(buf, size);
if (!status.isOK()) {
toolError() << "INVALID OBJECT - going to try and print out " << std::endl;
toolError() << "size: " << size << std::endl;
toolError() << "error: " << status.reason() << std::endl;
StringBuilder sb;
try {
o.toString(sb); // using StringBuilder version to get as much as possible
} catch (...) {
toolError() << "object up to error: " << sb.str() << endl;
throw;
}
toolError() << "complete object: " << sb.str() << endl;
// NOTE: continuing with object even though we know it is invalid.
}
}
if (!bsonToolGlobalParams.hasFilter || _matcher->matches(o)) {
gotObject( o );
processed++;
}
read += o.objsize();
num++;
if (!toolGlobalParams.quiet) {
m.hit(o.objsize());
}
}
fclose( file );
uassert(10265, "counts don't match", read == fileLength);
toolInfoOutput() << num << ((num == 1) ? " document" : " documents")
<< " found" << std::endl;
if (bsonToolGlobalParams.hasFilter) {
toolInfoOutput() << processed
<< ((processed == 1) ? " document" : " documents")
<< " processed" << std::endl;
}
return processed;
}
int Tool::main( int argc , char ** argv, char ** envp ) {
setupSignalHandlers(true);
static StaticObserver staticObserver;
mongo::runGlobalInitializersOrDie(argc, argv, envp);
// hide password from ps output
for (int i=0; i < (argc-1); ++i) {
if (!strcmp(argv[i], "-p") || !strcmp(argv[i], "--password")) {
char* arg = argv[i+1];
while (*arg) {
*arg++ = 'x';
}
}
}
if (!toolGlobalParams.useDirectClient) {
if (toolGlobalParams.noconnection) {
// do nothing
}
else {
string errmsg;
ConnectionString cs = ConnectionString::parse(toolGlobalParams.connectionString,
errmsg);
if ( ! cs.isValid() ) {
toolError() << "invalid hostname [" << toolGlobalParams.connectionString << "] "
<< errmsg << std::endl;
::_exit(-1);
}
_conn = cs.connect( errmsg );
if ( ! _conn ) {
toolError() << "couldn't connect to [" << toolGlobalParams.connectionString
<< "] " << errmsg << std::endl;
::_exit(-1);
}
toolInfoOutput() << "connected to: " << toolGlobalParams.connectionString
<< std::endl;
}
}
else {
verify( lastError.get( true ) );
Client::initThread("tools");
_conn = new DBDirectClient();
storageGlobalParams.dbpath = toolGlobalParams.dbpath;
try {
acquirePathLock();
}
catch ( DBException& ) {
toolError() << std::endl << "If you are running a mongod on the same "
"path you should connect to that instead of direct data "
"file access" << std::endl << std::endl;
dbexit( EXIT_FS );
::_exit(EXIT_FAILURE);
}
FileAllocator::get()->start();
dur::startup();
}
int ret = -1;
try {
if (!toolGlobalParams.useDirectClient && !toolGlobalParams.noconnection)
auth();
ret = run();
}
catch ( DBException& e ) {
toolError() << "assertion: " << e.toString() << std::endl;
ret = -1;
}
catch(const boost::filesystem::filesystem_error &fse) {
/*
https://jira.mongodb.org/browse/SERVER-2904
Simple tools that don't access the database, such as
bsondump, aren't throwing DBExceptions, but are throwing
boost exceptions.
The currently available set of error codes don't seem to match
boost documentation. boost::filesystem::not_found_error
(from http://www.boost.org/doc/libs/1_31_0/libs/filesystem/doc/exception.htm)
doesn't seem to exist in our headers. Also, fse.code() isn't
boost::system::errc::no_such_file_or_directory when this
happens, as you would expect. And, determined from
experimentation that the command-line argument gets turned into
"\\?" instead of "/?" !!!
*/
#if defined(_WIN32)
if (/*(fse.code() == boost::system::errc::no_such_file_or_directory) &&*/
(fse.path1() == "\\?"))
printHelp(cerr);
else
#endif // _WIN32
toolError() << "error: " << fse.what() << std::endl;
ret = -1;
}
if ( currentClient.get() )
currentClient.get()->shutdown();
if (toolGlobalParams.useDirectClient)
dbexit( EXIT_CLEAN );
fflush(stdout);
fflush(stderr);
::_exit(ret);
}
long long BSONTool::processFile( const boost::filesystem::path& root ) {
std::string fileName = root.string();
unsigned long long fileLength = file_size( root );
if ( fileLength == 0 ) {
toolInfoOutput() << "file " << fileName << " empty, skipping" << std::endl;
return 0;
}
FILE* file = fopen( fileName.c_str() , "rb" );
if ( ! file ) {
toolError() << "error opening file: " << fileName << " " << errnoWithDescription()
<< std::endl;
return 0;
}
#ifdef POSIX_FADV_SEQUENTIAL
posix_fadvise(fileno(file), 0, fileLength, POSIX_FADV_SEQUENTIAL);
#endif
if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) {
toolInfoOutput() << "\t file size: " << fileLength << std::endl;
}
unsigned long long read = 0;
unsigned long long num = 0;
unsigned long long processed = 0;
const int BUF_SIZE = BSONObjMaxUserSize + ( 1024 * 1024 );
boost::scoped_array<char> buf_holder(new char[BUF_SIZE]);
char * buf = buf_holder.get();
ProgressMeter m(fileLength);
if (!toolGlobalParams.quiet) {
m.setUnits( "bytes" );
}
while ( read < fileLength ) {
size_t amt = fread(buf, 1, 4, file);
verify( amt == 4 );
int size = ((int*)buf)[0];
uassert( 10264 , str::stream() << "invalid object size: " << size , size < BUF_SIZE );
amt = fread(buf+4, 1, size-4, file);
verify( amt == (size_t)( size - 4 ) );
BSONObj o( buf );
if (bsonToolGlobalParams.objcheck && !o.valid()) {
toolError() << "INVALID OBJECT - going to try and print out " << std::endl;
toolError() << "size: " << size << std::endl;
BSONObjIterator i(o);
while ( i.more() ) {
BSONElement e = i.next();
try {
e.validate();
}
catch ( ... ) {
toolError() << "\t\t NEXT ONE IS INVALID" << std::endl;
}
toolError() << "\t name : " << e.fieldName() << " " << typeName(e.type())
<< std::endl;
toolError() << "\t " << e << std::endl;
}
}
if (!bsonToolGlobalParams.hasFilter || _matcher->matches(o)) {
gotObject( o );
processed++;
}
read += o.objsize();
num++;
if (!toolGlobalParams.quiet) {
m.hit(o.objsize());
}
}
fclose( file );
uassert(10265, "counts don't match", read == fileLength);
toolInfoOutput() << num << " objects found" << std::endl;
if (bsonToolGlobalParams.hasFilter)
toolInfoOutput() << processed << " objects processed" << std::endl;
return processed;
}
