void drillDown( path root, bool use_db, bool use_coll, bool top_level=false ) { log(2) << "drillDown: " << root.string() << endl; // skip hidden files and directories if (root.leaf()[0] == '.' && root.leaf() != ".") return; if ( is_directory( root ) ) { directory_iterator end; directory_iterator i(root); path indexes; while ( i != end ) { path p = *i; i++; if (use_db) { if (is_directory(p)) { cerr << "ERROR: root directory must be a dump of a single database" << endl; cerr << " when specifying a db name with --db" << endl; printHelp(cout); return; } } if (use_coll) { if (is_directory(p) || i != end) { cerr << "ERROR: root directory must be a dump of a single collection" << endl; cerr << " when specifying a collection name with --collection" << endl; printHelp(cout); return; } } // don't insert oplog if (top_level && !use_db && p.leaf() == "oplog.bson") continue; if ( p.leaf() == "system.indexes.bson" ) indexes = p; else drillDown(p, use_db, use_coll); } if (!indexes.empty()) drillDown(indexes, use_db, use_coll); return; } if ( ! ( endsWith( root.string().c_str() , ".bson" ) || endsWith( root.string().c_str() , ".bin" ) ) ) { cerr << "don't know what to do with file [" << root.string() << "]" << endl; return; } log() << root.string() << endl; if ( root.leaf() == "system.profile.bson" ) { log() << "\t skipping" << endl; return; } string ns; if (use_db) { ns += _db; } else { string dir = root.branch_path().string(); if ( dir.find( "/" ) == string::npos ) ns += dir; else ns += dir.substr( dir.find_last_of( "/" ) + 1 ); if ( ns.size() == 0 ) ns = "test"; } assert( ns.size() ); if (use_coll) { ns += "." + _coll; } else { string l = root.leaf(); l = l.substr( 0 , l.find_last_of( "." ) ); ns += "." + l; } out() << "\t going into namespace [" << ns << "]" << endl; if ( _drop ) { if (root.leaf() != "system.users.bson" ) { out() << "\t dropping" << endl; conn().dropCollection( ns ); } else { // Create map of the users currently in the DB BSONObj fields = BSON("user" << 1); scoped_ptr<DBClientCursor> cursor(conn().query(ns, Query(), 0, 0, &fields)); while (cursor->more()) { BSONObj user = cursor->next(); _users.insert(user["user"].String()); } } } _curns = ns.c_str(); _curdb = NamespaceString(_curns).db; processFile( root ); if (_drop && root.leaf() == "system.users.bson") { // Delete any users that used to exist but weren't in the dump file for (set<string>::iterator it = _users.begin(); it != _users.end(); ++it) { BSONObj userMatch = BSON("user" << *it); conn().remove(ns, Query(userMatch)); } _users.clear(); } }
void drillDown( path root ) { if ( is_directory( root ) ) { directory_iterator end; directory_iterator i(root); while ( i != end ) { path p = *i; drillDown( p ); i++; } return; } if ( ! ( endsWith( root.string().c_str() , ".bson" ) || endsWith( root.string().c_str() , ".bin" ) ) ) { cerr << "don't know what to do with [" << root.string() << "]" << endl; return; } out() << root.string() << endl; string ns; { string dir = root.branch_path().string(); if ( dir.find( "/" ) == string::npos ) ns += dir; else ns += dir.substr( dir.find_last_of( "/" ) + 1 ); } { string l = root.leaf(); l = l.substr( 0 , l.find_last_of( "." ) ); ns += "." + l; } if ( boost::filesystem::file_size( root ) == 0 ) { out() << "file " + root.native_file_string() + " empty, aborting" << endl; return; } out() << "\t going into namespace [" << ns << "]" << endl; MemoryMappedFile mmf; assert( mmf.map( root.string().c_str() ) ); char * data = (char*)mmf.viewOfs(); int read = 0; int num = 0; while ( read < mmf.length() ) { BSONObj o( data ); conn().insert( ns.c_str() , o ); read += o.objsize(); data += o.objsize(); if ( ! ( ++num % 1000 ) ) out() << "read " << read << "/" << mmf.length() << " bytes so far. " << num << " objects" << endl; } out() << "\t " << num << " objects" << endl; }
void crlf_check::inspect( const string & library_name, const path & full_path, // example: c:/foo/boost/filesystem/path.hpp const string & contents ) // contents of file to be inspected { if (contents.find( "boostinspect:" "nocrlf" ) != string::npos) return; // this file deliberately contains errors const char test_file_name[] = "wrong_line_ends_test.cpp"; bool failed = false; // The understanding on line endings, as I remember it, was that // either "\n" or "\r\n" is OK, and they can be mixed, but "\r" alone // is not acceptable. Mixed line endings are allowed because Boost files // are commonly edited in both Windows and UNIX environments, and editors // in those environments generally accept either ending. Even Mac people // agreed with this policy. --Beman // Joerg's original implementation is saved below, // in case we change our minds! for ( std::string::const_iterator itr ( contents.begin() ); itr != contents.end(); ++itr ) { if ( *itr == '\r' && ((itr+1) == contents.end() || *(itr+1) != '\n') ) { failed = true; break; } } if (failed && full_path.leaf() != test_file_name) { ++m_files_with_errors; error( library_name, full_path, name() ); } if (!failed && full_path.leaf() == test_file_name) { ++m_files_with_errors; error( library_name, full_path, string(name()) + " should have cr-only line endings" ); } /* size_t cr_count = 0; size_t lf_count = 0; size_t crlf_count = 0; bool had_cr = false; for ( size_t i = 0; i < contents.length(); ++i ) { switch ( contents[i] ) { case '\r': had_cr = true; ++cr_count; break; case '\n': ++lf_count; if ( had_cr ) ++crlf_count; // fallthrough default: had_cr = false; break; } } if ( cr_count > 0 && lf_count != crlf_count ) { ++m_files_with_errors; error( library_name, full_path, desc() ); } */ }
void file_name_check::inspect( const string & library_name, const path & full_path ) { string::size_type pos; // called for each file and directory, so only the leaf need be tested string const leaf( full_path.leaf().string() ); // includes only allowable characters if ( (pos = leaf.find_first_not_of( allowable )) != string::npos ) { ++m_name_errors; error( library_name, full_path, loclink(full_path, string(name())) + " file or directory name contains unacceptable character '" + leaf[pos] + "'" ); } // allowable initial character if ( std::strchr( initial_char, leaf[0] ) == 0 ) { ++m_name_errors; error( library_name, full_path, loclink(full_path, string(name())) + " file or directory name begins with an unacceptable character" ); } // rules for dot characters differ slightly for directories and files if ( filesystem::is_directory( full_path ) ) { if ( std::strchr( leaf.c_str(), '.' ) ) { ++m_name_errors; error( library_name, full_path, loclink(full_path, string(name())) + " directory name contains a dot character ('.')" ); } } //else // not a directory //{ // // includes at most one dot character // const char * first_dot = std::strchr( leaf.c_str(), '.' ); // if ( first_dot && std::strchr( first_dot+1, '.' ) ) // { // ++m_name_errors; // error( library_name, full_path, string(name()) // + " file name with more than one dot character ('.')" ); // } //} // the path, including a presumed root, does not exceed the maximum size path const relative_path( relative_to( full_path, search_root_path() ) ); const unsigned max_relative_path = 207; // ISO 9660:1999 sets this limit const string generic_root( "boost_X_XX_X/" ); if ( relative_path.string().size() > ( max_relative_path - generic_root.size() ) ) { ++m_name_errors; error( library_name, full_path, loclink(full_path, string(name())) + " path will exceed " + boost::lexical_cast<string>(max_relative_path) + " characters in a directory tree with a root in the form " + generic_root + ", and this exceeds ISO 9660:1999 limit of 207" ) ; } }
void drillDown( path root, bool use_db = false, bool use_coll = false ) { log(2) << "drillDown: " << root.string() << endl; if ( is_directory( root ) ) { directory_iterator end; directory_iterator i(root); path indexes; while ( i != end ) { path p = *i; i++; if (use_db) { if (is_directory(p)) { cerr << "ERROR: root directory must be a dump of a single database" << endl; cerr << " when specifying a db name with --db" << endl; printHelp(cout); return; } } if (use_coll) { if (is_directory(p) || i != end) { cerr << "ERROR: root directory must be a dump of a single collection" << endl; cerr << " when specifying a collection name with --collection" << endl; printHelp(cout); return; } } if ( _indexesLast && p.leaf() == "system.indexes.bson" ) indexes = p; else drillDown(p, use_db, use_coll); } if (!indexes.empty()) drillDown(indexes, use_db, use_coll); return; } if ( ! ( endsWith( root.string().c_str() , ".bson" ) || endsWith( root.string().c_str() , ".bin" ) ) ) { cerr << "don't know what to do with [" << root.string() << "]" << endl; return; } log() << root.string() << endl; if ( root.leaf() == "system.profile.bson" ){ log() << "\t skipping" << endl; return; } string ns; if (use_db) { ns += _db; } else { string dir = root.branch_path().string(); if ( dir.find( "/" ) == string::npos ) ns += dir; else ns += dir.substr( dir.find_last_of( "/" ) + 1 ); if ( ns.size() == 0 ) ns = "test"; } assert( ns.size() ); if (use_coll) { ns += "." + _coll; } else { string l = root.leaf(); l = l.substr( 0 , l.find_last_of( "." ) ); ns += "." + l; } out() << "\t going into namespace [" << ns << "]" << endl; if ( _drop ){ out() << "\t dropping" << endl; conn().dropCollection( ns ); } _curns = ns.c_str(); processFile( root ); }
void drillDown( path root, bool use_db = false ) { log(2) << "drillDown: " << root.string() << endl; if ( is_directory( root ) ) { directory_iterator end; directory_iterator i(root); while ( i != end ) { path p = *i; if (use_db) { if (is_directory(p) || !(endsWith(p.string().c_str(), ".bson") || endsWith(p.string().c_str(), ".bin" ))) { cerr << "ERROR: root directory must be a dump of a single database" << endl; cerr << " when specifying a db name with --db" << endl; printHelp(cout); return; } } drillDown(p, use_db); i++; } return; } if ( ! ( endsWith( root.string().c_str() , ".bson" ) || endsWith( root.string().c_str() , ".bin" ) ) ) { cerr << "don't know what to do with [" << root.string() << "]" << endl; return; } out() << root.string() << endl; string ns; if (use_db) { ns += _db; } else { string dir = root.branch_path().string(); if ( dir.find( "/" ) == string::npos ) ns += dir; else ns += dir.substr( dir.find_last_of( "/" ) + 1 ); } { string l = root.leaf(); l = l.substr( 0 , l.find_last_of( "." ) ); ns += "." + l; } long long fileLength = file_size( root ); if ( fileLength == 0 ) { out() << "file " + root.native_file_string() + " empty, skipping" << endl; return; } out() << "\t going into namespace [" << ns << "]" << endl; string fileString = root.string(); ifstream file( fileString.c_str() , ios_base::in | ios_base::binary); if ( ! file.is_open() ){ log() << "error opening file: " << fileString << endl; return; } log(1) << "\t file size: " << fileLength << endl; long long read = 0; long long num = 0; int msgDelay = (int)(1000 * ( 1 + ( fileLength / ( 1024.0 * 1024 * 400 ) ) ) ); log(1) << "\t msg delay: " << msgDelay << endl; const int BUF_SIZE = 1024 * 1024 * 5; char * buf = (char*)malloc( BUF_SIZE ); while ( read < fileLength ) { file.read( buf , 4 ); int size = ((int*)buf)[0]; assert( size < BUF_SIZE ); file.read( buf + 4 , size - 4 ); BSONObj o( buf ); conn().insert( ns.c_str() , o ); read += o.objsize(); num++; if ( ( logLevel > 0 && num < 10 ) || ! ( num % msgDelay ) ) out() << "read " << read << "/" << fileLength << " bytes so far. (" << (int)( (read * 100) / fileLength) << "%) " << num << " objects" << endl; } free( buf ); out() << "\t " << num << " objects" << endl; }