void drillDown( path root, bool use_db, bool use_coll, bool top_level=false ) {
log(2) << "drillDown: " << root.string() << endl;
// skip hidden files and directories
if (root.leaf()[0] == '.' && root.leaf() != ".")
return;
if ( is_directory( root ) ) {
directory_iterator end;
directory_iterator i(root);
path indexes;
while ( i != end ) {
path p = *i;
i++;
if (use_db) {
if (is_directory(p)) {
cerr << "ERROR: root directory must be a dump of a single database" << endl;
cerr << " when specifying a db name with --db" << endl;
printHelp(cout);
return;
}
}
if (use_coll) {
if (is_directory(p) || i != end) {
cerr << "ERROR: root directory must be a dump of a single collection" << endl;
cerr << " when specifying a collection name with --collection" << endl;
printHelp(cout);
return;
}
}
// don't insert oplog
if (top_level && !use_db && p.leaf() == "oplog.bson")
continue;
if ( p.leaf() == "system.indexes.bson" )
indexes = p;
else
drillDown(p, use_db, use_coll);
}
if (!indexes.empty())
drillDown(indexes, use_db, use_coll);
return;
}
if ( ! ( endsWith( root.string().c_str() , ".bson" ) ||
endsWith( root.string().c_str() , ".bin" ) ) ) {
cerr << "don't know what to do with file [" << root.string() << "]" << endl;
return;
}
log() << root.string() << endl;
if ( root.leaf() == "system.profile.bson" ) {
log() << "\t skipping" << endl;
return;
}
string ns;
if (use_db) {
ns += _db;
}
else {
string dir = root.branch_path().string();
if ( dir.find( "/" ) == string::npos )
ns += dir;
else
ns += dir.substr( dir.find_last_of( "/" ) + 1 );
if ( ns.size() == 0 )
ns = "test";
}
assert( ns.size() );
if (use_coll) {
ns += "." + _coll;
}
else {
string l = root.leaf();
l = l.substr( 0 , l.find_last_of( "." ) );
ns += "." + l;
}
out() << "\t going into namespace [" << ns << "]" << endl;
if ( _drop ) {
if (root.leaf() != "system.users.bson" ) {
out() << "\t dropping" << endl;
conn().dropCollection( ns );
} else {
// Create map of the users currently in the DB
BSONObj fields = BSON("user" << 1);
scoped_ptr<DBClientCursor> cursor(conn().query(ns, Query(), 0, 0, &fields));
while (cursor->more()) {
BSONObj user = cursor->next();
_users.insert(user["user"].String());
}
}
}
_curns = ns.c_str();
_curdb = NamespaceString(_curns).db;
processFile( root );
if (_drop && root.leaf() == "system.users.bson") {
// Delete any users that used to exist but weren't in the dump file
for (set<string>::iterator it = _users.begin(); it != _users.end(); ++it) {
BSONObj userMatch = BSON("user" << *it);
conn().remove(ns, Query(userMatch));
}
_users.clear();
}
}
void drillDown( path root ) {
if ( is_directory( root ) ) {
directory_iterator end;
directory_iterator i(root);
while ( i != end ) {
path p = *i;
drillDown( p );
i++;
}
return;
}
if ( ! ( endsWith( root.string().c_str() , ".bson" ) ||
endsWith( root.string().c_str() , ".bin" ) ) ) {
cerr << "don't know what to do with [" << root.string() << "]" << endl;
return;
}
out() << root.string() << endl;
string ns;
{
string dir = root.branch_path().string();
if ( dir.find( "/" ) == string::npos )
ns += dir;
else
ns += dir.substr( dir.find_last_of( "/" ) + 1 );
}
{
string l = root.leaf();
l = l.substr( 0 , l.find_last_of( "." ) );
ns += "." + l;
}
if ( boost::filesystem::file_size( root ) == 0 ) {
out() << "file " + root.native_file_string() + " empty, aborting" << endl;
return;
}
out() << "\t going into namespace [" << ns << "]" << endl;
MemoryMappedFile mmf;
assert( mmf.map( root.string().c_str() ) );
char * data = (char*)mmf.viewOfs();
int read = 0;
int num = 0;
while ( read < mmf.length() ) {
BSONObj o( data );
conn().insert( ns.c_str() , o );
read += o.objsize();
data += o.objsize();
if ( ! ( ++num % 1000 ) )
out() << "read " << read << "/" << mmf.length() << " bytes so far. " << num << " objects" << endl;
}
out() << "\t " << num << " objects" << endl;
}
void crlf_check::inspect(
const string & library_name,
const path & full_path, // example: c:/foo/boost/filesystem/path.hpp
const string & contents ) // contents of file to be inspected
{
if (contents.find( "boostinspect:" "nocrlf" ) != string::npos) return;
// this file deliberately contains errors
const char test_file_name[] = "wrong_line_ends_test.cpp";
bool failed = false;
// The understanding on line endings, as I remember it, was that
// either "\n" or "\r\n" is OK, and they can be mixed, but "\r" alone
// is not acceptable. Mixed line endings are allowed because Boost files
// are commonly edited in both Windows and UNIX environments, and editors
// in those environments generally accept either ending. Even Mac people
// agreed with this policy. --Beman
// Joerg's original implementation is saved below,
// in case we change our minds!
for ( std::string::const_iterator itr ( contents.begin() );
itr != contents.end(); ++itr )
{
if ( *itr == '\r' && ((itr+1) == contents.end() || *(itr+1) != '\n') )
{
failed = true;
break;
}
}
if (failed && full_path.leaf() != test_file_name)
{
++m_files_with_errors;
error( library_name, full_path, name() );
}
if (!failed && full_path.leaf() == test_file_name)
{
++m_files_with_errors;
error( library_name, full_path, string(name()) + " should have cr-only line endings" );
}
/*
size_t cr_count = 0;
size_t lf_count = 0;
size_t crlf_count = 0;
bool had_cr = false;
for ( size_t i = 0; i < contents.length(); ++i )
{
switch ( contents[i] )
{
case '\r':
had_cr = true;
++cr_count;
break;
case '\n':
++lf_count;
if ( had_cr )
++crlf_count;
// fallthrough
default:
had_cr = false;
break;
}
}
if ( cr_count > 0 && lf_count != crlf_count )
{
++m_files_with_errors;
error( library_name, full_path, desc() );
}
*/
}
void file_name_check::inspect(
const string & library_name,
const path & full_path )
{
string::size_type pos;
// called for each file and directory, so only the leaf need be tested
string const leaf( full_path.leaf().string() );
// includes only allowable characters
if ( (pos = leaf.find_first_not_of( allowable )) != string::npos )
{
++m_name_errors;
error( library_name, full_path, loclink(full_path, string(name()))
+ " file or directory name contains unacceptable character '"
+ leaf[pos] + "'" );
}
// allowable initial character
if ( std::strchr( initial_char, leaf[0] ) == 0 )
{
++m_name_errors;
error( library_name, full_path, loclink(full_path, string(name()))
+ " file or directory name begins with an unacceptable character" );
}
// rules for dot characters differ slightly for directories and files
if ( filesystem::is_directory( full_path ) )
{
if ( std::strchr( leaf.c_str(), '.' ) )
{
++m_name_errors;
error( library_name, full_path, loclink(full_path, string(name()))
+ " directory name contains a dot character ('.')" );
}
}
//else // not a directory
//{
// // includes at most one dot character
// const char * first_dot = std::strchr( leaf.c_str(), '.' );
// if ( first_dot && std::strchr( first_dot+1, '.' ) )
// {
// ++m_name_errors;
// error( library_name, full_path, string(name())
// + " file name with more than one dot character ('.')" );
// }
//}
// the path, including a presumed root, does not exceed the maximum size
path const relative_path( relative_to( full_path, search_root_path() ) );
const unsigned max_relative_path = 207; // ISO 9660:1999 sets this limit
const string generic_root( "boost_X_XX_X/" );
if ( relative_path.string().size() >
( max_relative_path - generic_root.size() ) )
{
++m_name_errors;
error( library_name, full_path,
loclink(full_path, string(name()))
+ " path will exceed "
+ boost::lexical_cast<string>(max_relative_path)
+ " characters in a directory tree with a root in the form "
+ generic_root + ", and this exceeds ISO 9660:1999 limit of 207" )
;
}
}
void drillDown( path root, bool use_db = false, bool use_coll = false ) {
log(2) << "drillDown: " << root.string() << endl;
if ( is_directory( root ) ) {
directory_iterator end;
directory_iterator i(root);
path indexes;
while ( i != end ) {
path p = *i;
i++;
if (use_db) {
if (is_directory(p)) {
cerr << "ERROR: root directory must be a dump of a single database" << endl;
cerr << " when specifying a db name with --db" << endl;
printHelp(cout);
return;
}
}
if (use_coll) {
if (is_directory(p) || i != end) {
cerr << "ERROR: root directory must be a dump of a single collection" << endl;
cerr << " when specifying a collection name with --collection" << endl;
printHelp(cout);
return;
}
}
if ( _indexesLast && p.leaf() == "system.indexes.bson" )
indexes = p;
else
drillDown(p, use_db, use_coll);
}
if (!indexes.empty())
drillDown(indexes, use_db, use_coll);
return;
}
if ( ! ( endsWith( root.string().c_str() , ".bson" ) ||
endsWith( root.string().c_str() , ".bin" ) ) ) {
cerr << "don't know what to do with [" << root.string() << "]" << endl;
return;
}
log() << root.string() << endl;
if ( root.leaf() == "system.profile.bson" ){
log() << "\t skipping" << endl;
return;
}
string ns;
if (use_db) {
ns += _db;
}
else {
string dir = root.branch_path().string();
if ( dir.find( "/" ) == string::npos )
ns += dir;
else
ns += dir.substr( dir.find_last_of( "/" ) + 1 );
if ( ns.size() == 0 )
ns = "test";
}
assert( ns.size() );
if (use_coll) {
ns += "." + _coll;
} else {
string l = root.leaf();
l = l.substr( 0 , l.find_last_of( "." ) );
ns += "." + l;
}
out() << "\t going into namespace [" << ns << "]" << endl;
if ( _drop ){
out() << "\t dropping" << endl;
conn().dropCollection( ns );
}
_curns = ns.c_str();
processFile( root );
}
void drillDown( path root, bool use_db = false ) {
log(2) << "drillDown: " << root.string() << endl;
if ( is_directory( root ) ) {
directory_iterator end;
directory_iterator i(root);
while ( i != end ) {
path p = *i;
if (use_db) {
if (is_directory(p) ||
!(endsWith(p.string().c_str(), ".bson") ||
endsWith(p.string().c_str(), ".bin" ))) {
cerr << "ERROR: root directory must be a dump of a single database" << endl;
cerr << " when specifying a db name with --db" << endl;
printHelp(cout);
return;
}
}
drillDown(p, use_db);
i++;
}
return;
}
if ( ! ( endsWith( root.string().c_str() , ".bson" ) ||
endsWith( root.string().c_str() , ".bin" ) ) ) {
cerr << "don't know what to do with [" << root.string() << "]" << endl;
return;
}
out() << root.string() << endl;
string ns;
if (use_db) {
ns += _db;
} else {
string dir = root.branch_path().string();
if ( dir.find( "/" ) == string::npos )
ns += dir;
else
ns += dir.substr( dir.find_last_of( "/" ) + 1 );
}
{
string l = root.leaf();
l = l.substr( 0 , l.find_last_of( "." ) );
ns += "." + l;
}
long long fileLength = file_size( root );
if ( fileLength == 0 ) {
out() << "file " + root.native_file_string() + " empty, skipping" << endl;
return;
}
out() << "\t going into namespace [" << ns << "]" << endl;
string fileString = root.string();
ifstream file( fileString.c_str() , ios_base::in | ios_base::binary);
if ( ! file.is_open() ){
log() << "error opening file: " << fileString << endl;
return;
}
log(1) << "\t file size: " << fileLength << endl;
long long read = 0;
long long num = 0;
int msgDelay = (int)(1000 * ( 1 + ( fileLength / ( 1024.0 * 1024 * 400 ) ) ) );
log(1) << "\t msg delay: " << msgDelay << endl;
const int BUF_SIZE = 1024 * 1024 * 5;
char * buf = (char*)malloc( BUF_SIZE );
while ( read < fileLength ) {
file.read( buf , 4 );
int size = ((int*)buf)[0];
assert( size < BUF_SIZE );
file.read( buf + 4 , size - 4 );
BSONObj o( buf );
conn().insert( ns.c_str() , o );
read += o.objsize();
num++;
if ( ( logLevel > 0 && num < 10 ) || ! ( num % msgDelay ) )
out() << "read " << read << "/" << fileLength << " bytes so far. (" << (int)( (read * 100) / fileLength) << "%) " << num << " objects" << endl;
}
free( buf );
out() << "\t " << num << " objects" << endl;
}