Beispiel #1
0
    void drillDown( path root, bool use_db, bool use_coll, bool top_level=false ) {
        log(2) << "drillDown: " << root.string() << endl;

        // skip hidden files and directories
        if (root.leaf()[0] == '.' && root.leaf() != ".")
            return;

        if ( is_directory( root ) ) {
            directory_iterator end;
            directory_iterator i(root);
            path indexes;
            while ( i != end ) {
                path p = *i;
                i++;

                if (use_db) {
                    if (is_directory(p)) {
                        cerr << "ERROR: root directory must be a dump of a single database" << endl;
                        cerr << "       when specifying a db name with --db" << endl;
                        printHelp(cout);
                        return;
                    }
                }

                if (use_coll) {
                    if (is_directory(p) || i != end) {
                        cerr << "ERROR: root directory must be a dump of a single collection" << endl;
                        cerr << "       when specifying a collection name with --collection" << endl;
                        printHelp(cout);
                        return;
                    }
                }

                // don't insert oplog
                if (top_level && !use_db && p.leaf() == "oplog.bson")
                    continue;

                if ( p.leaf() == "system.indexes.bson" )
                    indexes = p;
                else
                    drillDown(p, use_db, use_coll);
            }

            if (!indexes.empty())
                drillDown(indexes, use_db, use_coll);

            return;
        }

        if ( ! ( endsWith( root.string().c_str() , ".bson" ) ||
                 endsWith( root.string().c_str() , ".bin" ) ) ) {
            cerr << "don't know what to do with file [" << root.string() << "]" << endl;
            return;
        }

        log() << root.string() << endl;

        if ( root.leaf() == "system.profile.bson" ) {
            log() << "\t skipping" << endl;
            return;
        }

        string ns;
        if (use_db) {
            ns += _db;
        }
        else {
            string dir = root.branch_path().string();
            if ( dir.find( "/" ) == string::npos )
                ns += dir;
            else
                ns += dir.substr( dir.find_last_of( "/" ) + 1 );

            if ( ns.size() == 0 )
                ns = "test";
        }

        assert( ns.size() );

        if (use_coll) {
            ns += "." + _coll;
        }
        else {
            string l = root.leaf();
            l = l.substr( 0 , l.find_last_of( "." ) );
            ns += "." + l;
        }

        out() << "\t going into namespace [" << ns << "]" << endl;

        if ( _drop ) {
            if (root.leaf() != "system.users.bson" ) {
                out() << "\t dropping" << endl;
                conn().dropCollection( ns );
            } else {
                // Create map of the users currently in the DB
                BSONObj fields = BSON("user" << 1);
                scoped_ptr<DBClientCursor> cursor(conn().query(ns, Query(), 0, 0, &fields));
                while (cursor->more()) {
                    BSONObj user = cursor->next();
                    _users.insert(user["user"].String());
                }
            }
        }

        _curns = ns.c_str();
        _curdb = NamespaceString(_curns).db;
        processFile( root );
        if (_drop && root.leaf() == "system.users.bson") {
            // Delete any users that used to exist but weren't in the dump file
            for (set<string>::iterator it = _users.begin(); it != _users.end(); ++it) {
                BSONObj userMatch = BSON("user" << *it);
                conn().remove(ns, Query(userMatch));
            }
            _users.clear();
        }
    }
Beispiel #2
0
    void drillDown( path root ) {

        if ( is_directory( root ) ) {
            directory_iterator end;
            directory_iterator i(root);
            while ( i != end ) {
                path p = *i;
                drillDown( p );
                i++;
            }
            return;
        }
        
        if ( ! ( endsWith( root.string().c_str() , ".bson" ) ||
                 endsWith( root.string().c_str() , ".bin" ) ) ) {
            cerr << "don't know what to do with [" << root.string() << "]" << endl;
            return;
        }
        
        out() << root.string() << endl;
        
        string ns;
        {
            string dir = root.branch_path().string();
            if ( dir.find( "/" ) == string::npos )
                ns += dir;
            else
                ns += dir.substr( dir.find_last_of( "/" ) + 1 );
        }
        
        {
            string l = root.leaf();
            l = l.substr( 0 , l.find_last_of( "." ) );
            ns += "." + l;
        }
        
        if ( boost::filesystem::file_size( root ) == 0 ) {
            out() << "file " + root.native_file_string() + " empty, aborting" << endl;
            return;
        }

        out() << "\t going into namespace [" << ns << "]" << endl;
        
        MemoryMappedFile mmf;
        assert( mmf.map( root.string().c_str() ) );
        
        char * data = (char*)mmf.viewOfs();
        int read = 0;
        
        int num = 0;
        
        while ( read < mmf.length() ) {
            BSONObj o( data );
            
            conn().insert( ns.c_str() , o );
            
            read += o.objsize();
            data += o.objsize();

            if ( ! ( ++num % 1000 ) )
                out() << "read " << read << "/" << mmf.length() << " bytes so far. " << num << " objects" << endl;
        }
        
        out() << "\t "  << num << " objects" << endl;
    }
Beispiel #3
0
   void crlf_check::inspect(
      const string & library_name,
      const path & full_path,   // example: c:/foo/boost/filesystem/path.hpp
      const string & contents )     // contents of file to be inspected
    {
      if (contents.find( "boostinspect:" "nocrlf" ) != string::npos) return;

      // this file deliberately contains errors
      const char test_file_name[] = "wrong_line_ends_test.cpp";

      bool failed = false;
      // The understanding on line endings, as I remember it, was that
      // either "\n" or "\r\n" is OK, and they can be mixed, but "\r" alone
      // is not acceptable. Mixed line endings are allowed because Boost files
      // are commonly edited in both Windows and UNIX environments, and editors
      // in those environments generally accept either ending. Even Mac people
      // agreed with this policy. --Beman

      // Joerg's original implementation is saved below,
      // in case we change our minds!

      for ( std::string::const_iterator itr ( contents.begin() );
        itr != contents.end(); ++itr )
      {
        if ( *itr == '\r' && ((itr+1) == contents.end() || *(itr+1) != '\n') )
        {
          failed = true;
          break;
        }
      }

      if (failed && full_path.leaf() != test_file_name)
      {
        ++m_files_with_errors;
        error( library_name, full_path, name() );
      }

      if (!failed && full_path.leaf() == test_file_name)
      {
        ++m_files_with_errors;
        error( library_name, full_path, string(name()) + " should have cr-only line endings" );
      }

/*
      size_t cr_count = 0;
      size_t lf_count = 0;
      size_t crlf_count = 0;
      bool had_cr = false;
      for ( size_t i = 0; i < contents.length(); ++i )
      {
        switch ( contents[i] )
        {
          case '\r':
            had_cr = true;
            ++cr_count;
            break;
          case '\n':
            ++lf_count;
            if ( had_cr )
              ++crlf_count;
            // fallthrough
          default:
            had_cr = false;
            break;
        }
      }
      if ( cr_count > 0 && lf_count != crlf_count )
      {
        ++m_files_with_errors;
        error( library_name, full_path, desc() );
      }
*/
    }
Beispiel #4
0
    void file_name_check::inspect(
      const string & library_name,
      const path & full_path )
    {
      string::size_type pos;

      //  called for each file and directory, so only the leaf need be tested
      string const leaf( full_path.leaf().string() );

      //  includes only allowable characters
      if ( (pos = leaf.find_first_not_of( allowable )) != string::npos )
      {
        ++m_name_errors;
        error( library_name, full_path, loclink(full_path, string(name()))
            + " file or directory name contains unacceptable character '"
            + leaf[pos] + "'" );
      }

      //  allowable initial character
      if ( std::strchr( initial_char, leaf[0] ) == 0 )
      {
        ++m_name_errors;
        error( library_name, full_path, loclink(full_path, string(name()))
            + " file or directory name begins with an unacceptable character" );
      }

      //  rules for dot characters differ slightly for directories and files
      if ( filesystem::is_directory( full_path ) )
      {
        if ( std::strchr( leaf.c_str(), '.' ) )
        {
          ++m_name_errors;
          error( library_name, full_path, loclink(full_path, string(name()))
              + " directory name contains a dot character ('.')" );
        }
      }
      //else // not a directory
      //{
      //  //  includes at most one dot character
      //  const char * first_dot = std::strchr( leaf.c_str(), '.' );
      //  if ( first_dot && std::strchr( first_dot+1, '.' ) )
      //  {
      //    ++m_name_errors;
      //    error( library_name, full_path, string(name())
      //        + " file name with more than one dot character ('.')" );
      //  }
      //}

      //  the path, including a presumed root, does not exceed the maximum size
      path const relative_path( relative_to( full_path, search_root_path() ) );
      const unsigned max_relative_path = 207; // ISO 9660:1999 sets this limit
      const string generic_root( "boost_X_XX_X/" );
      if ( relative_path.string().size() >
          ( max_relative_path - generic_root.size() ) )
      {
        ++m_name_errors;
        error( library_name, full_path,
            loclink(full_path, string(name()))
            + " path will exceed "
            + boost::lexical_cast<string>(max_relative_path)
            + " characters in a directory tree with a root in the form "
            + generic_root + ", and this exceeds ISO 9660:1999 limit of 207"  )
            ;
      }
    }
Beispiel #5
0
    void drillDown( path root, bool use_db = false, bool use_coll = false ) {
        log(2) << "drillDown: " << root.string() << endl;

        if ( is_directory( root ) ) {
            directory_iterator end;
            directory_iterator i(root);
            path indexes;
            while ( i != end ) {
                path p = *i;
                i++;

                if (use_db) {
                    if (is_directory(p)) {
                        cerr << "ERROR: root directory must be a dump of a single database" << endl;
                        cerr << "       when specifying a db name with --db" << endl;
                        printHelp(cout);
                        return;
                    }
                }

                if (use_coll) {
                    if (is_directory(p) || i != end) {
                        cerr << "ERROR: root directory must be a dump of a single collection" << endl;
                        cerr << "       when specifying a collection name with --collection" << endl;
                        printHelp(cout);
                        return;
                    }
                }

                if ( _indexesLast && p.leaf() == "system.indexes.bson" )
                    indexes = p;
                else
                    drillDown(p, use_db, use_coll);
            }

            if (!indexes.empty())
                drillDown(indexes, use_db, use_coll);

            return;
        }

        if ( ! ( endsWith( root.string().c_str() , ".bson" ) ||
                 endsWith( root.string().c_str() , ".bin" ) ) ) {
            cerr << "don't know what to do with [" << root.string() << "]" << endl;
            return;
        }

        log() << root.string() << endl;

        if ( root.leaf() == "system.profile.bson" ){
            log() << "\t skipping" << endl;
            return;
        }

        string ns;
        if (use_db) {
            ns += _db;
        } 
        else {
            string dir = root.branch_path().string();
            if ( dir.find( "/" ) == string::npos )
                ns += dir;
            else
                ns += dir.substr( dir.find_last_of( "/" ) + 1 );
            
            if ( ns.size() == 0 )
                ns = "test";
        }
        
        assert( ns.size() );

        if (use_coll) {
            ns += "." + _coll;
        } else {
            string l = root.leaf();
            l = l.substr( 0 , l.find_last_of( "." ) );
            ns += "." + l;
        }

        out() << "\t going into namespace [" << ns << "]" << endl;

        if ( _drop ){
            out() << "\t dropping" << endl;
            conn().dropCollection( ns );
        }
        
        _curns = ns.c_str();
        processFile( root );
    }
Beispiel #6
0
    void drillDown( path root, bool use_db = false ) {
        log(2) << "drillDown: " << root.string() << endl;

        if ( is_directory( root ) ) {
            directory_iterator end;
            directory_iterator i(root);
            while ( i != end ) {
                path p = *i;

                if (use_db) {
                    if (is_directory(p) ||
                        !(endsWith(p.string().c_str(), ".bson") ||
                          endsWith(p.string().c_str(), ".bin" ))) {
                        cerr << "ERROR: root directory must be a dump of a single database" << endl;
                        cerr << "       when specifying a db name with --db" << endl;
                        printHelp(cout);
                        return;
                    }
                }

                drillDown(p, use_db);
                i++;
            }
            return;
        }

        if ( ! ( endsWith( root.string().c_str() , ".bson" ) ||
                 endsWith( root.string().c_str() , ".bin" ) ) ) {
            cerr << "don't know what to do with [" << root.string() << "]" << endl;
            return;
        }

        out() << root.string() << endl;

        string ns;
        if (use_db) {
            ns += _db;
        } else {
            string dir = root.branch_path().string();
            if ( dir.find( "/" ) == string::npos )
                ns += dir;
            else
                ns += dir.substr( dir.find_last_of( "/" ) + 1 );
        }

        {
            string l = root.leaf();
            l = l.substr( 0 , l.find_last_of( "." ) );
            ns += "." + l;
        }

        long long fileLength = file_size( root );

        if ( fileLength == 0 ) {
            out() << "file " + root.native_file_string() + " empty, skipping" << endl;
            return;
        }

        out() << "\t going into namespace [" << ns << "]" << endl;

        string fileString = root.string();
        ifstream file( fileString.c_str() , ios_base::in | ios_base::binary);
        if ( ! file.is_open() ){
            log() << "error opening file: " << fileString << endl;
            return;
        }

        log(1) << "\t file size: " << fileLength << endl;

        long long read = 0;
        long long num = 0;

        int msgDelay = (int)(1000 * ( 1 + ( fileLength / ( 1024.0 * 1024 * 400 ) ) ) );
        log(1) << "\t msg delay: " << msgDelay << endl;

        const int BUF_SIZE = 1024 * 1024 * 5;
        char * buf = (char*)malloc( BUF_SIZE );

        while ( read < fileLength ) {
            file.read( buf , 4 );
            int size = ((int*)buf)[0];
            assert( size < BUF_SIZE );

            file.read( buf + 4 , size - 4 );

            BSONObj o( buf );
            conn().insert( ns.c_str() , o );

            read += o.objsize();
            num++;

            if ( ( logLevel > 0 && num < 10 ) || ! ( num % msgDelay ) )
                out() << "read " << read << "/" << fileLength << " bytes so far. (" << (int)( (read * 100) / fileLength) << "%) " << num << " objects" << endl;
        }

        free( buf );
        out() << "\t "  << num << " objects" << endl;
    }