FeltFile::FeltFile(const path & file)
: fileName_(file), changeEndianness_(false)
{
if ( ! exists(file) )
throw runtime_error("Cannot find file " + file.native_file_string() );
if ( is_directory(file) )
throw runtime_error(file.native_directory_string() + " is a directory, not a file");
string fileName = file.native_file_string();
feltFile_ = new boost::filesystem::ifstream(file);
word head;
feltFile_->read(reinterpret_cast<char*>(& head), sizeof(word));
if ( head < 997 or 999 < head )
changeEndianness_ = true;
// Block 1
block1_ = getBlock_(0);
if ( not complete() )
throw std::runtime_error("File update is not complete yet");
const size_t fieldCount = block1_[9];
fields_.reserve(fieldCount);
int i = 0;
while ( fields_.size() < fieldCount )
{
boost::shared_ptr<FeltField> f(new FeltField(*this, i ++));
if ( f->valid() )
fields_.push_back(f);
}
}
void drillDown( path root ) {
if ( is_directory( root ) ) {
directory_iterator end;
directory_iterator i(root);
while ( i != end ) {
path p = *i;
drillDown( p );
i++;
}
return;
}
if ( ! ( endsWith( root.string().c_str() , ".bson" ) ||
endsWith( root.string().c_str() , ".bin" ) ) ) {
cerr << "don't know what to do with [" << root.string() << "]" << endl;
return;
}
out() << root.string() << endl;
string ns;
{
string dir = root.branch_path().string();
if ( dir.find( "/" ) == string::npos )
ns += dir;
else
ns += dir.substr( dir.find_last_of( "/" ) + 1 );
}
{
string l = root.leaf();
l = l.substr( 0 , l.find_last_of( "." ) );
ns += "." + l;
}
if ( boost::filesystem::file_size( root ) == 0 ) {
out() << "file " + root.native_file_string() + " empty, aborting" << endl;
return;
}
out() << "\t going into namespace [" << ns << "]" << endl;
MemoryMappedFile mmf;
assert( mmf.map( root.string().c_str() ) );
char * data = (char*)mmf.viewOfs();
int read = 0;
int num = 0;
while ( read < mmf.length() ) {
BSONObj o( data );
conn().insert( ns.c_str() , o );
read += o.objsize();
data += o.objsize();
if ( ! ( ++num % 1000 ) )
out() << "read " << read << "/" << mmf.length() << " bytes so far. " << num << " objects" << endl;
}
out() << "\t " << num << " objects" << endl;
}
explicit basic_ofstream( const path & file_ph,
std::ios_base::openmode mode = std::ios_base::out )
: std::basic_ofstream<charT,traits>(
file_ph.native_file_string().c_str(), mode ) {}
void open( const path & file_ph,
std::ios_base::openmode mode )
{
std::basic_ifstream<charT,traits>::open(
file_ph.native_file_string().c_str(), mode );
}
explicit basic_ifstream( const path & file_ph )
: std::basic_ifstream<charT,traits>(
file_ph.native_file_string().c_str(), std::ios_base::in ) {}
void open( const path & file_ph )
{
std::basic_fstream<charT,traits>::open(
file_ph.native_file_string().c_str(),
std::ios_base::in|std::ios_base::out );
}
void drillDown( path root, bool use_db = false ) {
log(2) << "drillDown: " << root.string() << endl;
if ( is_directory( root ) ) {
directory_iterator end;
directory_iterator i(root);
while ( i != end ) {
path p = *i;
if (use_db) {
if (is_directory(p) ||
!(endsWith(p.string().c_str(), ".bson") ||
endsWith(p.string().c_str(), ".bin" ))) {
cerr << "ERROR: root directory must be a dump of a single database" << endl;
cerr << " when specifying a db name with --db" << endl;
printHelp(cout);
return;
}
}
drillDown(p, use_db);
i++;
}
return;
}
if ( ! ( endsWith( root.string().c_str() , ".bson" ) ||
endsWith( root.string().c_str() , ".bin" ) ) ) {
cerr << "don't know what to do with [" << root.string() << "]" << endl;
return;
}
out() << root.string() << endl;
string ns;
if (use_db) {
ns += _db;
} else {
string dir = root.branch_path().string();
if ( dir.find( "/" ) == string::npos )
ns += dir;
else
ns += dir.substr( dir.find_last_of( "/" ) + 1 );
}
{
string l = root.leaf();
l = l.substr( 0 , l.find_last_of( "." ) );
ns += "." + l;
}
long long fileLength = file_size( root );
if ( fileLength == 0 ) {
out() << "file " + root.native_file_string() + " empty, skipping" << endl;
return;
}
out() << "\t going into namespace [" << ns << "]" << endl;
string fileString = root.string();
ifstream file( fileString.c_str() , ios_base::in | ios_base::binary);
if ( ! file.is_open() ){
log() << "error opening file: " << fileString << endl;
return;
}
log(1) << "\t file size: " << fileLength << endl;
long long read = 0;
long long num = 0;
int msgDelay = (int)(1000 * ( 1 + ( fileLength / ( 1024.0 * 1024 * 400 ) ) ) );
log(1) << "\t msg delay: " << msgDelay << endl;
const int BUF_SIZE = 1024 * 1024 * 5;
char * buf = (char*)malloc( BUF_SIZE );
while ( read < fileLength ) {
file.read( buf , 4 );
int size = ((int*)buf)[0];
assert( size < BUF_SIZE );
file.read( buf + 4 , size - 4 );
BSONObj o( buf );
conn().insert( ns.c_str() , o );
read += o.objsize();
num++;
if ( ( logLevel > 0 && num < 10 ) || ! ( num % msgDelay ) )
out() << "read " << read << "/" << fileLength << " bytes so far. (" << (int)( (read * 100) / fileLength) << "%) " << num << " objects" << endl;
}
free( buf );
out() << "\t " << num << " objects" << endl;
}