bool load_from_stream(const std::string& filename, Fstream& fin,
const std::string& format) {
size_t linecount = 0;
std::vector< std::vector<std::string> > line_buffers;
std::vector<std::string> buffer;
buffer.reserve(max_buffer);
timer ti; ti.start();
while(fin.good() && !fin.eof()) {
std::string line;
std::getline(fin, line);
if(line.empty()) continue;
if(fin.fail()) break;
buffer.push_back(line);
++linecount;
if (buffer.size() == max_buffer) {
// add a worker
line_buffers.push_back(std::vector<std::string>());
line_buffers.back().swap(buffer);
pool.launch(boost::bind(&graph_loader::process_lines, this, line_buffers.back(), format, filename, linecount-buffer.size()+1));
}
}
// add the last worker
line_buffers.push_back(std::vector<std::string>());
line_buffers.back().swap(buffer);
pool.launch(boost::bind(&graph_loader::process_lines, this, line_buffers.back(), format, filename, linecount-buffer.size()+1));
pool.join();
line_buffers.clear();
return true;
}
bool load_from_stream(GraphType& g, std::string filename, Fstream& fin,
line_parser_type<GraphType>& line_parser) {
size_t linecount = 0;
timer ti; ti.start();
while(fin.good() && !fin.eof()) {
std::string line;
std::getline(fin, line);
if(line.empty()) continue;
if(fin.fail()) break;
const bool success = line_parser(g, filename, line);
if (!success) {
logstream(LOG_WARNING)
<< "Error parsing line " << linecount << " in "
<< filename << ": " << std::endl
<< "\t\"" << line << "\"" << std::endl;
return false;
}
++linecount;
if (ti.current_time() > 5.0) {
logstream(LOG_INFO) << linecount << " Lines read" << std::endl;
ti.start();
}
}
return true;
} // end of load from stream
static void do_open(Fstream& s,
const Path& path,
const std::ios_base::openmode mode,
typename boost::enable_if<fs::is_basic_path<Path> >::type* = 0)
{
try
{
if (!allow_read(path))
throw system_error(error_code(EACCES, get_system_category()));
stream_do_open(s, path, mode);
if (!s.is_open())
throw system_error(error_code(errno, get_system_category()));
}
catch (const system_error& e)
{
string what(e.std::runtime_error::what());
if (!what.empty())
what = ": " + what;
what = rwu_error_message(s) + what;
throw fs::filesystem_error(what, path, e.code());
}
}
static void stream_do_open(Fstream& s,
const fs::path& path,
const std::ios_base::openmode mode,
typename boost::enable_if<is_std_fstream<Fstream> >::type* = 0)
{
string file(path.external_file_string());
s.open(file.c_str(), mode);
}
static void stream_do_open(Fstream& s,
const Path& path,
const std::ios_base::openmode mode,
typename boost::enable_if<fs::is_basic_path<Path> >::type* = 0,
typename boost::enable_if<is_fs_fstream<Fstream> >::type* = 0)
{
s.open(path, mode);
}