bool load_from_stream(const std::string& filename, Fstream& fin,
const std::string& format) {
size_t linecount = 0;
std::vector< std::vector<std::string> > line_buffers;
std::vector<std::string> buffer;
buffer.reserve(max_buffer);
timer ti; ti.start();
while(fin.good() && !fin.eof()) {
std::string line;
std::getline(fin, line);
if(line.empty()) continue;
if(fin.fail()) break;
buffer.push_back(line);
++linecount;
if (buffer.size() == max_buffer) {
// add a worker
line_buffers.push_back(std::vector<std::string>());
line_buffers.back().swap(buffer);
pool.launch(boost::bind(&graph_loader::process_lines, this, line_buffers.back(), format, filename, linecount-buffer.size()+1));
}
}
// add the last worker
line_buffers.push_back(std::vector<std::string>());
line_buffers.back().swap(buffer);
pool.launch(boost::bind(&graph_loader::process_lines, this, line_buffers.back(), format, filename, linecount-buffer.size()+1));
pool.join();
line_buffers.clear();
return true;
}
bool load_from_stream(GraphType& g, std::string filename, Fstream& fin,
line_parser_type<GraphType>& line_parser) {
size_t linecount = 0;
timer ti; ti.start();
while(fin.good() && !fin.eof()) {
std::string line;
std::getline(fin, line);
if(line.empty()) continue;
if(fin.fail()) break;
const bool success = line_parser(g, filename, line);
if (!success) {
logstream(LOG_WARNING)
<< "Error parsing line " << linecount << " in "
<< filename << ": " << std::endl
<< "\t\"" << line << "\"" << std::endl;
return false;
}
++linecount;
if (ti.current_time() > 5.0) {
logstream(LOG_INFO) << linecount << " Lines read" << std::endl;
ti.start();
}
}
return true;
} // end of load from stream