int
main(int argc, char *argv[])
{
int sts;
int ctx;
int c;
char *infile;
int nfilelist;
int filenum;
char *archive = NULL;
int j;
char *buf;
fields_t *f;
char *s;
int gzipped;
FILE *fp;
metric_t *m;
handler_t *h;
int unhandled_metric_cnt = 0;
while ((c = pmGetOptions(argc, argv, &opts)) != EOF) {
switch (c) {
case 'F':
Fflag = 1;
break;
case 'v':
vflag++;
break;
}
}
nfilelist = argc - opts.optind - 1;
if (nfilelist < 1)
opts.errors++;
else
archive = argv[argc - 1];
if (opts.errors) {
pmUsageMessage(&opts);
exit(1);
}
if ((buf = malloc(BUFSIZE)) == NULL) {
perror("Error: out of memory:");
exit(1);
}
if (Fflag) {
snprintf(buf, BUFSIZE, "%s.meta", archive); unlink(buf);
snprintf(buf, BUFSIZE, "%s.index", archive); unlink(buf);
for (j=0;; j++) {
snprintf(buf, BUFSIZE, "%s.%d", archive, j);
if (unlink(buf) < 0)
break;
}
}
ctx = pmiStart(archive, 0);
if ((sts = pmiUseContext(ctx)) < 0) {
fprintf(stderr, "Error: pmiUseContext failed: %s\n", pmiErrStr(sts));
exit(1);
}
/*
* Define the metrics name space, see metrics.c (generated by pmdesc)
*/
for (m = metrics; m->name; m++) {
pmDesc *d = &m->desc;
sts = pmiAddMetric(m->name, d->pmid, d->type, d->indom, d->sem, d->units);
if (sts < 0) {
fprintf(stderr, "Error: failed to add metric %s: %s\n", m->name, pmiErrStr(sts));
exit(1);
}
}
/*
* Populate special case instance domains
*/
pmiAddInstance(pmInDom_build(LINUX_DOMAIN, LOADAVG_INDOM), "1 minute", 1);
pmiAddInstance(pmInDom_build(LINUX_DOMAIN, LOADAVG_INDOM), "5 minute", 5);
pmiAddInstance(pmInDom_build(LINUX_DOMAIN, LOADAVG_INDOM), "15 minute", 15);
indom_cnt[LOADAVG_INDOM] = 3;
for (filenum=0; filenum < nfilelist; filenum++) {
infile = argv[opts.optind + filenum];
gzipped = strstr(infile, ".gz") != NULL;
if (gzipped) {
snprintf(buf, BUFSIZE, "gzip -c -d %s", infile);
if ((fp = popen(buf, "r")) == NULL)
perror(buf);
}
else
if ((fp = fopen(infile, "r")) == NULL)
perror(infile);
if (fp == NULL) {
pmUsageMessage(&opts);
exit(1);
}
/*
* parse the header
*/
sts = header_handler(fp, infile, buf, BUFSIZE);
/*
* Parse remaining data stream for this input file
*/
while(fgets(buf, BUFSIZE, fp)) {
if ((s = strrchr(buf, '\n')) != NULL)
*s = '\0';
if (!buf[0])
continue;
f = fields_new(buf, strlen(buf)+1);
if (f->nfields > 0) {
if ((h = find_handler(f->fields[0])) == NULL) {
unhandled_metric_cnt++;
if (vflag > 1)
printf("Unhandled tag: \"%s\"\n", f->fields[0]);
}
else {
sts = h->handler(h, f);
if (sts < 0 && h->handler == timestamp_handler) {
fprintf(stderr, "Error: %s\n", pmiErrStr(sts));
exit(1);
}
}
}
fields_free(f);
}
/* final flush for this file */
if ((sts = timestamp_flush()) < 0) {
fprintf(stderr, "Error: failed to write final timestamp: %s\n", pmiErrStr(sts));
exit(1);
}
if (gzipped)
pclose(fp);
else
fclose(fp);
}
sts = pmiEnd();
if (unhandled_metric_cnt && vflag)
fprintf(stderr, "Warning: %d unhandled metric/values\n", unhandled_metric_cnt);
exit(0);
}
int main(int argc, char* argv[])
{
std::string server;
std::string port;
std::string user;
std::string folder;
std::string password;
std::vector<fs::path> from;
std::vector<fs::path> entity;
fs::path to;
//[Gmail]/Sent Mail
po::options_description general_options("General");
general_options.add_options()
("help", "list options");
po::options_description file_options("Load");
file_options.add_options()
("save-raw", po::value<fs::path>(&to), "path to save the data (after download phase)");
po::options_description download_options("Download");
download_options.add_options()
("server", po::value<std::string>(&server), "imap server dns/ip")
("port", po::value<std::string>(&port)->default_value("993"), "imap port")
("folder", po::value<std::string>(&folder)->default_value("Sent"), "imap folder")
("user", po::value<std::string>(&user), "imap username")
("password", po::value<std::string>(&password), "imap password (will ask if not specified)");
po::options_description run_options("Run");
po::options_description all_options("Email Topology Options");
all_options
.add(general_options)
.add(file_options)
.add(download_options);
if(argc < 2) {
std::cout << all_options << std::endl;
return 1;
}
po::variables_map vm;
try {
int options_style = po::command_line_style::default_style;
po::store(po::parse_command_line(argc, argv, all_options, options_style), vm);
po::notify(vm);
} catch(std::exception& e) {
std::cout << all_options << std::endl;
std::cout << "Command line parsing failed: " << e.what() << std::endl;
return 1;
}
if(vm.count("help")) {
std::cout << all_options << std::endl;
return 1;
}
email_id_bimap email_id;
connectedness_graph cg;
entity_map em;
initial_group_partition_map igpm;
if(!vm.count("save-raw")) {
std::cout << "you must specify --save-raw with a file name" << std::endl;
return 1;
}
if(!vm.count("password")) {
password = getpass("Password: "******"missing server for download" << std::endl;
return 1;
}
if(user.empty()) {
std::cout << "missing user for download" << std::endl;
return 1;
}
if(password.empty()) {
std::cout << "missing user for download" << std::endl;
return 1;
}
//this is our network block, downloads all messages headers
try
{
std::cout << "downloading " << folder << " from " << server << std::endl;
//use to dedupe if there are dupes
message_id_set message_id;
typedef boost::function<void (const std::string&, const std::list<std::string>& args)> untagged_handler;
std::string pending_tag = "* ";
std::list<std::string> pending_command;
pending_command.push_back("WAIT_FOR_ACK");
untagged_handler pending_handler;
unsigned int command_id = 0;
//The sequence of imap commands we want to run
std::list<std::list<std::string> > commands;
std::list<untagged_handler> handlers;
handlers.push_back(log_handler());
commands.push_back(std::list<std::string>());
std::ostringstream login_os;
login_os << "LOGIN \"" << user << "\" {" << password.size() << "}";
commands.back().push_back(login_os.str());
commands.back().push_back(password);
handlers.push_back(log_handler());
commands.push_back(std::list<std::string>());
commands.back().push_back("LIST \"\" *");
handlers.push_back(log_handler());
commands.push_back(std::list<std::string>());
commands.back().push_back("SELECT \"" + folder + "\"");
handlers.push_back(header_handler(email_id, cg, em, message_id, igpm));
commands.push_back(std::list<std::string>());
commands.back().push_back("FETCH 1:* (BODY.PEEK[HEADER.FIELDS (MESSAGE-ID FROM TO CC)])");
commands.push_back(std::list<std::string>());
handlers.push_back(log_handler());
commands.back().push_back("LOGOUT");
//open ssl connection to the server, no cert checking
asio::io_service io_service;
asio::ip::tcp::resolver resolver(io_service);
asio::ip::tcp::resolver::query query(server, port);
asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query);
asio::ssl::context context(io_service, asio::ssl::context::sslv23);
context.set_verify_mode(asio::ssl::context::verify_none);
asio::ssl::stream<asio::ip::tcp::socket> socket(io_service, context);
socket.lowest_layer().connect(*iterator);
socket.handshake(asio::ssl::stream_base::client);
asio::streambuf buf;
while(true) {
//read the next line of data
std::size_t line_length = asio::read_until(socket, buf, re_crlf);
std::string line(
asio::buffers_begin(buf.data()),
asio::buffers_begin(buf.data()) + line_length);
buf.consume(line_length);
boost::match_results<std::string::iterator> what;
std::size_t initial = 0;
std::list<std::string> args;
//the line may be split into segments with chunks of data embedded, this is the case
//for bodies or message header blocks that are returned, we only handle this case if it
//comes in untagged response (*) not a continuation (+), i think that is normal
while(regex_search(line.begin() + initial, line.end(), what, re_byte_buffer, boost::match_default)) {
unsigned int bytes = boost::lexical_cast<unsigned int>(what[1].str());
if(buf.size() < bytes)
asio::read(socket, buf, asio::transfer_at_least(bytes - buf.size()));
args.push_back(
std::string(
asio::buffers_begin(buf.data()),
asio::buffers_begin(buf.data()) + bytes));
buf.consume(bytes);
line.resize(what[1].second - line.begin());
initial = line.size();
//read the next line of data
line_length = asio::read_until(socket, buf, re_crlf);
line += std::string(
asio::buffers_begin(buf.data()),
asio::buffers_begin(buf.data()) + line_length);
buf.consume(line_length);
}
if(boost::algorithm::starts_with(line, pending_tag)) {
//if the command is being completed, then we will go here, bail out if the response wasn't ok
if(!boost::algorithm::starts_with(line, pending_tag + "OK")) {
std::cout << line;
throw std::runtime_error("command failed");
}
//pull the next command off the list
pending_tag = "A" + boost::lexical_cast<std::string>(command_id++) + " ";
if(commands.size() == 0)
break;
pending_handler = handlers.front();
pending_command = commands.front();
commands.pop_front();
handlers.pop_front();
//send the command along with any data arguments
std::cout << pending_tag << pending_command.front() << std::endl;
asio::write(socket, asio::buffer(pending_tag.data(), pending_tag.size()));
for(std::list<std::string>::iterator i = pending_command.begin(); i != pending_command.end(); ++i) {
if(i != pending_command.begin()) {
//print the continuation response
std::size_t line_length = asio::read_until(socket, buf, re_crlf);
std::string line(
asio::buffers_begin(buf.data()),
asio::buffers_begin(buf.data()) + line_length);
buf.consume(line_length);
std::cout << line << std::flush;
if(!boost::algorithm::starts_with(line, "+ ")) {
throw std::runtime_error("bad response when writing extra data");
}
} else {
//print it out as well (but not the args)
std::cout << *i << std::endl;
}
asio::write(socket, asio::buffer(i->data(), i->size()));
asio::write(socket, asio::buffer("\r\n", 2));
}
} else if(boost::algorithm::starts_with(line, "* ")) {
//if there is a registered handler, dispatch to it
if(pending_handler)
pending_handler(line, args);
} else {
throw std::runtime_error("unrecognized response");
}
}
}
catch (std::exception& e) {
std::cout << "Exception: " << e.what() << std::endl;
return 1;
}
std::cout << std::endl;
if(to.empty()) {
std::cout << "Missing output file for save" << std::endl;
return 1;
}
if(fs::exists(to))
fs::remove(to);
fs::ofstream out(to);
std::cout << "saving data to " << to.file_string();
for(connectedness_graph::vertex_iterator i = gr::vertices(cg).first; i != gr::vertices(cg).second; ++i) {
out << (unsigned long long)cg[*i].weight;
for(members_t::iterator j = cg[*i].members.begin(); j != cg[*i].members.end(); ++j) {
out << "\t" << email_id.by<bit>().equal_range(*j).first->second;
}
out << std::endl;
}
out << "-" << std::endl;
for(entity_map::iterator i = em.begin(); i != em.end(); ++i) {
out << i->first;
for(std::set<std::string>::iterator k = i->second.begin(); k != i->second.end(); ++k) {
out << "\t" << *k;
}
out << std::endl;
}
return 0;
}
