int main(int argc, char *argv[]) { bool didhelp = false; feature_recorder::set_main_threadid(); #ifdef BROKEN std::cerr << "WARNING: YOU ARE USING AN EXPERIMENTAL VERSION OF TCPFLOW \n"; std::cerr << "THAT DOES NOT WORK PROPERLY. PLEASE USE A RELEASE DOWNLOADED\n"; std::cerr << "FROM http://digitalcorpora.org/downloads/tcpflow\n"; std::cerr << "\n"; #endif bool force_binary_output = false; const char *device = "<default>"; const char *lockname = 0; int need_usage = 0; std::string reportfilename; std::vector<std::string> Rfiles; // files for finishing std::vector<std::string> rfiles; // files to read tcpdemux &demux = *tcpdemux::getInstance(); // the demux object we will be using. std::string command_line = dfxml_writer::make_command_line(argc,argv); std::string opt_unk_packets; bool opt_quiet = false; /* Set up debug system */ progname = argv[0]; init_debug(progname,1); /* Make sure that the system was compiled properly */ if(sizeof(struct be13::ip4)!=20 || sizeof(struct be13::tcphdr)!=20){ fprintf(stderr,"COMPILE ERROR.\n"); fprintf(stderr," sizeof(struct ip)=%d; should be 20.\n", (int)sizeof(struct be13::ip4)); fprintf(stderr," sizeof(struct tcphdr)=%d; should be 20.\n", (int)sizeof(struct be13::tcphdr)); fprintf(stderr,"CANNOT CONTINUE\n"); exit(1); } bool trailing_input_list = false; int arg; while ((arg = getopt(argc, argv, "aA:Bb:cCd:DE:e:E:F:f:Hhi:JlL:m:o:pqR:r:S:sT:Vvw:x:X:Z")) != EOF) { switch (arg) { case 'a': demux.opt.post_processing = true; demux.opt.opt_md5 = true; be13::plugin::scanners_enable_all(); break; case 'A': fprintf(stderr,"-AH has been deprecated. Just use -a\n"); need_usage=true; break; case 'b': demux.opt.max_bytes_per_flow = atoi(optarg); if(debug > 1) { std::cout << "capturing max of " << demux.opt.max_bytes_per_flow << " bytes per flow." << std::endl; } break; case 'B': force_binary_output = true; demux.opt.output_strip_nonprint = false; DEBUG(10) ("converting non-printable characters to '.'"); break; case 'C': demux.opt.console_output = true; DEBUG(10) ("printing packets to console only"); demux.opt.suppress_header = 1; DEBUG(10) ("packet header dump suppressed"); break; case 'c': demux.opt.console_output = true; DEBUG(10) ("printing packets to console only"); break; case 'd': if ((debug = atoi(optarg)) < 0) { debug = DEFAULT_DEBUG_LEVEL; DEBUG(1) ("warning: -d flag with 0 debug level '%s'", optarg); } break; case 'D': demux.opt.output_hex = true;DEBUG(10) ("Console output in hex"); demux.opt.output_strip_nonprint = false; DEBUG(10) ("Will not convert non-printablesto '.'"); break; case 'E': be13::plugin::scanners_disable_all(); be13::plugin::scanners_enable(optarg); break; case 'e': be13::plugin::scanners_enable(optarg); demux.opt.post_processing = true; // enable post processing if anything is turned on break; case 'F': for(const char *cc=optarg;*cc;cc++){ switch(*cc){ case 'c': replace(flow::filename_template,"%c","%C"); break; case 'k': flow::filename_template = "%K/" + flow::filename_template; break; case 'm': flow::filename_template = "%M000-%M999/%M%K/" + flow::filename_template; break; case 'g': flow::filename_template = "%G000000-%G999999/%G%M000-%G%M999/%G%M%K/" + flow::filename_template; break; case 't': flow::filename_template = "%tT" + flow::filename_template; break; case 'T': flow::filename_template = "%T" + flow::filename_template; break; case 'X': demux.opt.store_output = false;break; case 'M': demux.opt.opt_md5 = true;break; default: fprintf(stderr,"-F invalid format specification '%c'\n",*cc); need_usage = true; } } break; case 'f': { int mnew = atoi(optarg); DEBUG(1)("changing max_fds from %d to %d",demux.max_fds,mnew); demux.max_fds = mnew; break; } case 'i': device = optarg; break; case 'J': demux.opt.use_color = 1; DEBUG(10) ("using colors"); break; case 'l': trailing_input_list = true; break; case 'L': lockname = optarg; break; case 'm': demux.opt.max_seek = atoi(optarg); DEBUG(10) ("max_seek set to %d",demux.opt.max_seek); break; case 'o': demux.outdir = optarg; flow::outdir = optarg; break; case 'p': opt_no_promisc = true; DEBUG(10) ("NOT turning on promiscuous mode"); break; case 'q': opt_quiet = true; break; case 'R': Rfiles.push_back(optarg); break; case 'r': rfiles.push_back(optarg); break; case 'S': { std::vector<std::string> params = split(optarg,'='); if(params.size()!=2){ std::cerr << "Invalid paramter: " << optarg << "\n"; exit(1); } be_config.namevals[params[0]] = params[1]; continue; } case 's': demux.opt.output_strip_nonprint = 1; DEBUG(10) ("converting non-printable characters to '.'"); break; case 'T': flow::filename_template = optarg; if(flow::filename_template.find("%c")==string::npos){ flow::filename_template += std::string("%C%c"); // append %C%c if not present } break; case 'V': std::cout << PACKAGE_NAME << " " << PACKAGE_VERSION << "\n"; exit (1); case 'v': debug = 10; break; case 'w': opt_unk_packets = optarg;break; case 'x': be13::plugin::scanners_disable(optarg);break; case 'X': reportfilename = optarg;break; case 'Z': demux.opt.gzip_decompress = 0; break; case 'H': be13::plugin::info_scanners(true,true,scanners_builtin,'e','x'); didhelp = true; break; case 'h': case '?': usage(); didhelp = true; break; default: DEBUG(1) ("error: unrecognized switch '%c'", arg); need_usage = 1; break; } } if(didhelp) exit(0); if(demux.opt.post_processing && !demux.opt.store_output){ std::cerr << "ERROR: post_processing currently requires storing output.\n"; exit(1); } argc -= optind; argv += optind; /* Load all the scanners and enable the ones we care about */ if(demux.opt.opt_md5) be13::plugin::scanners_enable("md5"); be13::plugin::load_scanners(scanners_builtin,be_config); be13::plugin::scanners_process_enable_disable_commands(); /* If there is no report filename, call it report.xml in the output directory */ if( reportfilename.size()==0 ){ reportfilename = demux.outdir + "/" + DEFAULT_REPORT_FILENAME; } /* print help and exit if there was an error in the arguments */ if (need_usage) { usage(); exit(1); } /* remaining arguments are either an input list (-l flag) or a pcap expression (default) */ std::string expression = ""; if(trailing_input_list) { for(int ii = 0; ii < argc; ii++) { rfiles.push_back(argv[ii]); } } else { /* get the user's expression out of remainder of the arg... */ for(int i=0;i<argc;i++){ if(expression.size()>0) expression+=" "; expression += argv[i]; } } /* More option processing */ /* was a semaphore provided for the lock? */ if(lockname){ #if defined(HAVE_SEMAPHORE_H) && defined(HAVE_PTHREAD) semlock = sem_open(lockname,O_CREAT,0777,1); // get the semaphore #else fprintf(stderr,"%s: attempt to create lock pthreads not present\n",argv[0]); exit(1); #endif } if(force_binary_output) demux.opt.output_strip_nonprint = false; /* make sure outdir is a directory. If it isn't, try to make it.*/ struct stat stbuf; if(stat(demux.outdir.c_str(),&stbuf)==0){ if(!S_ISDIR(stbuf.st_mode)){ std::cerr << "outdir is not a directory: " << demux.outdir << "\n"; exit(1); } } else { if(MKDIR(demux.outdir.c_str(),0777)){ std::cerr << "cannot create " << demux.outdir << ": " << strerror(errno) << "\n"; exit(1); } } std::string input_fname; if(rfiles.size() > 0) { input_fname = rfiles.at(0); if(rfiles.size() > 1) { input_fname += ssprintf(" + %d more", rfiles.size() - 1); } } /* report file specified? */ if(reportfilename.size()>0){ xreport = new dfxml_writer(reportfilename,false); dfxml_create(*xreport,command_line); demux.xreport = xreport; } if(opt_unk_packets.size()>0){ if(input_fname.size()==0){ std::cerr << "currently the -w option requires the -r option\n"; exit(1); } if(access(input_fname.c_str(),R_OK)) die("cannot read: %s: %s",input_fname.c_str(),strerror(errno)); demux.save_unk_packets(opt_unk_packets,input_fname); } scanner_info si; si.config = &be_config; /* Debug prefix set? */ std::string debug_prefix=progname; si.get_config("debug-prefix",&debug_prefix,"Prefix for debug output"); init_debug(debug_prefix.c_str(),0); argc -= optind; argv += optind; DEBUG(10) ("%s version %s ", PACKAGE_NAME, PACKAGE_VERSION); feature_file_names_t feature_file_names; be13::plugin::get_scanner_feature_file_names(feature_file_names); feature_recorder_set fs(0); fs.init(feature_file_names,input_fname.size()>0 ? input_fname : device,demux.outdir,0); the_fs = &fs; demux.fs = &fs; si.get_config("tdelta",&datalink_tdelta,"Time offset for packets"); if(demux.xreport) demux.xreport->xmlout("tdelta",datalink_tdelta); /* Process r files and R files */ if(rfiles.size()==0 && Rfiles.size()==0){ /* live capture */ #if defined(HAVE_SETUID) && defined(HAVE_GETUID) /* Since we don't need network access, drop root privileges */ if(setuid(getuid())){ perror("setuid"); } #endif demux.start_new_connections = true; process_infile(expression,device,""); input_fname = device; } else { /* first pick up the new connections with -r */ demux.start_new_connections = true; for(std::vector<std::string>::const_iterator it=rfiles.begin();it!=rfiles.end();it++){ process_infile(expression,device,*it); } /* now pick up the outstanding connection with -R, but don't start new connections */ demux.start_new_connections = false; for(std::vector<std::string>::const_iterator it=Rfiles.begin();it!=Rfiles.end();it++){ process_infile(expression,device,*it); } } /* -1 causes pcap_loop to loop forever, but it finished when the input file is exhausted. */ DEBUG(2)("Open FDs at end of processing: %d",(int)demux.open_flows.size()); DEBUG(2)("demux.max_open_flows: %d",(int)demux.max_open_flows); DEBUG(2)("Flow map size at end of processing: %d",(int)demux.flow_map.size()); DEBUG(2)("Flows seen: %d",(int)demux.flow_counter); demux.close_all_fd(); be13::plugin::phase_shutdown(fs); /* * Note: funny formats below are a result of mingw problems with PRId64. */ const std::string total_flow_processed("Total flows processed: %" PRId64); const std::string total_packets_processed("Total packets processed: %" PRId64); DEBUG(2)(total_flow_processed.c_str(),demux.flow_counter); DEBUG(2)(total_packets_processed.c_str(),demux.packet_counter); if(xreport){ demux.remove_all_flows(); // empty the map to capture the state xreport->add_rusage(); xreport->pop(); // bulk_extractor xreport->close(); delete xreport; } if(demux.flow_counter > tcpdemux::WARN_TOO_MANY_FILES){ if(!opt_quiet){ /* Start counting how many files we have in the output directory. * If we find more than 10,000, print the warning, and keep counting... */ uint64_t filecount=0; DIR *dirp = opendir(demux.outdir.c_str()); if(dirp){ struct dirent *dp=0; while((dp=readdir(dirp))!=NULL){ filecount++; if(filecount==10000){ std::cerr << "*** tcpflow WARNING:\n"; std::cerr << "*** Modern operating systems do not perform well \n"; std::cerr << "*** with more than 10,000 entries in a directory.\n"; std::cerr << "***\n"; } } closedir(dirp); } if(filecount>=10000){ std::cerr << "*** tcpflow created " << filecount << " files in output directory " << demux.outdir << "\n"; std::cerr << "***\n"; std::cerr << "*** Next time, specify command-line options: -Fk , -Fm , or -Fg \n"; std::cerr << "*** This will automatically bin output into subdirectories.\n"; std::cerr << "*** type 'tcpflow -hhh' for more information.\n"; } } } exit(0); // return(0) causes crash on Windows }
int main(int argc,char **argv) { #ifdef HAVE_MCHECK mtrace(); #endif /* setup */ feature_recorder::set_main_threadid(); const char *progname = argv[0]; word_and_context_list alert_list; /* shold be flagged */ word_and_context_list stop_list; /* should be ignored */ scanner_info::scanner_config s_config; // the bulk extractor phase 1 config created from the command line BulkExtractor_Phase1::Config cfg; cfg.num_threads = threadpool::numCPU(); /* Options */ const char *opt_path = 0; int opt_recurse = 0; int opt_zap = 0; int opt_h = 0; int opt_H = 0; std::string opt_sampling_params; std::string opt_outdir; bool opt_write_feature_files = true; bool opt_write_sqlite3 = false; bool opt_enable_histograms=true; /* Startup */ setvbuf(stdout,0,_IONBF,0); // don't buffer stdout std::string command_line = dfxml_writer::make_command_line(argc,argv); std::vector<std::string> scanner_dirs; // where to look for scanners /* Add the default plugin_path */ add_if_present(scanner_dirs,"/usr/local/lib/bulk_extractor"); add_if_present(scanner_dirs,"/usr/lib/bulk_extractor"); add_if_present(scanner_dirs,"."); if (getenv("BE_PATH")) { std::vector<std::string> dirs = split(getenv("BE_PATH"),':'); for(std::vector<std::string>::const_iterator it = dirs.begin(); it!=dirs.end(); it++){ add_if_present(scanner_dirs,*it); } } #ifdef WIN32 setmode(1,O_BINARY); // make stdout binary threadpool::win32_init(); #endif /* look for usage first */ if(argc==1) opt_h=1; /* Process options */ int ch; while ((ch = getopt(argc, argv, "A:B:b:C:d:E:e:F:f:G:g:Hhij:M:m:o:P:p:q:Rr:S:s:VW:w:x:Y:z:Z")) != -1) { switch (ch) { case 'A': feature_recorder::offset_add = stoi64(optarg);break; case 'b': feature_recorder::banner_file = optarg; break; case 'C': feature_recorder::context_window_default = atoi(optarg);break; case 'd': { if(strcmp(optarg,"h")==0) debug_help(); int d = atoi(optarg); switch(d){ case DEBUG_ALLOCATE_512MiB: if(calloc(1024*1024*512,1)){ std::cerr << "-d1002 -- Allocating 512MB of RAM; may be repeated\n"; } else { std::cerr << "-d1002 -- CANNOT ALLOCATE MORE RAM\n"; } break; default: cfg.debug = d; break; } be13::plugin::set_scanner_debug(cfg.debug); } break; case 'E': be13::plugin::scanners_disable_all(); be13::plugin::scanners_enable(optarg); break; case 'e': be13::plugin::scanners_enable(optarg); break; case 'F': FindOpts::get().Files.push_back(optarg); break; case 'f': FindOpts::get().Patterns.push_back(optarg); break; case 'G': cfg.opt_pagesize = scaled_stoi64(optarg); break; case 'g': cfg.opt_marginsize = scaled_stoi64(optarg); break; case 'i': std::cout << "info mode:\n"; cfg.opt_info = true; break; case 'j': cfg.num_threads = atoi(optarg); break; case 'M': scanner_def::max_depth = atoi(optarg); break; case 'm': cfg.max_bad_alloc_errors = atoi(optarg); break; case 'o': opt_outdir = optarg;break; case 'P': scanner_dirs.push_back(optarg);break; case 'p': opt_path = optarg; break; case 'q': if(atoi(optarg)==-1) cfg.opt_quiet = 1;// -q -1 turns off notifications else cfg.opt_notify_rate = atoi(optarg); break; case 'r': if(alert_list.readfile(optarg)){ err(1,"Cannot read alert list %s",optarg); } break; case 'R': opt_recurse = 1; break; case 'S': { std::vector<std::string> params = split(optarg,'='); if(params.size()!=2){ std::cerr << "Invalid paramter: " << optarg << "\n"; exit(1); } s_config.namevals[params[0]] = params[1]; continue; } case 's': #if defined(HAVE_SRANDOM) && !defined(HAVE_SRANDOMDEV) srandom(time(0)); #endif #if defined(HAVE_SRANDOMDEV) srandomdev(); // if we are sampling initialize #endif opt_sampling_params = optarg; break; case 'V': std::cout << "bulk_extractor " << PACKAGE_VERSION << "\n"; exit (1); case 'W': fprintf(stderr,"-W has been deprecated. Specify with -S word_min=NN and -S word_max=NN\n"); exit(1); break; case 'w': if(stop_list.readfile(optarg)){ err(1,"Cannot read stop list %s",optarg); } break; case 'x': be13::plugin::scanners_disable(optarg); break; case 'Y': { std::string optargs = optarg; size_t dash = optargs.find('-'); if(dash==std::string::npos){ cfg.opt_offset_start = stoi64(optargs); } else { cfg.opt_offset_start = scaled_stoi64(optargs.substr(0,dash)); cfg.opt_offset_end = scaled_stoi64(optargs.substr(dash+1)); } break; } case 'z': cfg.opt_page_start = stoi64(optarg);break; case 'Z': opt_zap=true;break; case 'H': opt_H++;continue; case 'h': opt_h++;continue; } } cfg.validate(); argc -= optind; argv += optind; if(cfg.debug & DEBUG_PRINT_STEPS) std::cerr << "DEBUG: DEBUG_PRINT_STEPS\n"; if(cfg.debug & DEBUG_PEDANTIC) validateOrEscapeUTF8_validate = true; /* Create a configuration that will be used to initialize the scanners */ scanner_info si; s_config.debug = cfg.debug; si.config = &s_config; /* Make individual configuration options appear on the command line interface. */ si.get_config("work_start_work_end",&worker::opt_work_start_work_end, "Record work start and end of each scanner in report.xml file"); si.get_config("enable_histograms",&opt_enable_histograms, "Disable generation of histograms"); si.get_config("debug_histogram_malloc_fail_frequency",&HistogramMaker::debug_histogram_malloc_fail_frequency, "Set >0 to make histogram maker fail with memory allocations"); si.get_config("hash_alg",&be_hash_name,"Specifies hash algorithm to be used for all hash calculations"); si.get_config("dup_data_alerts",&be13::plugin::dup_data_alerts,"Notify when duplicate data is not processed"); si.get_config("write_feature_files",&opt_write_feature_files,"Write features to flat files"); si.get_config("write_feature_sqlite3",&opt_write_sqlite3,"Write feature files to report.sqlite3"); /* Make sure that the user selected a valid hash */ { uint8_t buf[1]; be_hash_func(buf,0); } /* Load all the scanners and enable the ones we care about */ be13::plugin::load_scanner_directories(scanner_dirs,s_config); be13::plugin::load_scanners(scanners_builtin,s_config); be13::plugin::scanners_process_enable_disable_commands(); /* Print usage if necessary */ if(opt_H){ be13::plugin::info_scanners(true,true,scanners_builtin,'e','x'); exit(0);} if(opt_h){ usage(progname);be13::plugin::info_scanners(false,true,scanners_builtin,'e','x'); exit(0);} /* Give an error if a find list was specified * but no scanner that uses the find list is enabled. */ if(!FindOpts::get().empty()) { /* Look through the enabled scanners and make sure that * at least one of them is a FIND scanner */ if(!be13::plugin::find_scanner_enabled()){ errx(1,"find words are specified with -F but no find scanner is enabled.\n"); } } if(opt_path){ if(argc!=1) errx(1,"-p requires a single argument."); process_path(argv[0],opt_path,cfg.opt_pagesize,cfg.opt_marginsize); exit(0); } if(opt_outdir.size()==0) errx(1,"error: -o outdir must be specified"); /* The zap option wipes the contents of a directory, useful for debugging */ if(opt_zap){ DIR *dirp = opendir(opt_outdir.c_str()); if(dirp){ struct dirent *dp; while ((dp = readdir(dirp)) != NULL){ std::string name = dp->d_name; if(name=="." || name=="..") continue; std::string fname = opt_outdir + std::string("/") + name; unlink(fname.c_str()); std::cout << "erasing " << fname << "\n"; } } if(rmdir(opt_outdir.c_str())){ std::cout << "rmdir " << opt_outdir << "\n"; } } /* Start the clock */ aftimer timer; timer.start(); /* If output directory does not exist, we are not restarting! */ std::string reportfilename = opt_outdir + "/report.xml"; BulkExtractor_Phase1::seen_page_ids_t seen_page_ids; // pages that do not need re-processing image_process *p = 0; // the image process iterator /* Get image or directory */ if (*argv == NULL) { if (opt_recurse) { fprintf(stderr,"filedir not provided\n"); } else { fprintf(stderr,"imagefile not provided\n"); } exit(1); } std::string image_fname = *argv; if(opt_outdir.size()==0){ fprintf(stderr,"output directory not provided\n"); exit(1); } if(directory_missing(opt_outdir) || directory_empty(opt_outdir)){ /* First time running */ /* Validate the args */ if ( argc !=1 ) errx(1,"Disk image option not provided. Run with -h for help."); validate_fn(image_fname); if (directory_missing(opt_outdir)) be_mkdir(opt_outdir); } else { /* Restarting */ std::cout << "Restarting from " << opt_outdir << "\n"; bulk_extractor_restarter r(opt_outdir,reportfilename,image_fname,seen_page_ids); /* Rename the old report and create a new one */ std::string old_reportfilename = reportfilename + "." + itos(time(0)); if(rename(reportfilename.c_str(),old_reportfilename.c_str())){ std::cerr << "Could not rename " << reportfilename << " to " << old_reportfilename << ": " << strerror(errno) << "\n"; exit(1); } } /* Open the image file (or the device) now */ p = image_process::open(image_fname,opt_recurse,cfg.opt_pagesize,cfg.opt_marginsize); if(!p) err(1,"Cannot open %s: ",image_fname.c_str()); /*** *** Create the feature recording set. *** Initialize the scanners. ****/ /* Determine the feature files that will be used */ feature_file_names_t feature_file_names; be13::plugin::get_scanner_feature_file_names(feature_file_names); uint32_t flags = 0; if (stop_list.size()>0) flags |= feature_recorder_set::CREATE_STOP_LIST_RECORDERS; if (opt_write_sqlite3) flags |= feature_recorder_set::ENABLE_SQLITE3_RECORDERS; if (!opt_write_feature_files) flags |= feature_recorder_set::DISABLE_FILE_RECORDERS; { feature_recorder_set fs(flags,be_hash,image_fname,opt_outdir); fs.init(feature_file_names); if(opt_enable_histograms) be13::plugin::add_enabled_scanner_histograms_to_feature_recorder_set(fs); be13::plugin::scanners_init(fs); fs.set_stop_list(&stop_list); fs.set_alert_list(&alert_list); /* Look for commands that impact per-recorders */ for(scanner_info::config_t::const_iterator it=s_config.namevals.begin();it!=s_config.namevals.end();it++){ /* see if there is a <recorder>: */ std::vector<std::string> params = split(it->first,':'); if(params.size()>=3 && params.at(0)=="fr"){ feature_recorder *fr = fs.get_name(params.at(1)); const std::string &cmd = params.at(2); if(fr){ if(cmd=="window") fr->set_context_window(stoi64(it->second)); if(cmd=="window_before") fr->set_context_window_before(stoi64(it->second)); if(cmd=="window_after") fr->set_context_window_after(stoi64(it->second)); } } /* See if there is a scanner? */ } /* Store the configuration in the XML file */ dfxml_writer *xreport = new dfxml_writer(reportfilename,false); dfxml_create(*xreport,command_line,cfg); xreport->xmlout("provided_filename",image_fname); // save this information /* provide documentation to the user; the DFXML information comes from elsewhere */ if(!cfg.opt_quiet){ std::cout << "bulk_extractor version: " << PACKAGE_VERSION << "\n"; #ifdef HAVE_GETHOSTNAME char hostname[1024]; gethostname(hostname,sizeof(hostname)); std::cout << "Hostname: " << hostname << "\n"; #endif std::cout << "Input file: " << image_fname << "\n"; std::cout << "Output directory: " << opt_outdir << "\n"; std::cout << "Disk Size: " << p->image_size() << "\n"; std::cout << "Threads: " << cfg.num_threads << "\n"; } /**************************************************************** *** THIS IS IT! PHASE 1! ****************************************************************/ if ( fs.flag_set(feature_recorder_set::ENABLE_SQLITE3_RECORDERS )) { fs.db_transaction_begin(); } BulkExtractor_Phase1 phase1(*xreport,timer,cfg); if(cfg.debug & DEBUG_PRINT_STEPS) std::cerr << "DEBUG: STARTING PHASE 1\n"; if(opt_sampling_params.size()>0) BulkExtractor_Phase1::set_sampling_parameters(cfg,opt_sampling_params); xreport->add_timestamp("phase1 start"); phase1.run(*p,fs,seen_page_ids); if(cfg.debug & DEBUG_PRINT_STEPS) std::cerr << "DEBUG: WAITING FOR WORKERS\n"; std::string md5_string; phase1.wait_for_workers(*p,&md5_string); delete p; // not strictly needed, but why not? p = 0; if ( fs.flag_set(feature_recorder_set::ENABLE_SQLITE3_RECORDERS )) { fs.db_transaction_commit(); } xreport->add_timestamp("phase1 end"); if(md5_string.size()>0){ std::cout << "MD5 of Disk Image: " << md5_string << "\n"; } /*** PHASE 2 --- Shutdown ***/ if(cfg.opt_quiet==0) std::cout << "Phase 2. Shutting down scanners\n"; xreport->add_timestamp("phase2 start"); be13::plugin::phase_shutdown(fs); xreport->add_timestamp("phase2 end"); /*** PHASE 3 --- Create Histograms ***/ if(cfg.opt_quiet==0) std::cout << "Phase 3. Creating Histograms\n"; xreport->add_timestamp("phase3 start"); if(opt_enable_histograms) fs.dump_histograms(0,histogram_dump_callback,0); // TK - add an xml error notifier! xreport->add_timestamp("phase3 end"); /*** PHASE 4 --- report and then print final usage information ***/ xreport->push("report"); xreport->xmlout("total_bytes",phase1.total_bytes); xreport->xmlout("elapsed_seconds",timer.elapsed_seconds()); xreport->xmlout("max_depth_seen",be13::plugin::get_max_depth_seen()); xreport->xmlout("dup_data_encountered",be13::plugin::dup_data_encountered); xreport->pop(); // report xreport->flush(); xreport->push("scanner_times"); fs.get_stats(xreport,stat_callback); xreport->pop(); xreport->add_rusage(); xreport->pop(); // bulk_extractor xreport->close(); if(cfg.opt_quiet==0){ float mb_per_sec = (phase1.total_bytes / 1000000.0) / timer.elapsed_seconds(); std::cout.precision(4); printf("Elapsed time: %g sec.\n",timer.elapsed_seconds()); printf("Total MB processed: %d\n",int(phase1.total_bytes / 100000)); printf("Overall performance: %g MBytes/sec (%g MBytes/sec/thread)\n", mb_per_sec,mb_per_sec/cfg.num_threads); if (fs.has_name("email")) { feature_recorder *fr = fs.get_name("email"); if(fr){ std::cout << "Total " << fr->name << " features found: " << fr->count() << "\n"; } } } } #ifdef HAVE_MCHECK muntrace(); #endif exit(0); }