PRIVATE void do_flags(char *flags)
{
char *s;
for (s = flags+1; *s != '\0'; s++)
switch (*s) {
case 'a': aflag++; break;
case 'd': dflag++; break;
case 'l': lflag++; break;
case 'q': qflag++; break;
case 's': sflag++; break;
case 't': tflag++; break;
case 'v': vflag++; break;
case 'i': iflag++; break;
case 'p':
if (--argc == 0) usage();
pflag++;
prelude = *++argv;
break;
case 'D':
for (s++; *s != '\0'; s++)
if (islower(*s))
debflag(*s)++;
else
debug_help();
s--;
break;
default: usage();
}
}
/* sec 0093 */
void succumb (void)
{
if (interaction == error_stop_mode)
interaction = scroll_mode;
if (log_opened)
error();
#ifdef DEBUG
if (interaction > 0)
debug_help();
#endif
history = error_stop_mode;
jump_out();
}
void debug_do_command(char * command)
{
int i;
int mode = 0;
if ( strlen(command) < 1 ) {
return;
}
Dc_debug_on = 0;
Dc_command_line = command;
scanner_start_command(command);
if (setjmp(dc_bad_arg) ) {
return;
}
dc_get_arg( ARG_ANY );
if ( !strcmp( Dc_arg, "debug" ) ) {
Dc_debug_on = 1;
dc_printf( "Command line: '%s'\n", Dc_command_line );
dc_get_arg( ARG_ANY );
}
if ( !stricmp( Dc_arg, "xyzzy" ) ) {
dc_printf("Nothing happens.\n");
return;
}
if ( !strcmp( Dc_arg, "?" ) ) {
mode = 1;
dc_get_arg( ARG_ANY );
if ( Dc_arg_type&ARG_NONE ) {
debug_help();
return;
}
}
if ( !strcmp( Dc_arg, "help" ) || !strcmp( Dc_arg, "man" ) ) {
mode = 2;
dc_get_arg( ARG_ANY );
if ( Dc_arg_type&ARG_NONE ) {
debug_help();
return;
}
}
if ( strstr( Dc_command_line, "?" ) ) {
mode = 2;
}
if ( !(Dc_arg_type&ARG_STRING) ) {
dc_printf( "Invalid keyword '%s'\n", Dc_arg );
return;
}
if (Dc_debug_on) {
dc_printf( "Searching for command '%s'\n", Dc_arg );
}
for (i=0; i<Num_debug_commands; i++ ) {
if ( !stricmp( Debug_command[i]->name, Dc_arg ) ) {
if (mode==0) {
if (Dc_debug_on) {
dc_printf( "Calling function '%s'\n", Dc_arg );
}
Dc_command = 1;
Dc_help = 0;
Dc_status = 1;
} else if (mode==1) {
if (Dc_debug_on) {
dc_printf( "Checking status for '%s'\n", Dc_arg );
}
Dc_command = 0;
Dc_help = 0;
Dc_status = 1;
} else {
if (Dc_debug_on) {
dc_printf( "Doing help for '%s'\n", Dc_arg );
}
Dc_command = 0;
Dc_help = 1;
Dc_status = 0;
}
(*Debug_command[i]->func)();
if (mode==0) {
dc_get_arg(ARG_ANY);
if (!(Dc_arg_type&ARG_NONE)) {
dc_printf( "Ignoring the unused command line tail '%s %s'\n", Dc_arg_org, Dc_command_line );
}
}
return;
}
}
dc_printf( "Unknown command '%s'\n", Dc_arg );
}
int main(int argc,char **argv)
{
#ifdef HAVE_MCHECK
mtrace();
#endif
/* setup */
feature_recorder::set_main_threadid();
const char *progname = argv[0];
word_and_context_list alert_list; /* shold be flagged */
word_and_context_list stop_list; /* should be ignored */
scanner_info::scanner_config s_config; // the bulk extractor phase 1 config created from the command line
BulkExtractor_Phase1::Config cfg;
cfg.num_threads = threadpool::numCPU();
/* Options */
const char *opt_path = 0;
int opt_recurse = 0;
int opt_zap = 0;
int opt_h = 0;
int opt_H = 0;
std::string opt_sampling_params;
std::string opt_outdir;
bool opt_write_feature_files = true;
bool opt_write_sqlite3 = false;
bool opt_enable_histograms=true;
/* Startup */
setvbuf(stdout,0,_IONBF,0); // don't buffer stdout
std::string command_line = dfxml_writer::make_command_line(argc,argv);
std::vector<std::string> scanner_dirs; // where to look for scanners
/* Add the default plugin_path */
add_if_present(scanner_dirs,"/usr/local/lib/bulk_extractor");
add_if_present(scanner_dirs,"/usr/lib/bulk_extractor");
add_if_present(scanner_dirs,".");
if (getenv("BE_PATH")) {
std::vector<std::string> dirs = split(getenv("BE_PATH"),':');
for(std::vector<std::string>::const_iterator it = dirs.begin(); it!=dirs.end(); it++){
add_if_present(scanner_dirs,*it);
}
}
#ifdef WIN32
setmode(1,O_BINARY); // make stdout binary
threadpool::win32_init();
#endif
/* look for usage first */
if(argc==1) opt_h=1;
/* Process options */
int ch;
while ((ch = getopt(argc, argv, "A:B:b:C:d:E:e:F:f:G:g:Hhij:M:m:o:P:p:q:Rr:S:s:VW:w:x:Y:z:Z")) != -1) {
switch (ch) {
case 'A': feature_recorder::offset_add = stoi64(optarg);break;
case 'b': feature_recorder::banner_file = optarg; break;
case 'C': feature_recorder::context_window_default = atoi(optarg);break;
case 'd':
{
if(strcmp(optarg,"h")==0) debug_help();
int d = atoi(optarg);
switch(d){
case DEBUG_ALLOCATE_512MiB:
if(calloc(1024*1024*512,1)){
std::cerr << "-d1002 -- Allocating 512MB of RAM; may be repeated\n";
} else {
std::cerr << "-d1002 -- CANNOT ALLOCATE MORE RAM\n";
}
break;
default:
cfg.debug = d;
break;
}
be13::plugin::set_scanner_debug(cfg.debug);
}
break;
case 'E':
be13::plugin::scanners_disable_all();
be13::plugin::scanners_enable(optarg);
break;
case 'e':
be13::plugin::scanners_enable(optarg);
break;
case 'F': FindOpts::get().Files.push_back(optarg); break;
case 'f': FindOpts::get().Patterns.push_back(optarg); break;
case 'G': cfg.opt_pagesize = scaled_stoi64(optarg); break;
case 'g': cfg.opt_marginsize = scaled_stoi64(optarg); break;
case 'i':
std::cout << "info mode:\n";
cfg.opt_info = true;
break;
case 'j': cfg.num_threads = atoi(optarg); break;
case 'M': scanner_def::max_depth = atoi(optarg); break;
case 'm': cfg.max_bad_alloc_errors = atoi(optarg); break;
case 'o': opt_outdir = optarg;break;
case 'P': scanner_dirs.push_back(optarg);break;
case 'p': opt_path = optarg; break;
case 'q':
if(atoi(optarg)==-1) cfg.opt_quiet = 1;// -q -1 turns off notifications
else cfg.opt_notify_rate = atoi(optarg);
break;
case 'r':
if(alert_list.readfile(optarg)){
err(1,"Cannot read alert list %s",optarg);
}
break;
case 'R': opt_recurse = 1; break;
case 'S':
{
std::vector<std::string> params = split(optarg,'=');
if(params.size()!=2){
std::cerr << "Invalid paramter: " << optarg << "\n";
exit(1);
}
s_config.namevals[params[0]] = params[1];
continue;
}
case 's':
#if defined(HAVE_SRANDOM) && !defined(HAVE_SRANDOMDEV)
srandom(time(0));
#endif
#if defined(HAVE_SRANDOMDEV)
srandomdev(); // if we are sampling initialize
#endif
opt_sampling_params = optarg;
break;
case 'V': std::cout << "bulk_extractor " << PACKAGE_VERSION << "\n"; exit (1);
case 'W':
fprintf(stderr,"-W has been deprecated. Specify with -S word_min=NN and -S word_max=NN\n");
exit(1);
break;
case 'w': if(stop_list.readfile(optarg)){
err(1,"Cannot read stop list %s",optarg);
}
break;
case 'x':
be13::plugin::scanners_disable(optarg);
break;
case 'Y': {
std::string optargs = optarg;
size_t dash = optargs.find('-');
if(dash==std::string::npos){
cfg.opt_offset_start = stoi64(optargs);
} else {
cfg.opt_offset_start = scaled_stoi64(optargs.substr(0,dash));
cfg.opt_offset_end = scaled_stoi64(optargs.substr(dash+1));
}
break;
}
case 'z': cfg.opt_page_start = stoi64(optarg);break;
case 'Z': opt_zap=true;break;
case 'H': opt_H++;continue;
case 'h': opt_h++;continue;
}
}
cfg.validate();
argc -= optind;
argv += optind;
if(cfg.debug & DEBUG_PRINT_STEPS) std::cerr << "DEBUG: DEBUG_PRINT_STEPS\n";
if(cfg.debug & DEBUG_PEDANTIC) validateOrEscapeUTF8_validate = true;
/* Create a configuration that will be used to initialize the scanners */
scanner_info si;
s_config.debug = cfg.debug;
si.config = &s_config;
/* Make individual configuration options appear on the command line interface. */
si.get_config("work_start_work_end",&worker::opt_work_start_work_end,
"Record work start and end of each scanner in report.xml file");
si.get_config("enable_histograms",&opt_enable_histograms,
"Disable generation of histograms");
si.get_config("debug_histogram_malloc_fail_frequency",&HistogramMaker::debug_histogram_malloc_fail_frequency,
"Set >0 to make histogram maker fail with memory allocations");
si.get_config("hash_alg",&be_hash_name,"Specifies hash algorithm to be used for all hash calculations");
si.get_config("dup_data_alerts",&be13::plugin::dup_data_alerts,"Notify when duplicate data is not processed");
si.get_config("write_feature_files",&opt_write_feature_files,"Write features to flat files");
si.get_config("write_feature_sqlite3",&opt_write_sqlite3,"Write feature files to report.sqlite3");
/* Make sure that the user selected a valid hash */
{
uint8_t buf[1];
be_hash_func(buf,0);
}
/* Load all the scanners and enable the ones we care about */
be13::plugin::load_scanner_directories(scanner_dirs,s_config);
be13::plugin::load_scanners(scanners_builtin,s_config);
be13::plugin::scanners_process_enable_disable_commands();
/* Print usage if necessary */
if(opt_H){ be13::plugin::info_scanners(true,true,scanners_builtin,'e','x'); exit(0);}
if(opt_h){ usage(progname);be13::plugin::info_scanners(false,true,scanners_builtin,'e','x'); exit(0);}
/* Give an error if a find list was specified
* but no scanner that uses the find list is enabled.
*/
if(!FindOpts::get().empty()) {
/* Look through the enabled scanners and make sure that
* at least one of them is a FIND scanner
*/
if(!be13::plugin::find_scanner_enabled()){
errx(1,"find words are specified with -F but no find scanner is enabled.\n");
}
}
if(opt_path){
if(argc!=1) errx(1,"-p requires a single argument.");
process_path(argv[0],opt_path,cfg.opt_pagesize,cfg.opt_marginsize);
exit(0);
}
if(opt_outdir.size()==0) errx(1,"error: -o outdir must be specified");
/* The zap option wipes the contents of a directory, useful for debugging */
if(opt_zap){
DIR *dirp = opendir(opt_outdir.c_str());
if(dirp){
struct dirent *dp;
while ((dp = readdir(dirp)) != NULL){
std::string name = dp->d_name;
if(name=="." || name=="..") continue;
std::string fname = opt_outdir + std::string("/") + name;
unlink(fname.c_str());
std::cout << "erasing " << fname << "\n";
}
}
if(rmdir(opt_outdir.c_str())){
std::cout << "rmdir " << opt_outdir << "\n";
}
}
/* Start the clock */
aftimer timer;
timer.start();
/* If output directory does not exist, we are not restarting! */
std::string reportfilename = opt_outdir + "/report.xml";
BulkExtractor_Phase1::seen_page_ids_t seen_page_ids; // pages that do not need re-processing
image_process *p = 0; // the image process iterator
/* Get image or directory */
if (*argv == NULL) {
if (opt_recurse) {
fprintf(stderr,"filedir not provided\n");
} else {
fprintf(stderr,"imagefile not provided\n");
}
exit(1);
}
std::string image_fname = *argv;
if(opt_outdir.size()==0){
fprintf(stderr,"output directory not provided\n");
exit(1);
}
if(directory_missing(opt_outdir) || directory_empty(opt_outdir)){
/* First time running */
/* Validate the args */
if ( argc !=1 ) errx(1,"Disk image option not provided. Run with -h for help.");
validate_fn(image_fname);
if (directory_missing(opt_outdir)) be_mkdir(opt_outdir);
} else {
/* Restarting */
std::cout << "Restarting from " << opt_outdir << "\n";
bulk_extractor_restarter r(opt_outdir,reportfilename,image_fname,seen_page_ids);
/* Rename the old report and create a new one */
std::string old_reportfilename = reportfilename + "." + itos(time(0));
if(rename(reportfilename.c_str(),old_reportfilename.c_str())){
std::cerr << "Could not rename " << reportfilename << " to " << old_reportfilename << ": " << strerror(errno) << "\n";
exit(1);
}
}
/* Open the image file (or the device) now */
p = image_process::open(image_fname,opt_recurse,cfg.opt_pagesize,cfg.opt_marginsize);
if(!p) err(1,"Cannot open %s: ",image_fname.c_str());
/***
*** Create the feature recording set.
*** Initialize the scanners.
****/
/* Determine the feature files that will be used */
feature_file_names_t feature_file_names;
be13::plugin::get_scanner_feature_file_names(feature_file_names);
uint32_t flags = 0;
if (stop_list.size()>0) flags |= feature_recorder_set::CREATE_STOP_LIST_RECORDERS;
if (opt_write_sqlite3) flags |= feature_recorder_set::ENABLE_SQLITE3_RECORDERS;
if (!opt_write_feature_files) flags |= feature_recorder_set::DISABLE_FILE_RECORDERS;
{
feature_recorder_set fs(flags,be_hash,image_fname,opt_outdir);
fs.init(feature_file_names);
if(opt_enable_histograms) be13::plugin::add_enabled_scanner_histograms_to_feature_recorder_set(fs);
be13::plugin::scanners_init(fs);
fs.set_stop_list(&stop_list);
fs.set_alert_list(&alert_list);
/* Look for commands that impact per-recorders */
for(scanner_info::config_t::const_iterator it=s_config.namevals.begin();it!=s_config.namevals.end();it++){
/* see if there is a <recorder>: */
std::vector<std::string> params = split(it->first,':');
if(params.size()>=3 && params.at(0)=="fr"){
feature_recorder *fr = fs.get_name(params.at(1));
const std::string &cmd = params.at(2);
if(fr){
if(cmd=="window") fr->set_context_window(stoi64(it->second));
if(cmd=="window_before") fr->set_context_window_before(stoi64(it->second));
if(cmd=="window_after") fr->set_context_window_after(stoi64(it->second));
}
}
/* See if there is a scanner? */
}
/* Store the configuration in the XML file */
dfxml_writer *xreport = new dfxml_writer(reportfilename,false);
dfxml_create(*xreport,command_line,cfg);
xreport->xmlout("provided_filename",image_fname); // save this information
/* provide documentation to the user; the DFXML information comes from elsewhere */
if(!cfg.opt_quiet){
std::cout << "bulk_extractor version: " << PACKAGE_VERSION << "\n";
#ifdef HAVE_GETHOSTNAME
char hostname[1024];
gethostname(hostname,sizeof(hostname));
std::cout << "Hostname: " << hostname << "\n";
#endif
std::cout << "Input file: " << image_fname << "\n";
std::cout << "Output directory: " << opt_outdir << "\n";
std::cout << "Disk Size: " << p->image_size() << "\n";
std::cout << "Threads: " << cfg.num_threads << "\n";
}
/****************************************************************
*** THIS IS IT! PHASE 1!
****************************************************************/
if ( fs.flag_set(feature_recorder_set::ENABLE_SQLITE3_RECORDERS )) {
fs.db_transaction_begin();
}
BulkExtractor_Phase1 phase1(*xreport,timer,cfg);
if(cfg.debug & DEBUG_PRINT_STEPS) std::cerr << "DEBUG: STARTING PHASE 1\n";
if(opt_sampling_params.size()>0) BulkExtractor_Phase1::set_sampling_parameters(cfg,opt_sampling_params);
xreport->add_timestamp("phase1 start");
phase1.run(*p,fs,seen_page_ids);
if(cfg.debug & DEBUG_PRINT_STEPS) std::cerr << "DEBUG: WAITING FOR WORKERS\n";
std::string md5_string;
phase1.wait_for_workers(*p,&md5_string);
delete p; // not strictly needed, but why not?
p = 0;
if ( fs.flag_set(feature_recorder_set::ENABLE_SQLITE3_RECORDERS )) {
fs.db_transaction_commit();
}
xreport->add_timestamp("phase1 end");
if(md5_string.size()>0){
std::cout << "MD5 of Disk Image: " << md5_string << "\n";
}
/*** PHASE 2 --- Shutdown ***/
if(cfg.opt_quiet==0) std::cout << "Phase 2. Shutting down scanners\n";
xreport->add_timestamp("phase2 start");
be13::plugin::phase_shutdown(fs);
xreport->add_timestamp("phase2 end");
/*** PHASE 3 --- Create Histograms ***/
if(cfg.opt_quiet==0) std::cout << "Phase 3. Creating Histograms\n";
xreport->add_timestamp("phase3 start");
if(opt_enable_histograms) fs.dump_histograms(0,histogram_dump_callback,0); // TK - add an xml error notifier!
xreport->add_timestamp("phase3 end");
/*** PHASE 4 --- report and then print final usage information ***/
xreport->push("report");
xreport->xmlout("total_bytes",phase1.total_bytes);
xreport->xmlout("elapsed_seconds",timer.elapsed_seconds());
xreport->xmlout("max_depth_seen",be13::plugin::get_max_depth_seen());
xreport->xmlout("dup_data_encountered",be13::plugin::dup_data_encountered);
xreport->pop(); // report
xreport->flush();
xreport->push("scanner_times");
fs.get_stats(xreport,stat_callback);
xreport->pop();
xreport->add_rusage();
xreport->pop(); // bulk_extractor
xreport->close();
if(cfg.opt_quiet==0){
float mb_per_sec = (phase1.total_bytes / 1000000.0) / timer.elapsed_seconds();
std::cout.precision(4);
printf("Elapsed time: %g sec.\n",timer.elapsed_seconds());
printf("Total MB processed: %d\n",int(phase1.total_bytes / 100000));
printf("Overall performance: %g MBytes/sec (%g MBytes/sec/thread)\n",
mb_per_sec,mb_per_sec/cfg.num_threads);
if (fs.has_name("email")) {
feature_recorder *fr = fs.get_name("email");
if(fr){
std::cout << "Total " << fr->name << " features found: " << fr->count() << "\n";
}
}
}
}
#ifdef HAVE_MCHECK
muntrace();
#endif
exit(0);
}