Ejemplo n.º 1
0
PRIVATE void do_flags(char *flags)
{
     char *s;

     for (s = flags+1; *s != '\0'; s++)
          switch (*s) {
          case 'a': aflag++; break;
	  case 'd': dflag++; break;
	  case 'l': lflag++; break;
          case 'q': qflag++; break;
          case 's': sflag++; break;
          case 't': tflag++; break;
          case 'v': vflag++; break;
          case 'i': iflag++; break;

          case 'p':
               if (--argc == 0) usage();
               pflag++;
               prelude = *++argv;
               break;

          case 'D':
               for (s++; *s != '\0'; s++)
                    if (islower(*s))
                         debflag(*s)++;
                    else
                         debug_help();
               s--;
               break;

          default:  usage();
          }
}
Ejemplo n.º 2
0
/* sec 0093 */
void succumb (void)
{
  if (interaction == error_stop_mode)
    interaction = scroll_mode;

  if (log_opened)
    error();

#ifdef DEBUG
  if (interaction > 0)
    debug_help();
#endif

  history = error_stop_mode;
  jump_out();
}
Ejemplo n.º 3
0
void debug_do_command(char * command)
{

	int i;
	int mode = 0;

	if ( strlen(command) < 1 ) {
		return;
	}

	Dc_debug_on = 0;
	Dc_command_line = command;
	scanner_start_command(command);

	if (setjmp(dc_bad_arg) ) {
		return;
	}

	dc_get_arg( ARG_ANY );

	if ( !strcmp( Dc_arg, "debug" ) ) {
		Dc_debug_on = 1;
		dc_printf( "Command line: '%s'\n", Dc_command_line );
		dc_get_arg( ARG_ANY );
	}

	if ( !stricmp( Dc_arg, "xyzzy" ) ) {
		dc_printf("Nothing happens.\n");
		return;
	}

	if ( !strcmp( Dc_arg, "?" ) ) {
		mode = 1;
		dc_get_arg( ARG_ANY );

		if ( Dc_arg_type&ARG_NONE ) {
			debug_help();
			return;
		}
	}

	if ( !strcmp( Dc_arg, "help" ) || !strcmp( Dc_arg, "man" ) ) {
		mode = 2;
		dc_get_arg( ARG_ANY );
		if ( Dc_arg_type&ARG_NONE ) {
			debug_help();
			return;
		}
	}

	if ( strstr( Dc_command_line, "?" ) ) {
		mode = 2;
	}

	if ( !(Dc_arg_type&ARG_STRING) ) {
		dc_printf( "Invalid keyword '%s'\n", Dc_arg );
		return;
	}


	if (Dc_debug_on) {
		dc_printf( "Searching for command '%s'\n", Dc_arg );
	}

	for (i=0; i<Num_debug_commands; i++ ) {
		if ( !stricmp( Debug_command[i]->name, Dc_arg ) ) {
		
			if (mode==0) {
				if (Dc_debug_on) {
					dc_printf( "Calling function '%s'\n", Dc_arg );
				}
				Dc_command = 1;
				Dc_help = 0;
				Dc_status = 1;
			} else if (mode==1) {
				if (Dc_debug_on) {
					dc_printf( "Checking status for '%s'\n", Dc_arg );
				}
				Dc_command = 0;
				Dc_help = 0;
				Dc_status = 1;
			} else {
				if (Dc_debug_on) {
					dc_printf( "Doing help for '%s'\n", Dc_arg );
				}
				Dc_command = 0;
				Dc_help = 1;
				Dc_status = 0;
			}

			(*Debug_command[i]->func)();

			if (mode==0) {
				dc_get_arg(ARG_ANY);
				if (!(Dc_arg_type&ARG_NONE)) {
					dc_printf( "Ignoring the unused command line tail '%s %s'\n", Dc_arg_org, Dc_command_line );
				}
			}

			return;
		}
	}

	dc_printf( "Unknown command '%s'\n", Dc_arg );
}
Ejemplo n.º 4
0
int main(int argc,char **argv)
{
#ifdef HAVE_MCHECK
    mtrace();
#endif
    /* setup */
    feature_recorder::set_main_threadid();
    const char *progname = argv[0];

    word_and_context_list alert_list;		/* shold be flagged */
    word_and_context_list stop_list;		/* should be ignored */

    scanner_info::scanner_config   s_config;    // the bulk extractor phase 1 config created from the command line
    BulkExtractor_Phase1::Config   cfg;
    cfg.num_threads = threadpool::numCPU();

    /* Options */
    const char *opt_path = 0;
    int         opt_recurse = 0;
    int         opt_zap = 0;
    int         opt_h = 0;
    int         opt_H = 0;
    std::string opt_sampling_params;
    std::string opt_outdir;
    bool        opt_write_feature_files = true;
    bool        opt_write_sqlite3     = false;
    bool        opt_enable_histograms=true;

    /* Startup */
    setvbuf(stdout,0,_IONBF,0);		// don't buffer stdout
    std::string command_line = dfxml_writer::make_command_line(argc,argv);
    std::vector<std::string> scanner_dirs; // where to look for scanners

    /* Add the default plugin_path */
    add_if_present(scanner_dirs,"/usr/local/lib/bulk_extractor");
    add_if_present(scanner_dirs,"/usr/lib/bulk_extractor");
    add_if_present(scanner_dirs,".");

    if (getenv("BE_PATH")) {
        std::vector<std::string> dirs = split(getenv("BE_PATH"),':');
        for(std::vector<std::string>::const_iterator it = dirs.begin(); it!=dirs.end(); it++){
            add_if_present(scanner_dirs,*it);
        }
    }

#ifdef WIN32
    setmode(1,O_BINARY);		// make stdout binary
    threadpool::win32_init();
#endif
    /* look for usage first */
    if(argc==1) opt_h=1;

    /* Process options */
    int ch;
    while ((ch = getopt(argc, argv, "A:B:b:C:d:E:e:F:f:G:g:Hhij:M:m:o:P:p:q:Rr:S:s:VW:w:x:Y:z:Z")) != -1) {
	switch (ch) {
	case 'A': feature_recorder::offset_add  = stoi64(optarg);break;
	case 'b': feature_recorder::banner_file = optarg; break;
	case 'C': feature_recorder::context_window_default = atoi(optarg);break;
	case 'd':
	{
            if(strcmp(optarg,"h")==0) debug_help();
	    int d = atoi(optarg);
	    switch(d){
	    case DEBUG_ALLOCATE_512MiB: 
		if(calloc(1024*1024*512,1)){
                    std::cerr << "-d1002 -- Allocating 512MB of RAM; may be repeated\n";
		} else {
                    std::cerr << "-d1002 -- CANNOT ALLOCATE MORE RAM\n";
		}
		break;
	    default:
		cfg.debug  = d;
		break;
	    }
            be13::plugin::set_scanner_debug(cfg.debug);
	}
	break;
	case 'E':
            be13::plugin::scanners_disable_all();
	    be13::plugin::scanners_enable(optarg);
	    break;
	case 'e':
	    be13::plugin::scanners_enable(optarg);
	    break;
	case 'F': FindOpts::get().Files.push_back(optarg); break;
	case 'f': FindOpts::get().Patterns.push_back(optarg); break;
	case 'G': cfg.opt_pagesize = scaled_stoi64(optarg); break;
	case 'g': cfg.opt_marginsize = scaled_stoi64(optarg); break;
        case 'i':
            std::cout << "info mode:\n";
            cfg.opt_info = true;
            break;
	case 'j': cfg.num_threads = atoi(optarg); break;
	case 'M': scanner_def::max_depth = atoi(optarg); break;
	case 'm': cfg.max_bad_alloc_errors = atoi(optarg); break;
	case 'o': opt_outdir = optarg;break;
	case 'P': scanner_dirs.push_back(optarg);break;
	case 'p': opt_path = optarg; break;
        case 'q':
	    if(atoi(optarg)==-1) cfg.opt_quiet = 1;// -q -1 turns off notifications
	    else cfg.opt_notify_rate = atoi(optarg);
	    break;
	case 'r':
	    if(alert_list.readfile(optarg)){
		err(1,"Cannot read alert list %s",optarg);
	    }
	    break;
	case 'R': opt_recurse = 1; break;
	case 'S':
	{
	    std::vector<std::string> params = split(optarg,'=');
	    if(params.size()!=2){
		std::cerr << "Invalid paramter: " << optarg << "\n";
		exit(1);
	    }
	    s_config.namevals[params[0]] = params[1];
	    continue;
	}
	case 's':
#if defined(HAVE_SRANDOM) && !defined(HAVE_SRANDOMDEV)
            srandom(time(0));
#endif
#if defined(HAVE_SRANDOMDEV)
            srandomdev();               // if we are sampling initialize
#endif
            opt_sampling_params = optarg;
            break;
	case 'V': std::cout << "bulk_extractor " << PACKAGE_VERSION << "\n"; exit (1);
	case 'W':
            fprintf(stderr,"-W has been deprecated. Specify with -S word_min=NN and -S word_max=NN\n");
            exit(1);
	    break;
	case 'w': if(stop_list.readfile(optarg)){
		err(1,"Cannot read stop list %s",optarg);
	    }
	    break;
	case 'x':
	    be13::plugin::scanners_disable(optarg);
	    break;
	case 'Y': {
	    std::string optargs = optarg;
	    size_t dash = optargs.find('-');
	    if(dash==std::string::npos){
		cfg.opt_offset_start = stoi64(optargs);
	    } else {
		cfg.opt_offset_start = scaled_stoi64(optargs.substr(0,dash));
		cfg.opt_offset_end   = scaled_stoi64(optargs.substr(dash+1));
	    }
	    break;
	}
	case 'z': cfg.opt_page_start = stoi64(optarg);break;
	case 'Z': opt_zap=true;break;
	case 'H': opt_H++;continue;
	case 'h': opt_h++;continue;
	}
    }

    cfg.validate();
    argc -= optind;
    argv += optind;

    if(cfg.debug & DEBUG_PRINT_STEPS) std::cerr << "DEBUG: DEBUG_PRINT_STEPS\n";
    if(cfg.debug & DEBUG_PEDANTIC) validateOrEscapeUTF8_validate = true;

    /* Create a configuration that will be used to initialize the scanners */
    scanner_info si;

    s_config.debug       = cfg.debug;
    si.config = &s_config;

    /* Make individual configuration options appear on the command line interface. */
    si.get_config("work_start_work_end",&worker::opt_work_start_work_end,
                  "Record work start and end of each scanner in report.xml file");
    si.get_config("enable_histograms",&opt_enable_histograms,
                  "Disable generation of histograms");
    si.get_config("debug_histogram_malloc_fail_frequency",&HistogramMaker::debug_histogram_malloc_fail_frequency,
                  "Set >0 to make histogram maker fail with memory allocations");
    si.get_config("hash_alg",&be_hash_name,"Specifies hash algorithm to be used for all hash calculations");
    si.get_config("dup_data_alerts",&be13::plugin::dup_data_alerts,"Notify when duplicate data is not processed");
    si.get_config("write_feature_files",&opt_write_feature_files,"Write features to flat files");
    si.get_config("write_feature_sqlite3",&opt_write_sqlite3,"Write feature files to report.sqlite3");

    /* Make sure that the user selected a valid hash */
    {
        uint8_t buf[1];
        be_hash_func(buf,0);
    }

    /* Load all the scanners and enable the ones we care about */

    be13::plugin::load_scanner_directories(scanner_dirs,s_config);
    be13::plugin::load_scanners(scanners_builtin,s_config); 
    be13::plugin::scanners_process_enable_disable_commands();

    /* Print usage if necessary */
    if(opt_H){ be13::plugin::info_scanners(true,true,scanners_builtin,'e','x'); exit(0);}
    if(opt_h){ usage(progname);be13::plugin::info_scanners(false,true,scanners_builtin,'e','x'); exit(0);}

    /* Give an error if a find list was specified
     * but no scanner that uses the find list is enabled.
     */

    if(!FindOpts::get().empty()) {
        /* Look through the enabled scanners and make sure that
	 * at least one of them is a FIND scanner
	 */
        if(!be13::plugin::find_scanner_enabled()){
            errx(1,"find words are specified with -F but no find scanner is enabled.\n");
        }
    }

    if(opt_path){
	if(argc!=1) errx(1,"-p requires a single argument.");
	process_path(argv[0],opt_path,cfg.opt_pagesize,cfg.opt_marginsize);
	exit(0);
    }
    if(opt_outdir.size()==0) errx(1,"error: -o outdir must be specified");

    /* The zap option wipes the contents of a directory, useful for debugging */
    if(opt_zap){
	DIR *dirp = opendir(opt_outdir.c_str());
	if(dirp){
	    struct dirent *dp;
	    while ((dp = readdir(dirp)) != NULL){
                std::string name = dp->d_name;
		if(name=="." || name=="..") continue;
                std::string fname = opt_outdir + std::string("/") + name;
		unlink(fname.c_str());
		std::cout << "erasing " << fname << "\n";
	    }
	}
	if(rmdir(opt_outdir.c_str())){
            std::cout << "rmdir " << opt_outdir << "\n";
        }
    }

    /* Start the clock */
    aftimer timer;
    timer.start();

    /* If output directory does not exist, we are not restarting! */
    std::string reportfilename = opt_outdir + "/report.xml";

    BulkExtractor_Phase1::seen_page_ids_t seen_page_ids; // pages that do not need re-processing
    image_process *p = 0;                                // the image process iterator

    /* Get image or directory */
    if (*argv == NULL) {
        if (opt_recurse) {
            fprintf(stderr,"filedir not provided\n");
        } else {
            fprintf(stderr,"imagefile not provided\n");
        }
        exit(1);
    }
    std::string image_fname = *argv;

    if(opt_outdir.size()==0){
        fprintf(stderr,"output directory not provided\n");
        exit(1);
    }

    if(directory_missing(opt_outdir) || directory_empty(opt_outdir)){
        /* First time running */
	/* Validate the args */
	if ( argc !=1 ) errx(1,"Disk image option not provided. Run with -h for help.");
	validate_fn(image_fname);
	if (directory_missing(opt_outdir)) be_mkdir(opt_outdir);
    } else {
	/* Restarting */
	std::cout << "Restarting from " << opt_outdir << "\n";
        bulk_extractor_restarter r(opt_outdir,reportfilename,image_fname,seen_page_ids);

        /* Rename the old report and create a new one */
        std::string old_reportfilename = reportfilename + "." + itos(time(0));
        if(rename(reportfilename.c_str(),old_reportfilename.c_str())){
            std::cerr << "Could not rename " << reportfilename << " to " << old_reportfilename << ": " << strerror(errno) << "\n";
            exit(1);
        }
    }

    /* Open the image file (or the device) now */
    p = image_process::open(image_fname,opt_recurse,cfg.opt_pagesize,cfg.opt_marginsize);
    if(!p) err(1,"Cannot open %s: ",image_fname.c_str());
    
    /***
     *** Create the feature recording set.
     *** Initialize the scanners.
     ****/

    /* Determine the feature files that will be used */
    feature_file_names_t feature_file_names;
    be13::plugin::get_scanner_feature_file_names(feature_file_names);
    uint32_t flags = 0;
    if (stop_list.size()>0)        flags |= feature_recorder_set::CREATE_STOP_LIST_RECORDERS;
    if (opt_write_sqlite3)         flags |= feature_recorder_set::ENABLE_SQLITE3_RECORDERS;
    if (!opt_write_feature_files)  flags |= feature_recorder_set::DISABLE_FILE_RECORDERS;

    {
        feature_recorder_set fs(flags,be_hash,image_fname,opt_outdir);
        fs.init(feature_file_names);
        if(opt_enable_histograms) be13::plugin::add_enabled_scanner_histograms_to_feature_recorder_set(fs);
        be13::plugin::scanners_init(fs);

        fs.set_stop_list(&stop_list);
        fs.set_alert_list(&alert_list);

        /* Look for commands that impact per-recorders */
        for(scanner_info::config_t::const_iterator it=s_config.namevals.begin();it!=s_config.namevals.end();it++){
            /* see if there is a <recorder>: */
            std::vector<std::string> params = split(it->first,':');
            if(params.size()>=3 && params.at(0)=="fr"){
                feature_recorder *fr = fs.get_name(params.at(1));
                const std::string &cmd = params.at(2);
                if(fr){
                    if(cmd=="window")        fr->set_context_window(stoi64(it->second));
                    if(cmd=="window_before") fr->set_context_window_before(stoi64(it->second));
                    if(cmd=="window_after")  fr->set_context_window_after(stoi64(it->second));
                }
            }
            /* See if there is a scanner? */
        }

        /* Store the configuration in the XML file */
        dfxml_writer  *xreport = new dfxml_writer(reportfilename,false);
        dfxml_create(*xreport,command_line,cfg);
        xreport->xmlout("provided_filename",image_fname); // save this information

        /* provide documentation to the user; the DFXML information comes from elsewhere */
        if(!cfg.opt_quiet){
            std::cout << "bulk_extractor version: " << PACKAGE_VERSION << "\n";
#ifdef HAVE_GETHOSTNAME
            char hostname[1024];
            gethostname(hostname,sizeof(hostname));
            std::cout << "Hostname: " << hostname << "\n";
#endif
            std::cout << "Input file: " << image_fname << "\n";
            std::cout << "Output directory: " << opt_outdir << "\n";
            std::cout << "Disk Size: " << p->image_size() << "\n";
            std::cout << "Threads: " << cfg.num_threads << "\n";
        }

        /****************************************************************
         *** THIS IS IT! PHASE 1!
         ****************************************************************/

        if ( fs.flag_set(feature_recorder_set::ENABLE_SQLITE3_RECORDERS )) {
            fs.db_transaction_begin();
        }
        BulkExtractor_Phase1 phase1(*xreport,timer,cfg);
        if(cfg.debug & DEBUG_PRINT_STEPS) std::cerr << "DEBUG: STARTING PHASE 1\n";

        if(opt_sampling_params.size()>0) BulkExtractor_Phase1::set_sampling_parameters(cfg,opt_sampling_params);
        xreport->add_timestamp("phase1 start");
        phase1.run(*p,fs,seen_page_ids);

        if(cfg.debug & DEBUG_PRINT_STEPS) std::cerr << "DEBUG: WAITING FOR WORKERS\n";
        std::string md5_string;
        phase1.wait_for_workers(*p,&md5_string);
        delete p;				// not strictly needed, but why not?
        p = 0;

        if ( fs.flag_set(feature_recorder_set::ENABLE_SQLITE3_RECORDERS )) {
            fs.db_transaction_commit();
        }
        xreport->add_timestamp("phase1 end");
        if(md5_string.size()>0){
            std::cout << "MD5 of Disk Image: " << md5_string << "\n";
        }

        /*** PHASE 2 --- Shutdown ***/
        if(cfg.opt_quiet==0) std::cout << "Phase 2. Shutting down scanners\n";
        xreport->add_timestamp("phase2 start");
        be13::plugin::phase_shutdown(fs);
        xreport->add_timestamp("phase2 end");

        /*** PHASE 3 --- Create Histograms ***/
        if(cfg.opt_quiet==0) std::cout << "Phase 3. Creating Histograms\n";
        xreport->add_timestamp("phase3 start");
        if(opt_enable_histograms) fs.dump_histograms(0,histogram_dump_callback,0);        // TK - add an xml error notifier!
        xreport->add_timestamp("phase3 end");

        /*** PHASE 4 ---  report and then print final usage information ***/
        xreport->push("report");
        xreport->xmlout("total_bytes",phase1.total_bytes);
        xreport->xmlout("elapsed_seconds",timer.elapsed_seconds());
        xreport->xmlout("max_depth_seen",be13::plugin::get_max_depth_seen());
        xreport->xmlout("dup_data_encountered",be13::plugin::dup_data_encountered);
        xreport->pop();			// report
        xreport->flush();

        xreport->push("scanner_times");
        fs.get_stats(xreport,stat_callback);
        xreport->pop();
        xreport->add_rusage();
        xreport->pop();			// bulk_extractor
        xreport->close();
        if(cfg.opt_quiet==0){
            float mb_per_sec = (phase1.total_bytes / 1000000.0) / timer.elapsed_seconds();

            std::cout.precision(4);
            printf("Elapsed time: %g sec.\n",timer.elapsed_seconds());
            printf("Total MB processed: %d\n",int(phase1.total_bytes / 100000));
        
            printf("Overall performance: %g MBytes/sec (%g MBytes/sec/thread)\n",
                   mb_per_sec,mb_per_sec/cfg.num_threads);
            if (fs.has_name("email")) {
                feature_recorder *fr = fs.get_name("email");
                if(fr){
                    std::cout << "Total " << fr->name << " features found: " << fr->count() << "\n";
                }
            }
        }
    }
#ifdef HAVE_MCHECK
    muntrace();
#endif
    exit(0);
}