void tcpdemux::post_process(tcpip *tcp) { std::stringstream xmladd; // for this <fileobject> if(opt.post_processing && tcp->file_created && tcp->last_byte>0){ /** * After the flow is finished, if more than a byte was * written, then put it in an SBUF and process it. if we are * doing post-processing. This is called from tcpip::~tcpip() * in tcpip.cpp. */ /* Open the fd if it is not already open */ tcp->open_file(); if(tcp->fd>=0){ sbuf_t *sbuf = sbuf_t::map_file(tcp->flow_pathname,tcp->fd); if(sbuf){ be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,*sbuf,*(fs),&xmladd)); delete sbuf; sbuf = 0; } } } tcp->close_file(); if(xreport) tcp->dump_xml(xreport,xmladd.str()); /** * Before we delete the tcp structure, save information about the saved flow */ save_flow(tcp); delete tcp; }
void worker::do_work(sbuf_t *sbuf) { /* If logging starting and ending, save the start */ if(opt_work_start_work_end){ std::stringstream ss; ss << "threadid='" << id << "'" << " pos0='" << dfxml_writer::xmlescape(sbuf->pos0.str()) << "'" << " pagesize='" << sbuf->pagesize << "'" << " bufsize='" << sbuf->bufsize << "'"; master.xreport.xmlout("debug:work_start","",ss.str(),true); } /** * HERE IT IS!!! * Construct a scanner_params() object from the sbuf that was pulled * off the work queue and call process_extract(). */ aftimer t; t.start(); be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,*sbuf,master.fs)); t.stop(); /* If we are logging starting and ending, save the end */ if(opt_work_start_work_end){ std::stringstream ss; ss << "threadid='" << id << "'" << " pos0='" << dfxml_writer::xmlescape(sbuf->pos0.str()) << "'" << " time='" << t.elapsed_seconds() << "'"; master.xreport.xmlout("debug:work_end","",ss.str(),true); } master.fs.flush_all(); }
int main(int argc,char **argv) { scanner_info::scanner_config s_config; be13::plugin::load_scanners(scanners_builtin,s_config); /* look for usage first */ if(argc==1 || (strcmp(argv[1],"-h")==0)){ usage(); return(1); } int ch; std::string opt_outdir; while ((ch = getopt(argc, argv, "e:o:s:x:h?")) != -1) { switch (ch) { case 'o': opt_outdir = optarg;break; case 'e': be13::plugin::scanners_enable(optarg);break; case 'x': be13::plugin::scanners_disable(optarg);break; case 's': { std::vector<std::string> params = split(optarg,'='); if(params.size()!=2){ std::cerr << "Invalid paramter: " << optarg << "\n"; exit(1); } //be_config[params[0]] = params[1]; continue; } case 'h': case '?':default: usage(); break; } } argc -= optind; argv += optind; if(argc!=1) usage(); //opt_scan_bulk_block_size = stoi64(be_config["bulk_block_size"]); be13::plugin::scanners_process_enable_disable_commands(); feature_file_names_t feature_file_names; be13::plugin::get_scanner_feature_file_names(feature_file_names); feature_recorder_set fs(0); // where the features will be put fs.init(feature_file_names,argv[0],opt_outdir); be13::plugin::scanners_init(fs); /* Make the sbuf */ sbuf_t *sbuf = sbuf_t::map_file(argv[0]); if(!sbuf){ err(1,"Cannot map file %s:%s\n",argv[0],strerror(errno)); } be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,*sbuf,fs)); be13::plugin::phase_shutdown(fs); fs.process_histograms(0); return(0); }
int bulk_extractor_analyze_buf(BEFILE *bef,uint8_t *buf,size_t buflen) { pos0_t pos0(""); const sbuf_t sbuf(pos0,buf,buflen,buflen,false); be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,sbuf,bef->cfs)); return 0; }
/** * After the flow is finished, put it in an SBUF and process it. * if we are doing post-processing. * This is called from tcpip::~tcpip() in tcpip.cpp. */ void tcpdemux::post_process_capture_flow(std::stringstream &xmladd, const std::string &flow_pathname) { int fd2 = retrying_open(flow_pathname,O_RDONLY|O_BINARY,0); if(fd2<0) { perror("open"); return; } sbuf_t *sbuf = sbuf_t::map_file(flow_pathname,pos0_t(flow_pathname),fd2); if(sbuf) { process_sbuf(scanner_params(scanner_params::scan,*sbuf,*fs,&xmladd)); } ::close(fd2); }
/* Found the end of the base64 string; process. */ inline void process(const class scanner_params &sp,const recursion_control_block &rcb,size_t start,size_t len) { //fprintf(stderr,"process start=%zd len=%zd\n",start,len); //fprintf(stderr,"To convert:\n"); const sbuf_t &sbuf = sp.sbuf; managed_malloc<unsigned char>base64_target(len); const char *src = (const char *)(sbuf.buf+start); if(len + start > sbuf.bufsize){ // make sure it doesn't go beyond buffer len = sbuf.bufsize-start; } //fwrite(src,len,1,stderr); //fprintf(stderr,"\n======\n"); int conv_len = b64_pton_forensic(src, len, // src,srclen base64_target.buf, len); // target, targetlen if(conv_len>0){ //fprintf(stderr,"conv_len=%zd\n",conv_len); //fwrite(base64_target.buf,1,conv_len,stderr); const pos0_t pos0_base64 = (sbuf.pos0 + start) + rcb.partName; const sbuf_t sbuf_base64(pos0_base64, base64_target.buf,conv_len,conv_len,false); // we will free (*rcb.callback)(scanner_params(sp,sbuf_base64)); } }
void scan_base64(const class scanner_params &sp,const recursion_control_block &rcb) { assert(sp.sp_version==scanner_params::CURRENT_SP_VERSION); if(sp.phase==scanner_params::PHASE_STARTUP){ assert(sp.info->si_version==scanner_info::CURRENT_SI_VERSION); sp.info->name = "base64"; sp.info->author = "Simson L. Garfinkel"; sp.info->description = "scans for Base64-encoded data"; sp.info->scanner_version= "1.0"; /* Create the base64 array */ memset(base64array,0,sizeof(base64array)); base64array[(int)'+'] = true; base64array[(int)'/'] = true; for(int ch='a';ch<='z';ch++){ base64array[ch] = true; } for(int ch='A';ch<='Z';ch++){ base64array[ch] = true; } for(int ch='0';ch<='9';ch++){ base64array[ch] = true; } return; /* No feature files created */ } if(sp.phase==scanner_params::PHASE_SHUTDOWN) return; if(sp.phase==scanner_params::PHASE_SCAN){ const sbuf_t &sbuf = sp.sbuf; /* base64 is a newline followed by at least two lines of constant length, * followed by an incomplete line ending with an equal sign. * Lines can be termianted by \n or \r\n. This code simply ignores \r, * which means that you can't have some lines terminated by \n and other * terminated by \n\r. * * Note that this doesn't scan base64-encoded blobs smaller than two lines. * Perhaps we should do that. */ for(size_t i=0;i<sbuf.pagesize;i++){ if(i==0 || sbuf[i]=='\n' ){ /* Try to figure out the line width; we only decode base64 * if we see two lines of the same width. */ ssize_t w1 = find64(sbuf,'\n',i+1); if(w1<0){ return; // no second delim } ssize_t linewidth1 = w1-i; // including \n if(i==0) linewidth1 += 1; // if we were not on a newline, add one if(linewidth1 < minlinewidth){ i=w1; // skip past this block continue; } ssize_t w2 = find64(sbuf,'\n',w1+1); if(w2<0){ return; // no third delim } ssize_t linewidth2 = w2-w1; if(linewidth1 != linewidth2){ i=w2; // lines are different sized; skip past both continue; } /* Now scan from w2 until we find a terminator: * - the '='. * - characters not in base64 * - the end of the sbuf. */ for(size_t j=w2+1;j<sbuf.size();j++){ /* Each line should be the same size. if this is an even module of * the start of the line and we don't have a line end, then the lines * are not properly formed. */ if(((j-w2) % linewidth1==0) && sbuf[j]!='\n'){ i = j; // advance to the end of this section break; // break out of the j loop } /* If we found a character that indicates the end of a BASE64 block * (a '=' or a '-' or a space), or we found an invalid base64 * charcter, or if we are on the last character of the sbuf, * then attempt to decode. */ char ch = sbuf[j]; bool eof = (j+1==sbuf.size()); if(eof || ch=='=' || ch=='-' || ch==' ' || (!isbase64(ch) && ch!='\n' && ch!='\r')){ size_t base64_len = j-i; if(eof || ch=='-') base64_len += 1; // we can include the termination character if(!eof && base64_len<base64_min){ // a short line? i = j; // skip this junk continue; } /* Found the end of the base64 string; process. */ managed_malloc<unsigned char>base64_target(base64_len); const char *src = (const char *)(sbuf.buf+i); if(base64_len + i > sbuf.bufsize){ // make sure it doesn't go beyond buffer base64_len = sbuf.bufsize-i; } int conv_len = b64_pton_forensic(src, base64_len, // src,srclen base64_target.buf, base64_len); // target, targetlen if(conv_len>0){ const pos0_t pos0_base64 = (sbuf.pos0 + i) + rcb.partName; const sbuf_t sbuf_base64(pos0_base64, base64_target.buf,conv_len,conv_len,false); // we will free (*rcb.callback)(scanner_params(sp,sbuf_base64)); } i = j; // advance past this section break; // break out of the j loop } } } } } }
int bulk_extractor_analyze_dev(BEFILE *bef,const char *fname,float frac,int pagesize) { bool sampling_mode = frac < 1.0; // are we in sampling mode or full-disk mode? struct stat st; if(stat(fname,&st)){ return -1; // cannot stat file } #if 0 if(S_ISREG(st.st_mode)){ // files we handle with a mapped sbuf const sbuf_t *sbuf = sbuf_t::map_file(fname); if(!sbuf) return -1; be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,*sbuf,bef->cfs)); delete sbuf; return 0; } #endif if(S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode) || S_ISREG(st.st_mode)){ /* A single-threaded sampling bulk_extractor. * It may be better to do this with two threads---one that does the reading (and seeking), * the other that doe the analysis. * * This looks like the code in phase1.cpp. */ BulkExtractor_Phase1::blocklist_t blocks_to_sample; BulkExtractor_Phase1::blocklist_t::const_iterator si = blocks_to_sample.begin(); // sampling iterator image_process *p = image_process::open(fname,false,pagesize,pagesize); image_process::iterator it = p->begin(); // get an iterator if(sampling_mode){ BulkExtractor_Phase1::make_sorted_random_blocklist(&blocks_to_sample, it.max_blocks(),frac); si = blocks_to_sample.begin(); // get the new beginning } while(true){ if(sampling_mode){ // sampling; position at the next block if(si==blocks_to_sample.end()) break; it.seek_block(*si); } else { if (it == p->end()){ // end of regular image break; } } try { sbuf_t *sbuf = it.sbuf_alloc(); if(sbuf==0) break; // eof be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,*sbuf,bef->cfs)); delete sbuf; } catch (const std::exception &e) { (*bef->cfs.cb)(bef->cfs.user,BULK_EXTRACTOR_API_EXCEPTION,0, e.what(),it.get_pos0().str().c_str(),"",0,"",0); } if(sampling_mode){ ++si; } else { ++it; } } } return 0; }
void process_path_printer(const scanner_params &sp) { /* 1. Get next token * 2. if prefix part is a number, skip forward that much in sbuf and repeat. * if the prefix is PRINT, print the buffer * if next part is a string, strip it and run that decoder. * if next part is a |, print * 3. If we are print, throw an exception to prevent continued analysis of buffer. */ std::string new_path = sp.sbuf.pos0.path; std::string prefix = get_and_remove_token(new_path); /* Time to print ?*/ if(prefix.size()==0 || prefix=="PRINT"){ uint64_t print_start = 0; uint64_t print_len = 4096; /* Check for options */ scanner_params::PrintOptions::iterator it; it = sp.print_options.find("Content-Length"); if(it!=sp.print_options.end()){ print_len = stoi64(it->second); } it = sp.print_options.find("Range"); if(it!=sp.print_options.end()){ if(it->second[5]=='='){ size_t dash = it->second.find('-'); std::string v1 = it->second.substr(6,dash-6); std::string v2 = it->second.substr(dash+1); print_start = stoi64(v1); print_len = stoi64(v2)-print_start+1; } } if(print_start>sp.sbuf.bufsize){ print_len = 0; // can't print anything } if(print_len>0 && print_start+print_len>sp.sbuf.bufsize){ print_len = sp.sbuf.bufsize-print_start; } switch(scanner_params::getPrintMode(sp.print_options)){ case scanner_params::MODE_HTTP: std::cout << "Content-Length: " << print_len << HTTP_EOL; std::cout << "Content-Range: bytes " << print_start << "-" << print_start+print_len-1 << HTTP_EOL; std::cout << "X-Range-Available: bytes " << 0 << "-" << sp.sbuf.bufsize-1 << HTTP_EOL; std::cout << HTTP_EOL; sp.sbuf.raw_dump(std::cout,print_start,print_len); // send to stdout as binary break; case scanner_params::MODE_RAW: std::cout << print_len << HTTP_EOL; std::cout.flush(); sp.sbuf.raw_dump(std::cout,print_start,print_len); // send to stdout as binary break; case scanner_params::MODE_HEX: sp.sbuf.hex_dump(std::cout,print_start,print_len); break; case scanner_params::MODE_NONE: break; } throw printing_done; //return; // our job is done } /* If we are in an offset block, process recursively with the offset */ if(isdigit(prefix[0])){ uint64_t offset = stoi64(prefix); if(offset>sp.sbuf.bufsize){ printf("Error: %s only has %u bytes; can't offset to %u\n", new_path.c_str(),(unsigned int)sp.sbuf.bufsize,(unsigned int)offset); return; } process_path_printer(scanner_params(scanner_params::PHASE_SCAN, sbuf_t(new_path,sp.sbuf+offset), sp.fs,sp.print_options)); return; } /* Find the scanner and use it */ scanner_t *s = be13::plugin::find_scanner(lowerstr(prefix)); if(s){ (*s)(scanner_params(scanner_params::PHASE_SCAN, sbuf_t(new_path,sp.sbuf), sp.fs,sp.print_options), recursion_control_block(process_path_printer,prefix)); return; } std::cerr << "Unknown name in path: " << prefix << "\n"; }