void tcpdemux::post_process(tcpip *tcp)
{
std::stringstream xmladd; // for this <fileobject>
if(opt.post_processing && tcp->file_created && tcp->last_byte>0){
/**
* After the flow is finished, if more than a byte was
* written, then put it in an SBUF and process it. if we are
* doing post-processing. This is called from tcpip::~tcpip()
* in tcpip.cpp.
*/
/* Open the fd if it is not already open */
tcp->open_file();
if(tcp->fd>=0){
sbuf_t *sbuf = sbuf_t::map_file(tcp->flow_pathname,tcp->fd);
if(sbuf){
be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,*sbuf,*(fs),&xmladd));
delete sbuf;
sbuf = 0;
}
}
}
tcp->close_file();
if(xreport) tcp->dump_xml(xreport,xmladd.str());
/**
* Before we delete the tcp structure, save information about the saved flow
*/
save_flow(tcp);
delete tcp;
}
void worker::do_work(sbuf_t *sbuf)
{
/* If logging starting and ending, save the start */
if(opt_work_start_work_end){
std::stringstream ss;
ss << "threadid='" << id << "'"
<< " pos0='" << dfxml_writer::xmlescape(sbuf->pos0.str()) << "'"
<< " pagesize='" << sbuf->pagesize << "'"
<< " bufsize='" << sbuf->bufsize << "'";
master.xreport.xmlout("debug:work_start","",ss.str(),true);
}
/**
* HERE IT IS!!!
* Construct a scanner_params() object from the sbuf that was pulled
* off the work queue and call process_extract().
*/
aftimer t;
t.start();
be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,*sbuf,master.fs));
t.stop();
/* If we are logging starting and ending, save the end */
if(opt_work_start_work_end){
std::stringstream ss;
ss << "threadid='" << id << "'"
<< " pos0='" << dfxml_writer::xmlescape(sbuf->pos0.str()) << "'"
<< " time='" << t.elapsed_seconds() << "'";
master.xreport.xmlout("debug:work_end","",ss.str(),true);
}
master.fs.flush_all();
}
int main(int argc,char **argv)
{
scanner_info::scanner_config s_config;
be13::plugin::load_scanners(scanners_builtin,s_config);
/* look for usage first */
if(argc==1 || (strcmp(argv[1],"-h")==0)){
usage();
return(1);
}
int ch;
std::string opt_outdir;
while ((ch = getopt(argc, argv, "e:o:s:x:h?")) != -1) {
switch (ch) {
case 'o': opt_outdir = optarg;break;
case 'e': be13::plugin::scanners_enable(optarg);break;
case 'x': be13::plugin::scanners_disable(optarg);break;
case 's':
{
std::vector<std::string> params = split(optarg,'=');
if(params.size()!=2){
std::cerr << "Invalid paramter: " << optarg << "\n";
exit(1);
}
//be_config[params[0]] = params[1];
continue;
}
case 'h': case '?':default:
usage();
break;
}
}
argc -= optind;
argv += optind;
if(argc!=1) usage();
//opt_scan_bulk_block_size = stoi64(be_config["bulk_block_size"]);
be13::plugin::scanners_process_enable_disable_commands();
feature_file_names_t feature_file_names;
be13::plugin::get_scanner_feature_file_names(feature_file_names);
feature_recorder_set fs(0); // where the features will be put
fs.init(feature_file_names,argv[0],opt_outdir);
be13::plugin::scanners_init(fs);
/* Make the sbuf */
sbuf_t *sbuf = sbuf_t::map_file(argv[0]);
if(!sbuf){
err(1,"Cannot map file %s:%s\n",argv[0],strerror(errno));
}
be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,*sbuf,fs));
be13::plugin::phase_shutdown(fs);
fs.process_histograms(0);
return(0);
}
int bulk_extractor_analyze_buf(BEFILE *bef,uint8_t *buf,size_t buflen)
{
pos0_t pos0("");
const sbuf_t sbuf(pos0,buf,buflen,buflen,false);
be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,sbuf,bef->cfs));
return 0;
}
/**
* After the flow is finished, put it in an SBUF and process it.
* if we are doing post-processing.
* This is called from tcpip::~tcpip() in tcpip.cpp.
*/
void tcpdemux::post_process_capture_flow(std::stringstream &xmladd,
const std::string &flow_pathname)
{
int fd2 = retrying_open(flow_pathname,O_RDONLY|O_BINARY,0);
if(fd2<0) {
perror("open");
return;
}
sbuf_t *sbuf = sbuf_t::map_file(flow_pathname,pos0_t(flow_pathname),fd2);
if(sbuf) {
process_sbuf(scanner_params(scanner_params::scan,*sbuf,*fs,&xmladd));
}
::close(fd2);
}
/* Found the end of the base64 string; process. */
inline void process(const class scanner_params &sp,const recursion_control_block &rcb,size_t start,size_t len)
{
//fprintf(stderr,"process start=%zd len=%zd\n",start,len);
//fprintf(stderr,"To convert:\n");
const sbuf_t &sbuf = sp.sbuf;
managed_malloc<unsigned char>base64_target(len);
const char *src = (const char *)(sbuf.buf+start);
if(len + start > sbuf.bufsize){ // make sure it doesn't go beyond buffer
len = sbuf.bufsize-start;
}
//fwrite(src,len,1,stderr);
//fprintf(stderr,"\n======\n");
int conv_len = b64_pton_forensic(src, len, // src,srclen
base64_target.buf, len); // target, targetlen
if(conv_len>0){
//fprintf(stderr,"conv_len=%zd\n",conv_len);
//fwrite(base64_target.buf,1,conv_len,stderr);
const pos0_t pos0_base64 = (sbuf.pos0 + start) + rcb.partName;
const sbuf_t sbuf_base64(pos0_base64, base64_target.buf,conv_len,conv_len,false); // we will free
(*rcb.callback)(scanner_params(sp,sbuf_base64));
}
}
void scan_base64(const class scanner_params &sp,const recursion_control_block &rcb)
{
assert(sp.sp_version==scanner_params::CURRENT_SP_VERSION);
if(sp.phase==scanner_params::PHASE_STARTUP){
assert(sp.info->si_version==scanner_info::CURRENT_SI_VERSION);
sp.info->name = "base64";
sp.info->author = "Simson L. Garfinkel";
sp.info->description = "scans for Base64-encoded data";
sp.info->scanner_version= "1.0";
/* Create the base64 array */
memset(base64array,0,sizeof(base64array));
base64array[(int)'+'] = true;
base64array[(int)'/'] = true;
for(int ch='a';ch<='z';ch++){ base64array[ch] = true; }
for(int ch='A';ch<='Z';ch++){ base64array[ch] = true; }
for(int ch='0';ch<='9';ch++){ base64array[ch] = true; }
return; /* No feature files created */
}
if(sp.phase==scanner_params::PHASE_SHUTDOWN) return;
if(sp.phase==scanner_params::PHASE_SCAN){
const sbuf_t &sbuf = sp.sbuf;
/* base64 is a newline followed by at least two lines of constant length,
* followed by an incomplete line ending with an equal sign.
* Lines can be termianted by \n or \r\n. This code simply ignores \r,
* which means that you can't have some lines terminated by \n and other
* terminated by \n\r.
*
* Note that this doesn't scan base64-encoded blobs smaller than two lines.
* Perhaps we should do that.
*/
for(size_t i=0;i<sbuf.pagesize;i++){
if(i==0 || sbuf[i]=='\n' ){
/* Try to figure out the line width; we only decode base64
* if we see two lines of the same width.
*/
ssize_t w1 = find64(sbuf,'\n',i+1);
if(w1<0){
return; // no second delim
}
ssize_t linewidth1 = w1-i; // including \n
if(i==0) linewidth1 += 1; // if we were not on a newline, add one
if(linewidth1 < minlinewidth){
i=w1; // skip past this block
continue;
}
ssize_t w2 = find64(sbuf,'\n',w1+1);
if(w2<0){
return; // no third delim
}
ssize_t linewidth2 = w2-w1;
if(linewidth1 != linewidth2){
i=w2; // lines are different sized; skip past both
continue;
}
/* Now scan from w2 until we find a terminator:
* - the '='.
* - characters not in base64
* - the end of the sbuf.
*/
for(size_t j=w2+1;j<sbuf.size();j++){
/* Each line should be the same size. if this is an even module of
* the start of the line and we don't have a line end, then the lines
* are not properly formed.
*/
if(((j-w2) % linewidth1==0) && sbuf[j]!='\n'){
i = j; // advance to the end of this section
break; // break out of the j loop
}
/* If we found a character that indicates the end of a BASE64 block
* (a '=' or a '-' or a space), or we found an invalid base64
* charcter, or if we are on the last character of the sbuf,
* then attempt to decode.
*/
char ch = sbuf[j];
bool eof = (j+1==sbuf.size());
if(eof || ch=='=' || ch=='-' || ch==' ' || (!isbase64(ch) && ch!='\n' && ch!='\r')){
size_t base64_len = j-i;
if(eof || ch=='-') base64_len += 1; // we can include the termination character
if(!eof && base64_len<base64_min){ // a short line?
i = j; // skip this junk
continue;
}
/* Found the end of the base64 string; process. */
managed_malloc<unsigned char>base64_target(base64_len);
const char *src = (const char *)(sbuf.buf+i);
if(base64_len + i > sbuf.bufsize){ // make sure it doesn't go beyond buffer
base64_len = sbuf.bufsize-i;
}
int conv_len = b64_pton_forensic(src, base64_len, // src,srclen
base64_target.buf, base64_len); // target, targetlen
if(conv_len>0){
const pos0_t pos0_base64 = (sbuf.pos0 + i) + rcb.partName;
const sbuf_t sbuf_base64(pos0_base64, base64_target.buf,conv_len,conv_len,false); // we will free
(*rcb.callback)(scanner_params(sp,sbuf_base64));
}
i = j; // advance past this section
break; // break out of the j loop
}
}
}
}
}
}
int bulk_extractor_analyze_dev(BEFILE *bef,const char *fname,float frac,int pagesize)
{
bool sampling_mode = frac < 1.0; // are we in sampling mode or full-disk mode?
struct stat st;
if(stat(fname,&st)){
return -1; // cannot stat file
}
#if 0
if(S_ISREG(st.st_mode)){ // files we handle with a mapped sbuf
const sbuf_t *sbuf = sbuf_t::map_file(fname);
if(!sbuf) return -1;
be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,*sbuf,bef->cfs));
delete sbuf;
return 0;
}
#endif
if(S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode) || S_ISREG(st.st_mode)){
/* A single-threaded sampling bulk_extractor.
* It may be better to do this with two threads---one that does the reading (and seeking),
* the other that doe the analysis.
*
* This looks like the code in phase1.cpp.
*/
BulkExtractor_Phase1::blocklist_t blocks_to_sample;
BulkExtractor_Phase1::blocklist_t::const_iterator si = blocks_to_sample.begin(); // sampling iterator
image_process *p = image_process::open(fname,false,pagesize,pagesize);
image_process::iterator it = p->begin(); // get an iterator
if(sampling_mode){
BulkExtractor_Phase1::make_sorted_random_blocklist(&blocks_to_sample, it.max_blocks(),frac);
si = blocks_to_sample.begin(); // get the new beginning
}
while(true){
if(sampling_mode){ // sampling; position at the next block
if(si==blocks_to_sample.end()) break;
it.seek_block(*si);
} else {
if (it == p->end()){ // end of regular image
break;
}
}
try {
sbuf_t *sbuf = it.sbuf_alloc();
if(sbuf==0) break; // eof
be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,*sbuf,bef->cfs));
delete sbuf;
}
catch (const std::exception &e) {
(*bef->cfs.cb)(bef->cfs.user,BULK_EXTRACTOR_API_EXCEPTION,0,
e.what(),it.get_pos0().str().c_str(),"",0,"",0);
}
if(sampling_mode){
++si;
} else {
++it;
}
}
}
return 0;
}
void process_path_printer(const scanner_params &sp)
{
/* 1. Get next token
* 2. if prefix part is a number, skip forward that much in sbuf and repeat.
* if the prefix is PRINT, print the buffer
* if next part is a string, strip it and run that decoder.
* if next part is a |, print
* 3. If we are print, throw an exception to prevent continued analysis of buffer.
*/
std::string new_path = sp.sbuf.pos0.path;
std::string prefix = get_and_remove_token(new_path);
/* Time to print ?*/
if(prefix.size()==0 || prefix=="PRINT"){
uint64_t print_start = 0;
uint64_t print_len = 4096;
/* Check for options */
scanner_params::PrintOptions::iterator it;
it = sp.print_options.find("Content-Length");
if(it!=sp.print_options.end()){
print_len = stoi64(it->second);
}
it = sp.print_options.find("Range");
if(it!=sp.print_options.end()){
if(it->second[5]=='='){
size_t dash = it->second.find('-');
std::string v1 = it->second.substr(6,dash-6);
std::string v2 = it->second.substr(dash+1);
print_start = stoi64(v1);
print_len = stoi64(v2)-print_start+1;
}
}
if(print_start>sp.sbuf.bufsize){
print_len = 0; // can't print anything
}
if(print_len>0 && print_start+print_len>sp.sbuf.bufsize){
print_len = sp.sbuf.bufsize-print_start;
}
switch(scanner_params::getPrintMode(sp.print_options)){
case scanner_params::MODE_HTTP:
std::cout << "Content-Length: " << print_len << HTTP_EOL;
std::cout << "Content-Range: bytes " << print_start << "-" << print_start+print_len-1 << HTTP_EOL;
std::cout << "X-Range-Available: bytes " << 0 << "-" << sp.sbuf.bufsize-1 << HTTP_EOL;
std::cout << HTTP_EOL;
sp.sbuf.raw_dump(std::cout,print_start,print_len); // send to stdout as binary
break;
case scanner_params::MODE_RAW:
std::cout << print_len << HTTP_EOL;
std::cout.flush();
sp.sbuf.raw_dump(std::cout,print_start,print_len); // send to stdout as binary
break;
case scanner_params::MODE_HEX:
sp.sbuf.hex_dump(std::cout,print_start,print_len);
break;
case scanner_params::MODE_NONE:
break;
}
throw printing_done;
//return; // our job is done
}
/* If we are in an offset block, process recursively with the offset */
if(isdigit(prefix[0])){
uint64_t offset = stoi64(prefix);
if(offset>sp.sbuf.bufsize){
printf("Error: %s only has %u bytes; can't offset to %u\n",
new_path.c_str(),(unsigned int)sp.sbuf.bufsize,(unsigned int)offset);
return;
}
process_path_printer(scanner_params(scanner_params::PHASE_SCAN,
sbuf_t(new_path,sp.sbuf+offset),
sp.fs,sp.print_options));
return;
}
/* Find the scanner and use it */
scanner_t *s = be13::plugin::find_scanner(lowerstr(prefix));
if(s){
(*s)(scanner_params(scanner_params::PHASE_SCAN,
sbuf_t(new_path,sp.sbuf),
sp.fs,sp.print_options),
recursion_control_block(process_path_printer,prefix));
return;
}
std::cerr << "Unknown name in path: " << prefix << "\n";
}