void tcpdemux::post_process(tcpip *tcp)
{
std::stringstream xmladd; // for this <fileobject>
if(opt.post_processing && tcp->file_created && tcp->last_byte>0){
/**
* After the flow is finished, if more than a byte was
* written, then put it in an SBUF and process it. if we are
* doing post-processing. This is called from tcpip::~tcpip()
* in tcpip.cpp.
*/
/* Open the fd if it is not already open */
tcp->open_file();
if(tcp->fd>=0){
sbuf_t *sbuf = sbuf_t::map_file(tcp->flow_pathname,pos0_t(tcp->flow_pathname),tcp->fd);
if(sbuf){
be13::plugin::process_sbuf(scanner_params(scanner_params::PHASE_SCAN,*sbuf,*(fs),&xmladd));
delete sbuf;
sbuf = 0;
}
}
}
tcp->close_file();
if(xreport) tcp->dump_xml(xreport,xmladd.str());
/**
* Before we delete the tcp structure, save information about the saved flow
*/
save_flow(tcp);
delete tcp;
}
static void process_gzip(class tcpdemux &demux,
std::stringstream &ss,
const std::string &fname,const unsigned char *base,size_t len)
{
if((len>4) && (base[0]==0x1f) && (base[1]==0x8b) && (base[2]==0x08) && (base[3]==0x00)) {
size_t uncompr_size = len * 16;
unsigned char *decompress_buf = (unsigned char *)malloc(uncompr_size);
if(decompress_buf==0) return; // too big?
z_stream zs;
memset(&zs,0,sizeof(zs));
zs.next_in = (Bytef *)base; // note that next_in should be typedef const but is not
zs.avail_in = len;
zs.next_out = decompress_buf;
zs.avail_out = uncompr_size;
int r = inflateInit2(&zs,16+MAX_WBITS);
if(r==0) {
r = inflate(&zs,Z_SYNC_FLUSH);
/* Ignore the error return; process data if we got anything */
if(zs.total_out>0) {
demux.write_to_file(ss,fname,sbuf_t(pos0_t(),decompress_buf,zs.total_out,zs.total_out,false));
}
inflateEnd(&zs);
}
free(decompress_buf);
}
}
int tcpdemux::process_ip4(const be13::packet_info &pi)
{
/* make sure that the packet is at least as long as the min IP header */
if (pi.ip_datalen < sizeof(struct be13::ip4)) {
DEBUG(6) ("received truncated IP datagram!");
return -1; // couldn't process
}
const struct be13::ip4 *ip_header = (struct be13::ip4 *) pi.ip_data;
DEBUG(100)("process_ip4. caplen=%d vlan=%d ip_p=%d",(int)pi.pcap_hdr->caplen,(int)pi.vlan(),(int)ip_header->ip_p);
if(debug>200){
sbuf_t sbuf(pos0_t(),(const uint8_t *)pi.ip_data,pi.ip_datalen,pi.ip_datalen,false);
sbuf.hex_dump(std::cerr);
}
/* for now we're only looking for TCP; throw away everything else */
if (ip_header->ip_p != IPPROTO_TCP) {
DEBUG(50) ("got non-TCP frame -- IP proto %d", ip_header->ip_p);
return -1; // couldn't process
}
/* check and see if we got everything. NOTE: we must use
* ip_total_len after this, because we may have captured bytes
* beyond the end of the packet (e.g. ethernet padding).
*/
size_t ip_len = ntohs(ip_header->ip_len);
if (pi.ip_datalen < ip_len) {
DEBUG(6) ("warning: captured only %ld bytes of %ld-byte IP datagram",
(long) pi.ip_datalen, (long) ip_len);
}
/* XXX - throw away everything but fragment 0; this version doesn't
* know how to do fragment reassembly.
*/
if (ntohs(ip_header->ip_off) & 0x1fff) {
DEBUG(2) ("warning: throwing away IP fragment from X to X");
return -1;
}
/* figure out where the IP header ends */
size_t ip_header_len = ip_header->ip_hl * 4;
/* make sure there's some data */
if (ip_header_len > ip_len) {
DEBUG(6) ("received truncated IP datagram!");
return -1;
}
/* do TCP processing, faking an ipv6 address */
uint16_t ip_payload_len = ip_len - ip_header_len;
ipaddr src(ip_header->ip_src.addr);
ipaddr dst(ip_header->ip_dst.addr);
return process_tcp(src, dst, AF_INET,
pi.ip_data + ip_header_len, ip_payload_len,
pi);
}
/**
* After the flow is finished, put it in an SBUF and process it.
* if we are doing post-processing.
* This is called from tcpip::~tcpip() in tcpip.cpp.
*/
void tcpdemux::post_process_capture_flow(std::stringstream &xmladd,
const std::string &flow_pathname)
{
int fd2 = retrying_open(flow_pathname,O_RDONLY|O_BINARY,0);
if(fd2<0) {
perror("open");
return;
}
sbuf_t *sbuf = sbuf_t::map_file(flow_pathname,pos0_t(flow_pathname),fd2);
if(sbuf) {
process_sbuf(scanner_params(scanner_params::scan,*sbuf,*fs,&xmladd));
}
::close(fd2);
}
sbuf_t *sbuf_t::map_file(const std::string &fname,int fd)
{
struct stat st;
if(fstat(fd,&st)){
close(fd);
return 0; /* cannot stat */
}
#ifdef HAVE_MMAP
uint8_t *buf = (uint8_t *)mmap(0,st.st_size,PROT_READ,MAP_FILE|MAP_SHARED,fd,0);
bool should_free = false;
bool should_unmap = true;
#else
uint8_t *buf = (uint8_t *)malloc(st.st_size);
if(buf==0){ /* malloc failed */
return 0;
}
lseek(fd,0,SEEK_SET); // go to beginning of file
size_t r = (size_t)read(fd,(void *)buf,st.st_size);
if(r!=(size_t)st.st_size){
free((void *)buf); /* read failed */
return 0;
}
close(fd);
fd = 0;
bool should_free = true;
bool should_unmap = false;
#endif
sbuf_t *sbuf = new sbuf_t(pos0_t(fname+sbuf_t::map_file_delimiter),
buf,
st.st_size,
st.st_size,
fd,
should_unmap,
should_free,
false); // the caller's job is to close
return sbuf;
}
void Handle80211DataFromAP(const struct timeval& t, const data_hdr_t *hdr, const u_char *rest, int len) {
if (opt_enforce_80211_frame_checksum && !fcs_ok) return;
#ifdef DEBUG_WIFI
cout << hdr->sa;
cout << " " << "802.11 data from AP:\t"
<< hdr->sa << " -> " << hdr->da << "\t" << len << endl;
#endif
struct timeval tv;
/* TK1: Does the pcap header make sense? */
/* TK2: How do we get and preserve the the three MAC addresses? */
printf("DATA_HDRLEN=%d DATA_WDS_HDRLEN=%d\n",DATA_HDRLEN,DATA_WDS_HDRLEN);
sbuf_t sb(pos0_t(),rest,len,len,0);
sb.hex_dump(std::cout);
rest += 10; // where does 10 come from?
len -= 10;
be13::packet_info pi(DLT_IEEE802_11,(const pcap_pkthdr *)0,(const u_char *)0,tvshift(tv,t),rest,len);
printf("pi.ip_version=%d\n",pi.ip_version());
process_packet_info(pi);
}
static void bulk_process_feature_file(const std::string &fn)
{
if(ends_with(fn,".txt")==false) return; // don't process binary files
if(ends_with(fn,"_histogram.txt")==true) return; // ignore histogram files
const string features(fn.substr(fn.rfind('/')+1,fn.size()-fn.rfind('/')-5));
const string name(features+": ");
const string SPACE(" ");
const string UNKNOWN("UNKNOWN");
bool tagfile = ends_with(fn,"_tags.txt");
ifstream f(fn.c_str());
if(!f.is_open()){
cerr << "Cannot open tag input file: " << fn << "\n";
return;
}
try {
string line;
while(getline(f,line)){
if(line.size()==0 || line[0]=='#' || line.substr(0,4)=="\357\273\277#") continue;
vector<string> fields = split(line,'\t'); // fields of the feature file
if(fields.size()<2) continue; // improper formatting
std::string &taglocation = fields[0];
std::string &tagtype = fields[1];
uint64_t offset = stoi64(taglocation);
uint64_t sector = offset / opt_bulk_block_size;
/* If the array hasn't been expanded to the point of this element, expand it with blanks */
while(sector > sector_typetags.size()){
sector_typetags.push_back(sector_typetag()); // expand to fill gap
}
if(tagfile){ // first pass
/* Process a tag */
vector<string> vals = split(taglocation,':');
if(vals.size()!=2){
std::cerr << "Invalid tag file line: " << line << " (size=" << vals.size() << ")\n";
exit(1);
}
uint32_t len = stoi(vals[1]);
// If no data for this sector, simply append this type
// and then continue
if(sector_typetags.size()==sector){
sector_typetags.push_back(sector_typetag(len,tagtype,string("")));
continue;
}
// We have new data for the same element. Which is better?
if(sector_typetags[sector].specificity() < sector_typetag::specificity(tagtype)){
// New is more specific than the old.
// Preserve the old one
sector_typetags[sector].scomment = sector_typetags[sector].stype + string("; ") + sector_typetags[sector].scomment;
sector_typetags[sector].stype = tagtype; // specify new tag type
} else {
// New is less specific than the old, so just make new type a comment.
sector_typetags[sector].scomment = tagtype + string("; ") + sector_typetags[sector].scomment;
}
continue;
}
/* Process a feature, which will add specificity to the tag */
if(sector_typetags.size()==sector){
/* Hm... No tag (and all tags got processed first), so this is unknown */
sector_typetags.push_back(sector_typetag(opt_bulk_block_size,UNKNOWN,SPACE));
}
/* append what we've learned regarding this feature */
// If we got an MD5 as field1 and there is a second field, go with that
sector_typetag &s = sector_typetags[sector];
int field = 1;
if(fields.size()>2 && fields[1].size()==32 && fields[2].size()>0 && fields[2][0]=='<'){
field = 2; // go with the second field
}
s.scomment += " " + name + fields[field];
// append any XML if it is present
if(field==1 && fields.size()>2 && fields[2].size()>0 && fields[2][0]=='<'){
s.scomment += " " + name + " " + fields[2];
}
// Scan through the feature indicator table and if we find a match note the type
for(int i=0;feature_indicators[i].feature_file_name;i++){
if(features!=feature_indicators[i].feature_file_name) continue;
if(feature_indicators[i].feature_content==0
|| fields[1].find(feature_indicators[i].feature_content)!=string::npos
|| fields[2].find(feature_indicators[i].feature_content)!=string::npos){
s.stype = pos0_t(fields[0]).alphaPart();
if(s.stype.size()>1){
char lastchar = s.stype.at(s.stype.size()-1);
if(lastchar!='-' && lastchar!='/') s.stype += string("-");
}
s.stype += feature_indicators[i].dfrws_type;
}
}
}
}
catch (const std::exception &e) {
cerr << "ERROR: " << e.what() << " processing tagfile " << fn << "\n";
}
}