int commit(PostgreSQLConnection &pg_conn, size_t &, const char *date) { if (!preparePGTable(pg_conn.connection(), NETWORK_TRAFFIC_SCHEMA_NAME, date, NETWORK_TRAFFIC_TABLE_SCHEMA)) { cerr << "networkTraffic: unable to prepare table." << endl; return -1; } if (!insertPGRow(pg_conn.connection(), NETWORK_TRAFFIC_SCHEMA_NAME, date, "%ud, %ul, %ul, %ul", liveIPs -> size(), numFlows, numPackets, numBytes)) { cerr << "networkTraffic: unable to insert network traffic info." << endl; return -1; } return 1; }
int commit(PostgreSQLConnection &pg_conn, size_t &flushSize, const char *date) { Clock clock("Inserted", "rows"); if (!preparePGTable(pg_conn.connection(), DARK_SPACE_TARGETS_SCHEMA_NAME, date, DARK_SPACE_TARGETS_TABLE_SCHEMA)) { return -1; } PGBulkInserter pgBulkInserter(pg_conn.connection(), DARK_SPACE_TARGETS_SCHEMA_NAME, date, flushSize, "%d, %ud, %ud, %ud, %ud, " \ "%ud, %ud, %ud, %ud, %ud, %ud, %ud, %ud, " \ "%d, %Vud"); cout << "Updating PostgreSQL database with dark space targets" << endl; clock.start(); for (map <TwoWayHostPair, DarkSpaceTarget>::iterator darkSpaceTargetItr = darkSpaceTargets.begin(); darkSpaceTargetItr != darkSpaceTargets.end(); ++darkSpaceTargetItr) { if (darkSpaceTargetItr -> second.initiator == TEMPORARILY_UNKNOWN_INITIATOR) { darkSpaceTargetItr -> second.initiator = PERMANENTLY_UNKNOWN_INITIATOR; } if (!pgBulkInserter.insert(NULL, darkSpaceTargetItr -> first.protocol, darkSpaceTargetItr -> first.internalIP, darkSpaceTargetItr -> first.externalIP, darkSpaceTargetItr -> first.internalPort, darkSpaceTargetItr -> first.externalPort, darkSpaceTargetItr -> second.initiator, darkSpaceTargetItr -> second.numBytes, darkSpaceTargetItr -> second.numPackets, darkSpaceTargetItr -> second.minPacketSize, darkSpaceTargetItr -> second.maxPacketSize, darkSpaceTargetItr -> second.startTime, darkSpaceTargetItr -> second.endTime, mySharedState -> ipInformation -> getASN(darkSpaceTargetItr -> first.externalIP), mySharedState -> ipInformation -> getCountry(darkSpaceTargetItr -> first.externalIP), (void*)&(darkSpaceTargetItr -> second.content))) { return -1; } clock.incrementOperations(); } if (pgBulkInserter.size() && !pgBulkInserter.flush()) { return -1; } clock.stop(); darkSpaceSources.clear(); darkSpaceTargets.clear(); return clock.operations(); }
bool populate_network_graph(UndirectedGraph &network_graph, uint32_t internal_ip_count, uint32_t external_ip_count, const string &host_index_pair_schema, const string &date, PostgreSQLConnection &pg_conn) { network_graph = UndirectedGraph(internal_ip_count + external_ip_count); cerr << "Network graph initialized" << endl; string query("SELECT * FROM \"" + host_index_pair_schema + "\".\"" + date + "\";"); PGresult *result(PQexec(pg_conn.connection(), query.c_str())); if (result == NULL || PQresultStatus(result) != PGRES_TUPLES_OK) { cerr << "Error: PostgreSQL: " << PQerrorMessage(pg_conn.connection()); return false; } uint32_t internal_id; uint32_t external_id; size_t row_count(PQntuples(result)); for (size_t i(0); i < row_count; ++i) { /* if ((i + 1) % 100000 == 0) { cerr << "DEBUG: " << i+1 << '/' << row_count << endl; } */ internal_id = lexical_cast<uint32_t>(PQgetvalue(result, i, 0)); external_id = lexical_cast<uint32_t>(PQgetvalue(result, i, 1)); add_edge(vertex(internal_id, network_graph), vertex(external_id, network_graph), network_graph); } PQclear(result); cerr << "Network graph populated." << endl; return true; }
int commit(PostgreSQLConnection &pg_conn, size_t &flushSize, const char *date) { Clock clock("Inserted", "rows"); if (!preparePGTable(pg_conn.connection(), DARK_SPACE_SOURCES_SCHEMA_NAME, date, DARK_SPACE_SOURCES_TABLE_SCHEMA)) { return -1; } PGBulkInserter pgBulkInserter(pg_conn.connection(), DARK_SPACE_SOURCES_SCHEMA_NAME, date, flushSize, "%d, %ud, %ud, %ud, %ud, " \ "%ud, %ud, %ud, %ud, %ud, %ud, %ud, %d, " \ "%Vud"); cout << "Updating PostgreSQL database with dark space sources" << endl; clock.start(); for (map <OneWayHostPair, DarkSpaceSource>::iterator darkSpaceSourceItr = darkSpaceSources.begin(); darkSpaceSourceItr != darkSpaceSources.end(); ++darkSpaceSourceItr) { if (!pgBulkInserter.insert(NULL, darkSpaceSourceItr -> first.protocol, darkSpaceSourceItr -> first.sourceIP, darkSpaceSourceItr -> first.destinationIP, darkSpaceSourceItr -> first.sourcePort, darkSpaceSourceItr -> first.destinationPort, darkSpaceSourceItr -> second.numBytes, darkSpaceSourceItr -> second.numPackets, darkSpaceSourceItr -> second.minPacketSize, darkSpaceSourceItr -> second.maxPacketSize, darkSpaceSourceItr -> second.startTime, darkSpaceSourceItr -> second.endTime, ipInformation -> getASN(darkSpaceSourceItr -> first.sourceIP), ipInformation -> getCountry(darkSpaceSourceItr -> first.sourceIP), (void*)&darkSpaceSourceItr -> second.content)) { return -1; } clock.incrementOperations(); } if (pgBulkInserter.size() && !pgBulkInserter.flush()) { return -1; } clock.stop(); darkSpaceSources.clear(); return clock.operations(); }
uint32_t get_host_from_index(uint32_t index, const string &host_index_schema, const string &date, PostgreSQLConnection &pg_conn) { string query("SELECT host FROM \"" + host_index_schema + "\".\"" + date + "\" WHERE id = '" + lexical_cast<string>(index) + "';"); PGresult *result(PQexec(pg_conn.connection(), query.c_str())); if (result == NULL || PQresultStatus(result) != PGRES_TUPLES_OK) { cerr << "Error: PostgreSQL: " << PQerrorMessage(pg_conn.connection()); return false; } size_t row_count(PQntuples(result)); if (row_count == 0) { return 0; } uint32_t ret(lexical_cast<uint32_t>(PQgetvalue(result, 0, 0))); PQclear(result); return ret; }
int commit(PostgreSQLConnection &pg_conn, size_t &flushSize, const char *date) { set <uint32_t> p2pNodes; for(rolesItr = roles->begin(); rolesItr != roles->end(); ++rolesItr){ if(rolesItr->second._role == MULTIMEDIA_P2P_NODE || rolesItr->second._role == UNCLASSIFIED_P2P_NODE || rolesItr->second._role == ENCRYPTED_P2P_NODE){ p2pNodes.insert(rolesItr->first); } } if( !(p2pNodes.size())){ cout<<"No P2P Traffic"<<endl; } std::set <uint32_t>::iterator p2pNodesEnd(p2pNodes.end()); PGBulkInserter pgBulkInserter(pg_conn.connection(), BRUTE_FORCERS_SCHEMA_NAME, date, flushSize, "%ud, %ud, %ud, %ud, %ud, %ud, %ud, %ud, %d"); Clock clock("Inserted", "rows"); if (!preparePGTable(pg_conn.connection(), BRUTE_FORCERS_SCHEMA_NAME, date, BRUTE_FORCERS_TABLE_SCHEMA, false)){ return -1; } uint32_t sourceIP, destinationIP; uint16_t destinationPort; ///Map that stores Fanout of attacker IP's map <uint32_t, uint32_t> attackerIPCount; map <uint32_t, uint32_t> ::iterator attackerItr; ///Map that stores Fanin of attacked IP's map <uint32_t, uint32_t> attackedIPCount; map <uint32_t, uint32_t> ::iterator attackedItr; map <BruteHostPair, BruteForcers>::iterator bruteEnd(bruteIPs.end()); map <BruteHostPair, BruteForcers>::iterator bruteTemp; uint32_t attackerIP,attackedIP; clock.start(); //Removing connections which timed out (FIN/RST flag was not seen) bruteItr= bruteIPs.begin(); while(bruteItr != bruteEnd){ if(bruteItr->second.numInits == 0){ bruteTemp = bruteItr; ++bruteItr; bruteIPs.erase(bruteTemp); } else{ ++bruteItr; } } if(bruteIPs.empty()) return 0; //Gathering number of attempts,average duration and data-connection ratio for median analysis for(bruteItr = bruteIPs.begin(); bruteItr != bruteEnd; ++bruteItr ){ //pushing in number of handshakes,average duration and data/initialization ratio in to vectors numInits.push_back((*bruteItr).second.numInits); avgDuration.push_back((*bruteItr).second.avgDuration); bytesInitsRatio.push_back((*bruteItr).second.bytesInitsRatio); //Compete List of IP's <--Comment this section--> /*cout<<ntop(bruteItr -> first.internalIP) <<"\t"<<ntop(bruteItr -> first.externalIP)<<"\t"; if(bruteItr -> second.typeInit == INTERNAL_INITIATOR) cout<<"INTERNAL INITIATOR"; else cout<<"EXTERNAL INITIATOR"; cout<<"\t"<<bruteItr -> second.numInits <<"\t"<<bruteItr -> second.avgDuration<<"secs" <<"\t"<<bruteItr -> second.numBytes<<"bytes" <<"\t"<<bruteItr -> second.bytesInitsRatio<<endl;*/ } //Calculating Medain sort(numInits.begin(),numInits.end()); sort(avgDuration.begin(),avgDuration.end()); sort(bytesInitsRatio.begin(),bytesInitsRatio.end()); numInitsMedian= *(numInits.begin()+numInits.size()/2); avgDurationMedian=*(avgDuration.begin()+avgDuration.size()/2); bytesInitsMedian=*(bytesInitsRatio.begin()+bytesInitsRatio.size()/2); //Median statistics <--Comment this section--> /*cout<<"Median of number of Initilizations:"<<numInitsMedian<<endl; cout<<"Median of average duration:"<<avgDurationMedian<<endl; cout<<"Median of ratio of Data and #Inits "<<bytesInitsMedian<<endl; cout<<endl;*/ numInits.clear(); avgDuration.clear(); bytesInitsRatio.clear(); //Gathering Fanout and Fanin for(bruteItr = bruteIPs.begin(); bruteItr != bruteEnd; ++bruteItr ){ if((bruteItr -> second.numInits > numInitsMedian && bruteItr -> second.bytesInitsRatio < bytesInitsMedian )|| (bruteItr -> second.avgDuration < avgDurationMedian)){ if( bruteItr -> second.typeInit == EXTERNAL_INITIATOR ){ attackerIP=bruteItr -> first.externalIP; attackedIP=bruteItr -> first.internalIP; } else{ attackerIP=bruteItr -> first.internalIP; attackedIP=bruteItr -> first.externalIP; } attackerItr=attackerIPCount.find(attackerIP); attackedItr=attackedIPCount.find(attackedIP); if(attackerItr == attackerIPCount.end()) attackerIPCount.insert(make_pair(attackerIP,1)); else ++(attackerItr -> second); if(attackedItr == attackedIPCount.end()) attackedIPCount.insert(make_pair(attackedIP,1)); else ++(attackedItr -> second); } } ///Final list of Brute IP's filtered on either of these two conditions: ///(1)number of attempts > median of number of attempts and number of attempts > max number of trials for forgotten password ///(2)avg duration of connection attempts <= median of avg duration of connection attempts and Fanout > minimum set number of IPs ///(3)avg duration of connection attempts <= median of avg duration of connection attempts and Fanin > minimum set number of IPs ///Condition (1) catches bruteForcers repeatedly bruteForcing a particular target IP ///Condition (2) and (3) catch distributed brute Forcer attacks for(bruteItr = bruteIPs.begin(); bruteItr != bruteEnd; ++bruteItr ){ if( bruteItr -> second.typeInit == EXTERNAL_INITIATOR ){ attackerIP=bruteItr -> first.externalIP; attackedIP=bruteItr -> first.internalIP; } else{ attackerIP=bruteItr -> first.internalIP; attackedIP=bruteItr -> first.externalIP; } attackerItr=attackerIPCount.find(attackerIP); attackedItr=attackedIPCount.find(attackedIP); if(p2pNodes.find(attackerIP) == p2pNodesEnd && ((bruteItr -> second.numInits > numInitsMedian && bruteItr -> second.bytesInitsRatio < bytesInitsMedian && bruteItr -> second.numInits > attemptLimit)|| (bruteItr -> second.avgDuration < avgDurationMedian && attackerItr -> second > minFanout)|| (bruteItr -> second.avgDuration < avgDurationMedian && attackedItr -> second > minFanin)) ){ if(bruteItr -> second.typeInit == INTERNAL_INITIATOR){ sourceIP = bruteItr -> first.internalIP; destinationIP = bruteItr -> first.externalIP; destinationPort = bruteItr -> first.externalPort; } else{ sourceIP = bruteItr -> first.externalIP; destinationIP = bruteItr -> first.internalIP; destinationPort = bruteItr -> first.internalPort; } //Priting BruteForcers <--Comment this section--> /*cout << ntop(sourceIP) << " -----> " << ntop(destinationIP) <<":" << destinationPort <<"\t" << bruteItr -> second.numInits<<"\t" << bruteItr -> second.avgDuration.seconds()<<"secs"<<"\t" << bruteItr -> second.numBytes<<"bytes"<<"\t" << bruteItr -> second.bytesInitsRatio<<endl; */ if (!pgBulkInserter.insert(NULL, sourceIP, destinationIP, destinationPort, bruteItr -> second.numInits, bruteItr->second.numBytes, bruteItr -> second.firstStartTime, bruteItr -> second.lastEndTime, ipInformation -> getASN(destinationIP), ipInformation -> getCountry(destinationIP))) { return -1; } clock.incrementOperations(); } } if (pgBulkInserter.size() && !pgBulkInserter.flush()) { return -1; } clock.stop(); bruteIPs.clear(); attackerIPCount.clear(); attackedIPCount.clear(); return clock.operations(); }