int commit(PostgreSQLConnection &pg_conn, size_t &, const char *date) {
if (!preparePGTable(pg_conn.connection(), NETWORK_TRAFFIC_SCHEMA_NAME, date,
NETWORK_TRAFFIC_TABLE_SCHEMA))
{
cerr << "networkTraffic: unable to prepare table." << endl;
return -1;
}
if (!insertPGRow(pg_conn.connection(), NETWORK_TRAFFIC_SCHEMA_NAME, date,
"%ud, %ul, %ul, %ul", liveIPs -> size(), numFlows,
numPackets, numBytes))
{
cerr << "networkTraffic: unable to insert network traffic info." << endl;
return -1;
}
return 1;
}
int commit(PostgreSQLConnection &pg_conn, size_t &flushSize, const char *date) {
Clock clock("Inserted", "rows");
if (!preparePGTable(pg_conn.connection(), DARK_SPACE_TARGETS_SCHEMA_NAME, date,
DARK_SPACE_TARGETS_TABLE_SCHEMA)) {
return -1;
}
PGBulkInserter pgBulkInserter(pg_conn.connection(), DARK_SPACE_TARGETS_SCHEMA_NAME,
date, flushSize, "%d, %ud, %ud, %ud, %ud, " \
"%ud, %ud, %ud, %ud, %ud, %ud, %ud, %ud, " \
"%d, %Vud");
cout << "Updating PostgreSQL database with dark space targets" << endl;
clock.start();
for (map <TwoWayHostPair, DarkSpaceTarget>::iterator darkSpaceTargetItr = darkSpaceTargets.begin();
darkSpaceTargetItr != darkSpaceTargets.end(); ++darkSpaceTargetItr) {
if (darkSpaceTargetItr -> second.initiator == TEMPORARILY_UNKNOWN_INITIATOR) {
darkSpaceTargetItr -> second.initiator = PERMANENTLY_UNKNOWN_INITIATOR;
}
if (!pgBulkInserter.insert(NULL, darkSpaceTargetItr -> first.protocol,
darkSpaceTargetItr -> first.internalIP,
darkSpaceTargetItr -> first.externalIP,
darkSpaceTargetItr -> first.internalPort,
darkSpaceTargetItr -> first.externalPort,
darkSpaceTargetItr -> second.initiator,
darkSpaceTargetItr -> second.numBytes,
darkSpaceTargetItr -> second.numPackets,
darkSpaceTargetItr -> second.minPacketSize,
darkSpaceTargetItr -> second.maxPacketSize,
darkSpaceTargetItr -> second.startTime,
darkSpaceTargetItr -> second.endTime,
mySharedState -> ipInformation -> getASN(darkSpaceTargetItr -> first.externalIP),
mySharedState -> ipInformation -> getCountry(darkSpaceTargetItr -> first.externalIP),
(void*)&(darkSpaceTargetItr -> second.content))) {
return -1;
}
clock.incrementOperations();
}
if (pgBulkInserter.size() && !pgBulkInserter.flush()) {
return -1;
}
clock.stop();
darkSpaceSources.clear();
darkSpaceTargets.clear();
return clock.operations();
}
bool populate_network_graph(UndirectedGraph &network_graph,
uint32_t internal_ip_count,
uint32_t external_ip_count,
const string &host_index_pair_schema,
const string &date,
PostgreSQLConnection &pg_conn)
{
network_graph = UndirectedGraph(internal_ip_count + external_ip_count);
cerr << "Network graph initialized" << endl;
string query("SELECT * FROM \"" + host_index_pair_schema + "\".\"" + date + "\";");
PGresult *result(PQexec(pg_conn.connection(), query.c_str()));
if (result == NULL || PQresultStatus(result) != PGRES_TUPLES_OK) {
cerr << "Error: PostgreSQL: " << PQerrorMessage(pg_conn.connection());
return false;
}
uint32_t internal_id;
uint32_t external_id;
size_t row_count(PQntuples(result));
for (size_t i(0); i < row_count; ++i) {
/*
if ((i + 1) % 100000 == 0) {
cerr << "DEBUG: " << i+1 << '/' << row_count << endl;
}
*/
internal_id = lexical_cast<uint32_t>(PQgetvalue(result, i, 0));
external_id = lexical_cast<uint32_t>(PQgetvalue(result, i, 1));
add_edge(vertex(internal_id, network_graph),
vertex(external_id, network_graph),
network_graph);
}
PQclear(result);
cerr << "Network graph populated." << endl;
return true;
}
int commit(PostgreSQLConnection &pg_conn, size_t &flushSize, const char *date) {
Clock clock("Inserted", "rows");
if (!preparePGTable(pg_conn.connection(), DARK_SPACE_SOURCES_SCHEMA_NAME, date,
DARK_SPACE_SOURCES_TABLE_SCHEMA)) {
return -1;
}
PGBulkInserter pgBulkInserter(pg_conn.connection(), DARK_SPACE_SOURCES_SCHEMA_NAME,
date, flushSize, "%d, %ud, %ud, %ud, %ud, " \
"%ud, %ud, %ud, %ud, %ud, %ud, %ud, %d, " \
"%Vud");
cout << "Updating PostgreSQL database with dark space sources" << endl;
clock.start();
for (map <OneWayHostPair, DarkSpaceSource>::iterator darkSpaceSourceItr = darkSpaceSources.begin();
darkSpaceSourceItr != darkSpaceSources.end(); ++darkSpaceSourceItr) {
if (!pgBulkInserter.insert(NULL, darkSpaceSourceItr -> first.protocol,
darkSpaceSourceItr -> first.sourceIP,
darkSpaceSourceItr -> first.destinationIP,
darkSpaceSourceItr -> first.sourcePort,
darkSpaceSourceItr -> first.destinationPort,
darkSpaceSourceItr -> second.numBytes,
darkSpaceSourceItr -> second.numPackets,
darkSpaceSourceItr -> second.minPacketSize,
darkSpaceSourceItr -> second.maxPacketSize,
darkSpaceSourceItr -> second.startTime,
darkSpaceSourceItr -> second.endTime,
ipInformation -> getASN(darkSpaceSourceItr -> first.sourceIP),
ipInformation -> getCountry(darkSpaceSourceItr -> first.sourceIP),
(void*)&darkSpaceSourceItr -> second.content)) {
return -1;
}
clock.incrementOperations();
}
if (pgBulkInserter.size() && !pgBulkInserter.flush()) {
return -1;
}
clock.stop();
darkSpaceSources.clear();
return clock.operations();
}
uint32_t get_host_from_index(uint32_t index,
const string &host_index_schema,
const string &date,
PostgreSQLConnection &pg_conn)
{
string query("SELECT host FROM \"" + host_index_schema + "\".\"" + date +
"\" WHERE id = '" + lexical_cast<string>(index) + "';");
PGresult *result(PQexec(pg_conn.connection(), query.c_str()));
if (result == NULL || PQresultStatus(result) != PGRES_TUPLES_OK) {
cerr << "Error: PostgreSQL: " << PQerrorMessage(pg_conn.connection());
return false;
}
size_t row_count(PQntuples(result));
if (row_count == 0) {
return 0;
}
uint32_t ret(lexical_cast<uint32_t>(PQgetvalue(result, 0, 0)));
PQclear(result);
return ret;
}
int commit(PostgreSQLConnection &pg_conn, size_t &flushSize, const char *date) {
set <uint32_t> p2pNodes;
for(rolesItr = roles->begin(); rolesItr != roles->end(); ++rolesItr){
if(rolesItr->second._role == MULTIMEDIA_P2P_NODE ||
rolesItr->second._role == UNCLASSIFIED_P2P_NODE ||
rolesItr->second._role == ENCRYPTED_P2P_NODE){
p2pNodes.insert(rolesItr->first);
}
}
if( !(p2pNodes.size())){
cout<<"No P2P Traffic"<<endl;
}
std::set <uint32_t>::iterator p2pNodesEnd(p2pNodes.end());
PGBulkInserter pgBulkInserter(pg_conn.connection(), BRUTE_FORCERS_SCHEMA_NAME, date,
flushSize, "%ud, %ud, %ud, %ud, %ud, %ud, %ud, %ud, %d");
Clock clock("Inserted", "rows");
if (!preparePGTable(pg_conn.connection(), BRUTE_FORCERS_SCHEMA_NAME, date,
BRUTE_FORCERS_TABLE_SCHEMA, false)){
return -1;
}
uint32_t sourceIP, destinationIP;
uint16_t destinationPort;
///Map that stores Fanout of attacker IP's
map <uint32_t, uint32_t> attackerIPCount;
map <uint32_t, uint32_t> ::iterator attackerItr;
///Map that stores Fanin of attacked IP's
map <uint32_t, uint32_t> attackedIPCount;
map <uint32_t, uint32_t> ::iterator attackedItr;
map <BruteHostPair, BruteForcers>::iterator
bruteEnd(bruteIPs.end());
map <BruteHostPair, BruteForcers>::iterator
bruteTemp;
uint32_t attackerIP,attackedIP;
clock.start();
//Removing connections which timed out (FIN/RST flag was not seen)
bruteItr= bruteIPs.begin();
while(bruteItr != bruteEnd){
if(bruteItr->second.numInits == 0){
bruteTemp = bruteItr;
++bruteItr;
bruteIPs.erase(bruteTemp);
}
else{
++bruteItr;
}
}
if(bruteIPs.empty())
return 0;
//Gathering number of attempts,average duration and data-connection ratio for median analysis
for(bruteItr = bruteIPs.begin(); bruteItr != bruteEnd; ++bruteItr ){
//pushing in number of handshakes,average duration and data/initialization ratio in to vectors
numInits.push_back((*bruteItr).second.numInits);
avgDuration.push_back((*bruteItr).second.avgDuration);
bytesInitsRatio.push_back((*bruteItr).second.bytesInitsRatio);
//Compete List of IP's <--Comment this section-->
/*cout<<ntop(bruteItr -> first.internalIP)
<<"\t"<<ntop(bruteItr -> first.externalIP)<<"\t";
if(bruteItr -> second.typeInit == INTERNAL_INITIATOR)
cout<<"INTERNAL INITIATOR";
else
cout<<"EXTERNAL INITIATOR";
cout<<"\t"<<bruteItr -> second.numInits
<<"\t"<<bruteItr -> second.avgDuration<<"secs"
<<"\t"<<bruteItr -> second.numBytes<<"bytes"
<<"\t"<<bruteItr -> second.bytesInitsRatio<<endl;*/
}
//Calculating Medain
sort(numInits.begin(),numInits.end());
sort(avgDuration.begin(),avgDuration.end());
sort(bytesInitsRatio.begin(),bytesInitsRatio.end());
numInitsMedian= *(numInits.begin()+numInits.size()/2);
avgDurationMedian=*(avgDuration.begin()+avgDuration.size()/2);
bytesInitsMedian=*(bytesInitsRatio.begin()+bytesInitsRatio.size()/2);
//Median statistics <--Comment this section-->
/*cout<<"Median of number of Initilizations:"<<numInitsMedian<<endl;
cout<<"Median of average duration:"<<avgDurationMedian<<endl;
cout<<"Median of ratio of Data and #Inits "<<bytesInitsMedian<<endl;
cout<<endl;*/
numInits.clear();
avgDuration.clear();
bytesInitsRatio.clear();
//Gathering Fanout and Fanin
for(bruteItr = bruteIPs.begin(); bruteItr != bruteEnd; ++bruteItr ){
if((bruteItr -> second.numInits > numInitsMedian && bruteItr -> second.bytesInitsRatio < bytesInitsMedian )||
(bruteItr -> second.avgDuration < avgDurationMedian)){
if( bruteItr -> second.typeInit == EXTERNAL_INITIATOR ){
attackerIP=bruteItr -> first.externalIP;
attackedIP=bruteItr -> first.internalIP;
}
else{
attackerIP=bruteItr -> first.internalIP;
attackedIP=bruteItr -> first.externalIP;
}
attackerItr=attackerIPCount.find(attackerIP);
attackedItr=attackedIPCount.find(attackedIP);
if(attackerItr == attackerIPCount.end())
attackerIPCount.insert(make_pair(attackerIP,1));
else
++(attackerItr -> second);
if(attackedItr == attackedIPCount.end())
attackedIPCount.insert(make_pair(attackedIP,1));
else
++(attackedItr -> second);
}
}
///Final list of Brute IP's filtered on either of these two conditions:
///(1)number of attempts > median of number of attempts and number of attempts > max number of trials for forgotten password
///(2)avg duration of connection attempts <= median of avg duration of connection attempts and Fanout > minimum set number of IPs
///(3)avg duration of connection attempts <= median of avg duration of connection attempts and Fanin > minimum set number of IPs
///Condition (1) catches bruteForcers repeatedly bruteForcing a particular target IP
///Condition (2) and (3) catch distributed brute Forcer attacks
for(bruteItr = bruteIPs.begin(); bruteItr != bruteEnd; ++bruteItr ){
if( bruteItr -> second.typeInit == EXTERNAL_INITIATOR ){
attackerIP=bruteItr -> first.externalIP;
attackedIP=bruteItr -> first.internalIP;
}
else{
attackerIP=bruteItr -> first.internalIP;
attackedIP=bruteItr -> first.externalIP;
}
attackerItr=attackerIPCount.find(attackerIP);
attackedItr=attackedIPCount.find(attackedIP);
if(p2pNodes.find(attackerIP) == p2pNodesEnd &&
((bruteItr -> second.numInits > numInitsMedian && bruteItr -> second.bytesInitsRatio < bytesInitsMedian
&& bruteItr -> second.numInits > attemptLimit)||
(bruteItr -> second.avgDuration < avgDurationMedian && attackerItr -> second > minFanout)||
(bruteItr -> second.avgDuration < avgDurationMedian && attackedItr -> second > minFanin)) ){
if(bruteItr -> second.typeInit == INTERNAL_INITIATOR){
sourceIP = bruteItr -> first.internalIP;
destinationIP = bruteItr -> first.externalIP;
destinationPort = bruteItr -> first.externalPort;
}
else{
sourceIP = bruteItr -> first.externalIP;
destinationIP = bruteItr -> first.internalIP;
destinationPort = bruteItr -> first.internalPort;
}
//Priting BruteForcers <--Comment this section-->
/*cout << ntop(sourceIP) << " -----> "
<< ntop(destinationIP) <<":"
<< destinationPort <<"\t"
<< bruteItr -> second.numInits<<"\t"
<< bruteItr -> second.avgDuration.seconds()<<"secs"<<"\t"
<< bruteItr -> second.numBytes<<"bytes"<<"\t"
<< bruteItr -> second.bytesInitsRatio<<endl;
*/
if (!pgBulkInserter.insert(NULL, sourceIP, destinationIP,
destinationPort,
bruteItr -> second.numInits,
bruteItr->second.numBytes,
bruteItr -> second.firstStartTime,
bruteItr -> second.lastEndTime,
ipInformation -> getASN(destinationIP),
ipInformation -> getCountry(destinationIP))) {
return -1;
}
clock.incrementOperations();
}
}
if (pgBulkInserter.size() && !pgBulkInserter.flush()) {
return -1;
}
clock.stop();
bruteIPs.clear();
attackerIPCount.clear();
attackedIPCount.clear();
return clock.operations();
}
