void assignAction(vector<string> &flow,vector<size_t> &flow_cnt,
vector<int> &flowaction,vector<size_t> &pkt_num_action,
int actionSize)
{
cout<<"* Assign actions ... ..."<<endl<<endl;
vector<string> actionBound;
actionBound.push_back("64.0.0.0");// = {"25.0.0.0","32.0.0.0"};
actionBound.push_back("128.0.0.0");
actionBound.push_back("192.0.0.0");// = {"25.0.0.0","32.0.0.0"};
actionBound.push_back("255.255.255.255");
pkt_num_action.assign(actionSize,0);
for(int i = 0; i < flow.size(); i++)
{
for (int ai = 0; ai < actionSize; ai ++)
{
if(parseIPV4string(flow[i].c_str())<
parseIPV4string(actionBound[ai].c_str()))
{
flowaction.push_back((ai));
pkt_num_action[ai] += (flow_cnt[i]);
break;
}
}
}
}
bool assignAction(vector<string> &key,vector<int> &keyaction,int &actionSize)
{
cout<<"* Assign action ... ..."<<endl;
vector<string> actionBound;
actionBound.push_back("64.0.0.0");
actionBound.push_back("128.0.0.0");
actionBound.push_back("192.0.0.0");
actionBound.push_back("255.255.255.255");
keyaction.clear();
for(int i = 0; i < key.size(); i++)
{
for (int ai = 0; ai < actionSize; ai ++)
{
if(parseIPV4string(key[i].c_str())<
parseIPV4string(actionBound[ai].c_str()))
{
keyaction.push_back((ai));
break;
}
}
}
return 1;
}
bool clusterAction(vector<string> &flow, vector<int> &flowaction,
vector<size_t> &mask, vector<size_t> &index)
{
vector<string> subIpBinary;
size_t ip_num = flow.size();
const char *charIP;
size_t ip, subIP;
for (size_t i = 0; i < ip_num; i++)
{
charIP = flow[i].c_str(); // get IP string
ip = parseIPV4string(charIP); //convert IP to int
int mi = 0;
subIP = ip & mask[mi]; // mask
subIpBinary.push_back(DecToBin(subIP)); // store /8
}
//--------------------------------------------------------
// unique items in subIPv8
cout<<"* Assign flows to subtrees according to prefix/8 and actions ... ..."
<<endl;
vector<string> subIPv8;
subIPv8 = subIpBinary; // copy prefix length /8 prefix and group
{
vector<string>::iterator it;
it = unique (subIPv8.begin(), subIPv8.end());
subIPv8.resize( distance(subIPv8.begin(),it));
}
// ------------------------------------------------------
// clustering according to prefix length /8
index.clear();
{
size_t pos = 0;
vector<string>::iterator posIt = subIpBinary.begin();
for(size_t i = 1; i < subIPv8.size(); i++)
{
pos = find(posIt,subIpBinary.end(),subIPv8[i])-subIpBinary.begin();
index.push_back(pos);
posIt = subIpBinary.begin();
advance (posIt,pos);
}
index.push_back(subIpBinary.end()-subIpBinary.begin());
}
cout<<"* /8 index size: "<<index.size()<<endl;
//---------------------------------------------------------------
// Cluster according to actions
vector<int> flowactionunique;
flowactionunique = flowaction;
{
vector<int>::iterator it;
it = unique (flowactionunique.begin(), flowactionunique.end());
flowactionunique.resize( distance(flowactionunique.begin(),it));
}
cout<<"* Unique flowaction size: "<<flowactionunique.size()<<endl;
vector<size_t> index1;
for(size_t i = 1; i < flowactionunique.size(); i++)
{
size_t It = find(flowaction.begin(),flowaction.end(),flowactionunique[i])
-flowaction.begin();//find(~L(:,h1),1); // the first 0 in bucket h1
index1.push_back(It);
}
// ----------------------------------------------------------------
// merge index and index1
vector<size_t> index2;
merge(index.begin(), index.end(), index1.begin(), index1.end(),
back_inserter(index2));
{
vector<size_t>::iterator it;
it = unique (index2.begin(), index2.end());
index2.resize( distance(index2.begin(),it));
}
index.clear();
index = index2;
cout<<"* Index size: "<<index.size()<<endl;
// Release space
vector<size_t>().swap(index2);
vector<string>().swap(subIPv8);
vector<int>().swap(flowactionunique);
vector<string>().swap(subIpBinary);
return 1;
}
size_t aggregation(vector<string> &keyIns,vector<int> &keyPrefixIns,
vector<int> &keyActionIns, vector<size_t> &maskes,
int actionSize, float &storage, bool isInit,
int &fingerprintOld,vector<int> &uniqueAggKeyprefixes, strings& overKeys, size_ts& overKeyNos,
floats& haoOvers, float target)
{
// ---------------------------
vector<size_t> indexes;
clusterAction(keyIns, keyActionIns, maskes,indexes);
// -----------------------------
// find min value and max value for each subtrie
vector<size_t> IPUpperBound;
vector<size_t> IPLowerBound;
IPLowerBound.push_back(parseIPV4string("0.0.0.0"));
IPUpperBound.push_back(parseIPV4string("0.255.255.255"));
for(uint16_t i = 0; i < indexes.size()-1; i ++)
{
size_t lowerBound = parseIPV4string(keyIns[indexes[i]].c_str()) & maskes[0];
size_t upperBound = lowerBound + (1<<24)-1;
IPLowerBound.push_back(lowerBound);
IPUpperBound.push_back(upperBound);
}
// ----------------------------------------------------------
// Define Tries
uint16_t trieNum = indexes.size(); // trie number
Trie *bTrie; // define trees
bTrie = new Trie[trieNum]; // define trees
// ---------------------------------------------------------
/* Insert keys to trie */
insertWordTrieSimple(bTrie, trieNum, indexes, keyIns, keyPrefixIns,
keyActionIns);
// -------------------------------------------------------------
// classify tries according to actions
ActionOrder actionOrder[actionSize];
cout<<"* ActionOder clustering ..."<<endl;
for(int ai = 0; ai < actionSize; ai++)
{
actionOrder[ai].aTrieOder.clear();
}
for(int ai = 0; ai < actionSize; ai++)
{
for (int ti = 0; ti < indexes.size(); ti++)
{
if (bTrie[ti].maction == ai)
{
actionOrder[ai].aTrieOder.push_back(ti);
}
}
}
// -------------------------------------------------------------------------
/* Aggregation */
/* Init variables */
size_t countKey = 0;
size_t countAggregateKey = 0;
size_t countBlackKey =0;
size_t countOriKey =0;
vector<string> keys;
vector<int> keyActions;
vector<string> blackKeys;
vector<int> blackkeyPrefixes;
vector<string> aggregateKeys;
int aggrPrefixlength = AGGR_PREFIX;
for(int ai = 0; ai < actionSize; ai++)
{
g_vcountkey[ai] = 0;
g_vcountblackkey[ai] = 0;
// ----------------------------------------------------------
/* aggregate Trie */
aggregateTrie(bTrie, ai, actionOrder, countKey,countAggregateKey,
countBlackKey,countOriKey, keys,
keyActions,blackKeys,blackkeyPrefixes, aggregateKeys, aggrPrefixlength, 1, isInit);
cout<<"Action: "<<ai<<" threshold: "<<g_vweightThld[ai]<<" ";
}
cout<<endl;
cout<<"* orikey "<<countOriKey<<" agg "<<
countAggregateKey<<" blkey "<<countBlackKey<<" countKey "<<countKey<<endl;
// ------------------------------------------------
// Find the actions to compress and decompress
ints compressActions;
ints decompressActions;
compressAct(haoOvers, target, actionSize, compressActions, decompressActions);
// --------------------------------------------
// Decompress aggregate keys
strings aggrIPs;
ints aggrPrefixes;
size_t aggrCount = 0;
cout<<"overbig Keys size: "<<overKeys.size()<<endl;
for(size_t i = 0; i< overKeys.size(); i++)
{
size_t ipInt = parseIPV4string(overKeys[i].c_str());
// lookup key in aggregate table, if inside, get the aggregate key
bool isAggregatekey = 0;
//if(cuckooAggrKeyTable.mm > 1)
{
for(int mi = 0; mi < uniqueAggKeyprefixes.size(); mi++)
{
size_t subIP = ipInt & maskes[uniqueAggKeyprefixes[mi]-8];
string flowstr = parsedec2IPV4(subIP);
int prefix = uniqueAggKeyprefixes[mi];
isAggregatekey = cuckooAggrKeyTable.LookUpKey(flowstr,prefix);
if (isAggregatekey)
{
aggrIPs.push_back(flowstr);
aggrPrefixes.push_back(prefix);
// Get the aggregate keys and decompress them
for(uint16_t ti = 0; ti < trieNum; ti++)
{
if(ipInt >= IPLowerBound[ti] && ipInt < IPUpperBound[ti])
{
// If the action is to compress, no need to decompress the aggregate keys
for(int ai = 0; ai <compressActions.size(); ai++)
{
if(ti/64 == compressActions[ai])
{
break;
}
}
bTrie[ti].searchAggrPrefix(DecToBin(subIP),prefix, aggrCount);
break;
}
}
// Find one match and break
break;
}
}
}
}
// ------------------------------------------------
// load aggregate previous keys and decompress them
ifstream aggrFile(AGGRFILENAME.c_str());
string aggrIPStr;
int aggrPrefix;
while(aggrFile>>aggrIPStr>>aggrPrefix && aggrIPs.size()<3500)
{
aggrIPs.push_back(aggrIPStr);
aggrPrefixes.push_back(aggrPrefix);
size_t aggrIPInt = parseIPV4string(aggrIPStr.c_str());
// get the aggregate key
for(uint16_t ti = 0; ti < trieNum; ti++)
{
if(aggrIPInt >= IPLowerBound[ti] && aggrIPInt < IPUpperBound[ti])
{
/*for(int ai = 0; ai <compressActions.size(); ai++)
{
if(ti/64 == compressActions[ai])
{
break;
}
}*/
bTrie[ti].searchAggrPrefix(DecToBin(aggrIPInt),aggrPrefix, aggrCount);
break;
}
}
}
aggrFile.clear();
aggrFile.close();
vector<size_t>().swap(IPLowerBound);
vector<size_t>().swap(IPUpperBound);
//---------------------------------------
// write to aggr file;
ofstream aggrFileOut("aggrFile");
for(size_t i = 0; i < aggrIPs.size(); i++)
{
aggrFileOut<<aggrIPs[i]<<" "<<aggrPrefixes[i]<<endl;
}
aggrFileOut.clear();
aggrFileOut.close();
vector<string>().swap(aggrIPs);
vector<int>().swap(aggrPrefixes);
// -------------------------------------------
// If decompress alters fingerprint length, compress other keys
countKey += aggrCount; // key number after decompress
cout<<"Increase aggrcount: "<<aggrCount<<endl;
uint16_t cuckooBlackSize = CUCKOO_BLACK_SIZE;
float loadFactor = 0.90;
int fingerprint = (storage*1024.0f*8.0f-(cuckooBlackSize)*39/0.9-FLOW_EST_SIZE*17/0.9)*loadFactor/(countKey) -3;
int iteartion = 0;
int maxIteration = 12;
ints prefixlength;
prefixlength.assign(actionSize,20);
while(fingerprint!=fingerprintOld && iteartion<maxIteration)
{
iteartion++;
// -----------------------------------------
// aggregate all other keys
/* Aggregation */
/* Init variables */
countKey = 0;
countAggregateKey = 0;
countBlackKey =0;
countOriKey =0;
keys.clear();
keyActions.clear();
blackKeys.clear();
blackkeyPrefixes.clear();
aggregateKeys.clear();
for(int ai = 0; ai < actionSize; ai++)
{
g_vcountkey[ai] = 0;
g_vcountblackkey[ai] = 0;
if(fingerprint>fingerprintOld) // decompress, increase threshold
{
g_vweightThld[ai] += 0.2*g_vweightThld[ai]*(fingerprint - fingerprintOld);
prefixlength[ai] ++;
}
else if( g_vweightThld[ai] > 0)
{
g_vweightThld[ai] += 0.1*g_vweightThld[ai]*(fingerprint - fingerprintOld) ;
prefixlength[ai] --;
}
//g_vweightThld[ai] = 0;
// ----------------------------------------------------------
/* aggregate Trie */
aggregateTrie(bTrie, ai, actionOrder, countKey,countAggregateKey,
countBlackKey,countOriKey, keys,keyActions,blackKeys,blackkeyPrefixes, aggregateKeys, aggrPrefixlength, 1, isInit);
cout<<"* threshold: "<<g_vweightThld[ai]<<" prefixlength: "<<prefixlength[ai]<<endl;
}
cout<<" coutkey "<<"agg "<<"blkey "<<"orikey "<<countOriKey<<" "<<
countAggregateKey<<" "<<countBlackKey<<" "<<countKey<<endl;
fingerprint = (storage*1024.0f*8.0f*loadFactor-(cuckooBlackSize)*39/0.9-FLOW_EST_SIZE*17/0.9)/(countKey) -3;
cout<<"Fingerprint: "<<fingerprint<<endl;
}
// -----------------------------------------
// Get aggregate result
countKey = 0;
countAggregateKey = 0;
countBlackKey =0;
countOriKey =0;
keys.clear();
keyActions.clear();
blackKeys.clear();
blackkeyPrefixes.clear();
aggregateKeys.clear();
vector<char> word;
size_t recoverCount = 0;
for(uint16_t ti = 0; ti < trieNum; ti++)
{
bTrie[ti].nodeCount(bTrie[ti].root,countKey,countAggregateKey,countBlackKey,countOriKey);
bTrie[ti].printNode(bTrie[ti].root,word,keys,keyActions,blackKeys,blackkeyPrefixes,aggregateKeys);
bTrie[ti].recoverTrie(bTrie[ti].root8, recoverCount);
}
cout<<" coutkey "<<"agg "<<"blkey "<<"orikey "<<countOriKey<<" "<<
countAggregateKey<<" "<<countBlackKey<<" "<<countKey<<endl;
// ----------------------------------------------
// Compute Unique prefixes
vector<int> keyPrefixes;
for(size_t i = 0; i < keys.size(); i++)
{
size_t found = keys[i].find('/');
string prefixstr = keys[i].substr(found+1,
keys[i].size()-found);
keyPrefixes.push_back(str2num(prefixstr));
}
uniqueAggKeyprefixes.clear();
prefixNum(keyPrefixes,uniqueAggKeyprefixes);
// ----------------------------------
/* Insert to cuckoo filter */
// parameters for cuckoo filter
int slotNo = 4;
size_t keySize = keys.size();
size_t bucketNo = size_t(keySize/(loadFactor*slotNo))+1;
fingerprint = (storage*1024.0f*8.0f-(cuckooBlackSize)*39/0.9-FLOW_EST_SIZE*17/0.9)*loadFactor/(countKey) -3;
//fingerprintOld = fingerprint;
long maxNumKicks = 1000;
cout<<"* Fingerprint length: "<<fingerprint<<endl;
//init cuckoo filter
cuckooFilter.ClearTable();
cuckooFilter.cuckooFilterInit(bucketNo,fingerprint,slotNo,maxNumKicks);
// add key to cuckoo filter
addCuckooFilter(keys, keyActions);
// --------------------------------------
// Add aggregate keys to cuckooTable
size_t aggregateKeySize = aggregateKeys.size();
bucketNo = long(aggregateKeySize/(loadFactor*slotNo));
cuckooAggrKeyTable.CuckooTableInit(bucketNo,fingerprint,slotNo,maxNumKicks);
for(size_t i = 0; i < aggregateKeySize; i++)
{
size_t found = aggregateKeys[i].find('/');
string str = aggregateKeys[i].substr(0,found);
string prefixstr = aggregateKeys[i].substr(found+1,
aggregateKeys[i].size()-found);
cuckooAggrKeyTable.AddKey(str,str2num(prefixstr));
}
vector<string>().swap(keys);
vector<int>().swap(keyActions);
vector<string>().swap(blackKeys);
vector<int>().swap(blackkeyPrefixes);
vector<string>().swap(aggregateKeys);
// --------------------------------------
// Add blackkey to cuckooTable
/*cout<<"* Add blackkey to cuckooTable!"<<endl;
size_t blackKeySize = blackKeys.size();
size_t bucketSize = int((blackKeySize)/(loadFactor*slotNo))+1;
int MaxKickoutNum = 1000;
cuckooBlackKeyTable.ClearTable();
cuckooBlackKeyTable.CuckooTableInit(bucketSize,fingerprint,slotNo,
MaxKickoutNum);
for(size_t i = 0; i < blackKeySize; i++)
{
cuckooBlackKeyTable.AddKeyPrefix(blackKeys[i],blackkeyPrefixes[i], 4);
}
blackKeys.clear();
blackkeyPrefixes.clear();*/
// ---------------------------------------
for(int i = 0; i < trieNum; i++)
{
bTrie[i].deleteChild(bTrie[i].root);
}
if(!bTrie)
delete[] bTrie;
cout<<"* Aggregation Return! "<<endl;
return countKey;
}
/*
* Opens the network config file for parsing.
*
* The network config file is small enough fo us to be able to use aJSON (as
* opposed to creating our own parser, as with the door and pin configs.)
*/
bool Network::parseNetworkConfiguration(Stream& stream) {
aJsonObject *root = aJson.parse(&aJsonStream(&stream));
aJsonObject* dhcpEnabled = aJson.getObjectItem(root, "DHCPEnabled");
aJsonObject* macAddress = aJson.getObjectItem(root, "MAC");
aJsonObject* ipObject = aJson.getObjectItem(root, "IP");
aJsonObject* gatewayObject = aJson.getObjectItem(root, "Gateway");
aJsonObject* subnetObject = aJson.getObjectItem(root, "Subnet");
aJsonObject* dnsObject = aJson.getObjectItem(root, "DNS");
aJsonObject* httpPortObject = aJson.getObjectItem(root, "HTTPPort");
aJsonObject* websocketPortObject = aJson.getObjectItem(root, "WebsocketPort");
if (!dhcpEnabled) {
cout << F("DHCPEnabled key not found in network config file.");
return false;
}
else if (!macAddress) {
cout << F("MAC key not found in network config file.");
return false;
}
else if (!ipObject) {
cout << F("IP key not found in network config file.");
return false;
}
else if (!gatewayObject) {
cout << F("Gateway key not found in network config file.");
return false;
}
else if (!subnetObject) {
cout << F("Subnet key not found in network config file.");
return false;
}
else if (!dnsObject) {
cout << F("DNS key not found in network config file.");
return false;
}
else if (!httpPortObject) {
cout << F("HTTPPort key not found in network config file.");
return false;
}
else if (!websocketPortObject) {
cout << F("WebsocketPort key not found in network config file.");
return false;
}
// DHCP related
use_dhcp = dhcpEnabled->valuebool;
// Parse the MAC address
char macHexStr[2];
int macAddressPos = -1;
for (int i=0; i<6; i++) {
macHexStr[0] = macAddress->valuestring[++macAddressPos];
macHexStr[1] = macAddress->valuestring[++macAddressPos];
macAddressPos++; //Skip the ":" separator.
mac[i] = (uint8_t)strtol(macHexStr, NULL, 16);
}
// IP related
uint8_t ipbytes[4];
parseIPV4string(ipObject->valuestring, ipbytes);
ip = IPAddress(ipbytes);
parseIPV4string(gatewayObject->valuestring, ipbytes);
gateway = IPAddress(ipbytes);
parseIPV4string(subnetObject->valuestring, ipbytes);
subnet = IPAddress(ipbytes);
parseIPV4string(dnsObject->valuestring, ipbytes);
dns = IPAddress(ipbytes);
// Ports
httpPort = (int)httpPortObject->valueint;
websocketPort = (int)websocketPortObject->valueint;
// Free allocated memory.
aJson.deleteItem(root);
// We've successfully parsed the network file.
return true;
}
