void assignAction(vector<string> &flow,vector<size_t> &flow_cnt, vector<int> &flowaction,vector<size_t> &pkt_num_action, int actionSize) { cout<<"* Assign actions ... ..."<<endl<<endl; vector<string> actionBound; actionBound.push_back("64.0.0.0");// = {"25.0.0.0","32.0.0.0"}; actionBound.push_back("128.0.0.0"); actionBound.push_back("192.0.0.0");// = {"25.0.0.0","32.0.0.0"}; actionBound.push_back("255.255.255.255"); pkt_num_action.assign(actionSize,0); for(int i = 0; i < flow.size(); i++) { for (int ai = 0; ai < actionSize; ai ++) { if(parseIPV4string(flow[i].c_str())< parseIPV4string(actionBound[ai].c_str())) { flowaction.push_back((ai)); pkt_num_action[ai] += (flow_cnt[i]); break; } } } }
bool assignAction(vector<string> &key,vector<int> &keyaction,int &actionSize) { cout<<"* Assign action ... ..."<<endl; vector<string> actionBound; actionBound.push_back("64.0.0.0"); actionBound.push_back("128.0.0.0"); actionBound.push_back("192.0.0.0"); actionBound.push_back("255.255.255.255"); keyaction.clear(); for(int i = 0; i < key.size(); i++) { for (int ai = 0; ai < actionSize; ai ++) { if(parseIPV4string(key[i].c_str())< parseIPV4string(actionBound[ai].c_str())) { keyaction.push_back((ai)); break; } } } return 1; }
bool clusterAction(vector<string> &flow, vector<int> &flowaction, vector<size_t> &mask, vector<size_t> &index) { vector<string> subIpBinary; size_t ip_num = flow.size(); const char *charIP; size_t ip, subIP; for (size_t i = 0; i < ip_num; i++) { charIP = flow[i].c_str(); // get IP string ip = parseIPV4string(charIP); //convert IP to int int mi = 0; subIP = ip & mask[mi]; // mask subIpBinary.push_back(DecToBin(subIP)); // store /8 } //-------------------------------------------------------- // unique items in subIPv8 cout<<"* Assign flows to subtrees according to prefix/8 and actions ... ..." <<endl; vector<string> subIPv8; subIPv8 = subIpBinary; // copy prefix length /8 prefix and group { vector<string>::iterator it; it = unique (subIPv8.begin(), subIPv8.end()); subIPv8.resize( distance(subIPv8.begin(),it)); } // ------------------------------------------------------ // clustering according to prefix length /8 index.clear(); { size_t pos = 0; vector<string>::iterator posIt = subIpBinary.begin(); for(size_t i = 1; i < subIPv8.size(); i++) { pos = find(posIt,subIpBinary.end(),subIPv8[i])-subIpBinary.begin(); index.push_back(pos); posIt = subIpBinary.begin(); advance (posIt,pos); } index.push_back(subIpBinary.end()-subIpBinary.begin()); } cout<<"* /8 index size: "<<index.size()<<endl; //--------------------------------------------------------------- // Cluster according to actions vector<int> flowactionunique; flowactionunique = flowaction; { vector<int>::iterator it; it = unique (flowactionunique.begin(), flowactionunique.end()); flowactionunique.resize( distance(flowactionunique.begin(),it)); } cout<<"* Unique flowaction size: "<<flowactionunique.size()<<endl; vector<size_t> index1; for(size_t i = 1; i < flowactionunique.size(); i++) { size_t It = find(flowaction.begin(),flowaction.end(),flowactionunique[i]) -flowaction.begin();//find(~L(:,h1),1); // the first 0 in bucket h1 index1.push_back(It); } // ---------------------------------------------------------------- // merge index and index1 vector<size_t> index2; merge(index.begin(), index.end(), index1.begin(), index1.end(), back_inserter(index2)); { vector<size_t>::iterator it; it = unique (index2.begin(), index2.end()); index2.resize( distance(index2.begin(),it)); } index.clear(); index = index2; cout<<"* Index size: "<<index.size()<<endl; // Release space vector<size_t>().swap(index2); vector<string>().swap(subIPv8); vector<int>().swap(flowactionunique); vector<string>().swap(subIpBinary); return 1; }
size_t aggregation(vector<string> &keyIns,vector<int> &keyPrefixIns, vector<int> &keyActionIns, vector<size_t> &maskes, int actionSize, float &storage, bool isInit, int &fingerprintOld,vector<int> &uniqueAggKeyprefixes, strings& overKeys, size_ts& overKeyNos, floats& haoOvers, float target) { // --------------------------- vector<size_t> indexes; clusterAction(keyIns, keyActionIns, maskes,indexes); // ----------------------------- // find min value and max value for each subtrie vector<size_t> IPUpperBound; vector<size_t> IPLowerBound; IPLowerBound.push_back(parseIPV4string("0.0.0.0")); IPUpperBound.push_back(parseIPV4string("0.255.255.255")); for(uint16_t i = 0; i < indexes.size()-1; i ++) { size_t lowerBound = parseIPV4string(keyIns[indexes[i]].c_str()) & maskes[0]; size_t upperBound = lowerBound + (1<<24)-1; IPLowerBound.push_back(lowerBound); IPUpperBound.push_back(upperBound); } // ---------------------------------------------------------- // Define Tries uint16_t trieNum = indexes.size(); // trie number Trie *bTrie; // define trees bTrie = new Trie[trieNum]; // define trees // --------------------------------------------------------- /* Insert keys to trie */ insertWordTrieSimple(bTrie, trieNum, indexes, keyIns, keyPrefixIns, keyActionIns); // ------------------------------------------------------------- // classify tries according to actions ActionOrder actionOrder[actionSize]; cout<<"* ActionOder clustering ..."<<endl; for(int ai = 0; ai < actionSize; ai++) { actionOrder[ai].aTrieOder.clear(); } for(int ai = 0; ai < actionSize; ai++) { for (int ti = 0; ti < indexes.size(); ti++) { if (bTrie[ti].maction == ai) { actionOrder[ai].aTrieOder.push_back(ti); } } } // ------------------------------------------------------------------------- /* Aggregation */ /* Init variables */ size_t countKey = 0; size_t countAggregateKey = 0; size_t countBlackKey =0; size_t countOriKey =0; vector<string> keys; vector<int> keyActions; vector<string> blackKeys; vector<int> blackkeyPrefixes; vector<string> aggregateKeys; int aggrPrefixlength = AGGR_PREFIX; for(int ai = 0; ai < actionSize; ai++) { g_vcountkey[ai] = 0; g_vcountblackkey[ai] = 0; // ---------------------------------------------------------- /* aggregate Trie */ aggregateTrie(bTrie, ai, actionOrder, countKey,countAggregateKey, countBlackKey,countOriKey, keys, keyActions,blackKeys,blackkeyPrefixes, aggregateKeys, aggrPrefixlength, 1, isInit); cout<<"Action: "<<ai<<" threshold: "<<g_vweightThld[ai]<<" "; } cout<<endl; cout<<"* orikey "<<countOriKey<<" agg "<< countAggregateKey<<" blkey "<<countBlackKey<<" countKey "<<countKey<<endl; // ------------------------------------------------ // Find the actions to compress and decompress ints compressActions; ints decompressActions; compressAct(haoOvers, target, actionSize, compressActions, decompressActions); // -------------------------------------------- // Decompress aggregate keys strings aggrIPs; ints aggrPrefixes; size_t aggrCount = 0; cout<<"overbig Keys size: "<<overKeys.size()<<endl; for(size_t i = 0; i< overKeys.size(); i++) { size_t ipInt = parseIPV4string(overKeys[i].c_str()); // lookup key in aggregate table, if inside, get the aggregate key bool isAggregatekey = 0; //if(cuckooAggrKeyTable.mm > 1) { for(int mi = 0; mi < uniqueAggKeyprefixes.size(); mi++) { size_t subIP = ipInt & maskes[uniqueAggKeyprefixes[mi]-8]; string flowstr = parsedec2IPV4(subIP); int prefix = uniqueAggKeyprefixes[mi]; isAggregatekey = cuckooAggrKeyTable.LookUpKey(flowstr,prefix); if (isAggregatekey) { aggrIPs.push_back(flowstr); aggrPrefixes.push_back(prefix); // Get the aggregate keys and decompress them for(uint16_t ti = 0; ti < trieNum; ti++) { if(ipInt >= IPLowerBound[ti] && ipInt < IPUpperBound[ti]) { // If the action is to compress, no need to decompress the aggregate keys for(int ai = 0; ai <compressActions.size(); ai++) { if(ti/64 == compressActions[ai]) { break; } } bTrie[ti].searchAggrPrefix(DecToBin(subIP),prefix, aggrCount); break; } } // Find one match and break break; } } } } // ------------------------------------------------ // load aggregate previous keys and decompress them ifstream aggrFile(AGGRFILENAME.c_str()); string aggrIPStr; int aggrPrefix; while(aggrFile>>aggrIPStr>>aggrPrefix && aggrIPs.size()<3500) { aggrIPs.push_back(aggrIPStr); aggrPrefixes.push_back(aggrPrefix); size_t aggrIPInt = parseIPV4string(aggrIPStr.c_str()); // get the aggregate key for(uint16_t ti = 0; ti < trieNum; ti++) { if(aggrIPInt >= IPLowerBound[ti] && aggrIPInt < IPUpperBound[ti]) { /*for(int ai = 0; ai <compressActions.size(); ai++) { if(ti/64 == compressActions[ai]) { break; } }*/ bTrie[ti].searchAggrPrefix(DecToBin(aggrIPInt),aggrPrefix, aggrCount); break; } } } aggrFile.clear(); aggrFile.close(); vector<size_t>().swap(IPLowerBound); vector<size_t>().swap(IPUpperBound); //--------------------------------------- // write to aggr file; ofstream aggrFileOut("aggrFile"); for(size_t i = 0; i < aggrIPs.size(); i++) { aggrFileOut<<aggrIPs[i]<<" "<<aggrPrefixes[i]<<endl; } aggrFileOut.clear(); aggrFileOut.close(); vector<string>().swap(aggrIPs); vector<int>().swap(aggrPrefixes); // ------------------------------------------- // If decompress alters fingerprint length, compress other keys countKey += aggrCount; // key number after decompress cout<<"Increase aggrcount: "<<aggrCount<<endl; uint16_t cuckooBlackSize = CUCKOO_BLACK_SIZE; float loadFactor = 0.90; int fingerprint = (storage*1024.0f*8.0f-(cuckooBlackSize)*39/0.9-FLOW_EST_SIZE*17/0.9)*loadFactor/(countKey) -3; int iteartion = 0; int maxIteration = 12; ints prefixlength; prefixlength.assign(actionSize,20); while(fingerprint!=fingerprintOld && iteartion<maxIteration) { iteartion++; // ----------------------------------------- // aggregate all other keys /* Aggregation */ /* Init variables */ countKey = 0; countAggregateKey = 0; countBlackKey =0; countOriKey =0; keys.clear(); keyActions.clear(); blackKeys.clear(); blackkeyPrefixes.clear(); aggregateKeys.clear(); for(int ai = 0; ai < actionSize; ai++) { g_vcountkey[ai] = 0; g_vcountblackkey[ai] = 0; if(fingerprint>fingerprintOld) // decompress, increase threshold { g_vweightThld[ai] += 0.2*g_vweightThld[ai]*(fingerprint - fingerprintOld); prefixlength[ai] ++; } else if( g_vweightThld[ai] > 0) { g_vweightThld[ai] += 0.1*g_vweightThld[ai]*(fingerprint - fingerprintOld) ; prefixlength[ai] --; } //g_vweightThld[ai] = 0; // ---------------------------------------------------------- /* aggregate Trie */ aggregateTrie(bTrie, ai, actionOrder, countKey,countAggregateKey, countBlackKey,countOriKey, keys,keyActions,blackKeys,blackkeyPrefixes, aggregateKeys, aggrPrefixlength, 1, isInit); cout<<"* threshold: "<<g_vweightThld[ai]<<" prefixlength: "<<prefixlength[ai]<<endl; } cout<<" coutkey "<<"agg "<<"blkey "<<"orikey "<<countOriKey<<" "<< countAggregateKey<<" "<<countBlackKey<<" "<<countKey<<endl; fingerprint = (storage*1024.0f*8.0f*loadFactor-(cuckooBlackSize)*39/0.9-FLOW_EST_SIZE*17/0.9)/(countKey) -3; cout<<"Fingerprint: "<<fingerprint<<endl; } // ----------------------------------------- // Get aggregate result countKey = 0; countAggregateKey = 0; countBlackKey =0; countOriKey =0; keys.clear(); keyActions.clear(); blackKeys.clear(); blackkeyPrefixes.clear(); aggregateKeys.clear(); vector<char> word; size_t recoverCount = 0; for(uint16_t ti = 0; ti < trieNum; ti++) { bTrie[ti].nodeCount(bTrie[ti].root,countKey,countAggregateKey,countBlackKey,countOriKey); bTrie[ti].printNode(bTrie[ti].root,word,keys,keyActions,blackKeys,blackkeyPrefixes,aggregateKeys); bTrie[ti].recoverTrie(bTrie[ti].root8, recoverCount); } cout<<" coutkey "<<"agg "<<"blkey "<<"orikey "<<countOriKey<<" "<< countAggregateKey<<" "<<countBlackKey<<" "<<countKey<<endl; // ---------------------------------------------- // Compute Unique prefixes vector<int> keyPrefixes; for(size_t i = 0; i < keys.size(); i++) { size_t found = keys[i].find('/'); string prefixstr = keys[i].substr(found+1, keys[i].size()-found); keyPrefixes.push_back(str2num(prefixstr)); } uniqueAggKeyprefixes.clear(); prefixNum(keyPrefixes,uniqueAggKeyprefixes); // ---------------------------------- /* Insert to cuckoo filter */ // parameters for cuckoo filter int slotNo = 4; size_t keySize = keys.size(); size_t bucketNo = size_t(keySize/(loadFactor*slotNo))+1; fingerprint = (storage*1024.0f*8.0f-(cuckooBlackSize)*39/0.9-FLOW_EST_SIZE*17/0.9)*loadFactor/(countKey) -3; //fingerprintOld = fingerprint; long maxNumKicks = 1000; cout<<"* Fingerprint length: "<<fingerprint<<endl; //init cuckoo filter cuckooFilter.ClearTable(); cuckooFilter.cuckooFilterInit(bucketNo,fingerprint,slotNo,maxNumKicks); // add key to cuckoo filter addCuckooFilter(keys, keyActions); // -------------------------------------- // Add aggregate keys to cuckooTable size_t aggregateKeySize = aggregateKeys.size(); bucketNo = long(aggregateKeySize/(loadFactor*slotNo)); cuckooAggrKeyTable.CuckooTableInit(bucketNo,fingerprint,slotNo,maxNumKicks); for(size_t i = 0; i < aggregateKeySize; i++) { size_t found = aggregateKeys[i].find('/'); string str = aggregateKeys[i].substr(0,found); string prefixstr = aggregateKeys[i].substr(found+1, aggregateKeys[i].size()-found); cuckooAggrKeyTable.AddKey(str,str2num(prefixstr)); } vector<string>().swap(keys); vector<int>().swap(keyActions); vector<string>().swap(blackKeys); vector<int>().swap(blackkeyPrefixes); vector<string>().swap(aggregateKeys); // -------------------------------------- // Add blackkey to cuckooTable /*cout<<"* Add blackkey to cuckooTable!"<<endl; size_t blackKeySize = blackKeys.size(); size_t bucketSize = int((blackKeySize)/(loadFactor*slotNo))+1; int MaxKickoutNum = 1000; cuckooBlackKeyTable.ClearTable(); cuckooBlackKeyTable.CuckooTableInit(bucketSize,fingerprint,slotNo, MaxKickoutNum); for(size_t i = 0; i < blackKeySize; i++) { cuckooBlackKeyTable.AddKeyPrefix(blackKeys[i],blackkeyPrefixes[i], 4); } blackKeys.clear(); blackkeyPrefixes.clear();*/ // --------------------------------------- for(int i = 0; i < trieNum; i++) { bTrie[i].deleteChild(bTrie[i].root); } if(!bTrie) delete[] bTrie; cout<<"* Aggregation Return! "<<endl; return countKey; }
/* * Opens the network config file for parsing. * * The network config file is small enough fo us to be able to use aJSON (as * opposed to creating our own parser, as with the door and pin configs.) */ bool Network::parseNetworkConfiguration(Stream& stream) { aJsonObject *root = aJson.parse(&aJsonStream(&stream)); aJsonObject* dhcpEnabled = aJson.getObjectItem(root, "DHCPEnabled"); aJsonObject* macAddress = aJson.getObjectItem(root, "MAC"); aJsonObject* ipObject = aJson.getObjectItem(root, "IP"); aJsonObject* gatewayObject = aJson.getObjectItem(root, "Gateway"); aJsonObject* subnetObject = aJson.getObjectItem(root, "Subnet"); aJsonObject* dnsObject = aJson.getObjectItem(root, "DNS"); aJsonObject* httpPortObject = aJson.getObjectItem(root, "HTTPPort"); aJsonObject* websocketPortObject = aJson.getObjectItem(root, "WebsocketPort"); if (!dhcpEnabled) { cout << F("DHCPEnabled key not found in network config file."); return false; } else if (!macAddress) { cout << F("MAC key not found in network config file."); return false; } else if (!ipObject) { cout << F("IP key not found in network config file."); return false; } else if (!gatewayObject) { cout << F("Gateway key not found in network config file."); return false; } else if (!subnetObject) { cout << F("Subnet key not found in network config file."); return false; } else if (!dnsObject) { cout << F("DNS key not found in network config file."); return false; } else if (!httpPortObject) { cout << F("HTTPPort key not found in network config file."); return false; } else if (!websocketPortObject) { cout << F("WebsocketPort key not found in network config file."); return false; } // DHCP related use_dhcp = dhcpEnabled->valuebool; // Parse the MAC address char macHexStr[2]; int macAddressPos = -1; for (int i=0; i<6; i++) { macHexStr[0] = macAddress->valuestring[++macAddressPos]; macHexStr[1] = macAddress->valuestring[++macAddressPos]; macAddressPos++; //Skip the ":" separator. mac[i] = (uint8_t)strtol(macHexStr, NULL, 16); } // IP related uint8_t ipbytes[4]; parseIPV4string(ipObject->valuestring, ipbytes); ip = IPAddress(ipbytes); parseIPV4string(gatewayObject->valuestring, ipbytes); gateway = IPAddress(ipbytes); parseIPV4string(subnetObject->valuestring, ipbytes); subnet = IPAddress(ipbytes); parseIPV4string(dnsObject->valuestring, ipbytes); dns = IPAddress(ipbytes); // Ports httpPort = (int)httpPortObject->valueint; websocketPort = (int)websocketPortObject->valueint; // Free allocated memory. aJson.deleteItem(root); // We've successfully parsed the network file. return true; }