void CMaquetteImpl::send(CPacketToSend& packet, bool wait /*= false*/)
{
const data_t data = packet.encode();
LOG4CPLUS_DEBUG(logger, "Sending " << CPacket::Type::toString(data[0]) << ": " << data.size() << " byte\n" << CUtils::dump(data).c_str());
// Send message to the server
// See https://casablanca.codeplex.com/wikipage?title=Web%20Socket&referringTitle=Documentation
auto buf = std::make_shared<producer_consumer_buffer<byte>>();
auto task = buf->putn_nocopy(data.data(), data.size())
.then([=](size_t size) {
websocket_outgoing_message msg;
msg.set_binary_message(buf->create_istream(), size);
pplx::task<void>task = m_client->send(msg);
task.then([](pplx::task<void> t) {
try {
t.get();
} catch(const websocket_exception& ex) {
LOG4CPLUS_ERROR(logger, "websocket_callback_client::send() failed: " << ex.what());
}
});
return task;
});
if(wait) task.wait();
if(packet.type() != CPacket::Type::PINGREQ) {
m_keepAliveTimer.restart();
}
}
void CTCPProfile::Compress(const data_t &data, data_t &output) {
size_t outputInSize = output.size();
const iphdr* ip = reinterpret_cast<const iphdr*>(&data[0]);
const tcphdr* tcp = reinterpret_cast<const tcphdr*>(ip+ip->ihl*4);
UpdateIpIdOffset(ip);
if (IR_State == state)
{
CreateIR(ip, tcp, output);
}
else
{
CreateCO(ip, tcp, output);
}
UpdateIpInformation(ip);
AdvanceState(false, false);
increaseMsn();
// Append payload
// TODO, handle TCP options
output.insert(output.end(), data.begin() + sizeof(iphdr) + sizeof(tcphdr), data.end());
++numberOfPacketsSent;
dataSizeCompressed += output.size() - outputInSize;
dataSizeUncompressed += data.size();
}
void random_forest_p(const data_t& train, const vec_data_t& test, preds_t& train_preds, vec_preds_t& test_preds, args_t& args) {
int trees = args.trees;
int numthreads=args.processors, i, x, t;
int fk = args.kfeatures;
if (numthreads > trees)
numthreads = trees;
trees = (trees / numthreads) * numthreads;
int trees_per_thread = trees / numthreads;
thread** threads = new thread*[numthreads];
vector< vec_preds_t > seg;
vector< preds_t > train_seg;
for (i=0; i < numthreads; i++) {
vec_preds_t p;
init_pred_vec(test, p);
seg.push_back(p);
preds_t tp;
init_pred(train, tp);
train_seg.push_back(tp);
}
for (i=0;i<numthreads;i++)
threads[i] = new thread(bind(multiple_forest, trees_per_thread, cref(train), cref(test), ref(train_seg[i]), ref(seg[i]), ref(args)));
for (i=0;i<numthreads;i++){
threads[i]->join();
delete threads[i];
}
fprintf(stderr, "done threading\n");
delete[] threads;
// congregate results of the threads
for (i = 0; i < numthreads; i++)
for (t = 0; t < test.size(); t++)
for (int j = 0; j < test[t].size(); j++)
test_preds[t][j] += seg[i][t][j];
for (i = 0; i < numthreads; i++)
for (int j = 0; j < train.size(); j++)
train_preds[j] += train_seg[i][j];
// average results of the trees
for (i = 0; i < test.size(); i++)
for (int j = 0; j < test[i].size(); j++)
test_preds[i][j] /= trees;
for (int j = 0; j < train.size(); j++)
train_preds[j] /= trees;
// write results to file
for (i = 0; i < test.size(); i++) {
tuple::write_to_file(test_preds[i], test[i], args.test_outs[i]);
}
}
secure_session_t(const data_t& id, const data_t& priv_key, secure_session_callback_interface_t* callbacks):
_session(NULL),
_res(0){
_callback.get_public_key_for_id=themispp::get_public_key_for_id_callback;
_callback.send_data=themispp::send_callback;
_callback.receive_data=themispp::receive_callback;
_callback.state_changed=NULL;
_callback.user_data=callbacks;
_session=secure_session_create(&id[0], id.size(), &priv_key[0], priv_key.size(), &_callback);
if(!_session)
throw themispp::exception_t("Secure Session failde creating");
}
static bool range(data_t const& args, data_t& into) {
// unpack arguments
if (args.size() != 2)
return false;
auto f = args[0], l = args[1];
if (l>f)
into.reserve(1 + l - f + into.size());
for(; f<=l; ++f)
into.push_back(f); // to optimize
return true;
}
double variation(data_t &dataset, size_t split){
data_t::iterator i;
double mean = 0;
double variation = 0;
for (i = dataset.begin(); i != dataset.end(); i++){
mean += i->at(split);
}
mean /= dataset.size();
for (i = dataset.begin(); i != dataset.end(); i++){
variation += pow(i->at(split) - mean, 2.0);
}
variation /= dataset.size();
return variation;
}
static bool fibonacci(data_t const& args, data_t& into) {
// unpack arguments
if (args.size() != 2)
return false;
auto f = args[0], l = args[1];
// iterate
uint64_t gen0 = 0, gen1 = 1, next = gen0 + gen1;
for(auto i = 0u; i <= l; ++i)
{
std::cout << "fibonacci(" << f << ", " << l << ") at i == " << i << ", next: " << next << "\n";
switch(i) {
case 0:
if (i>=f) into.push_back(gen0);
break;
case 1:
if (i>=f) into.push_back(gen1);
break;
default:
{
next = gen0 + gen1;
if (i>=f) into.push_back(next);
gen0 = gen1;
gen1 = next;
break;
}
}
}
std::cout << "fibonacci(" << f << ", " << l << ") result: " << karma::format_delimited(karma::auto_, ';', into) << "\n";
// done
return true;
}
string TH_0x20::makeStringFromData(const data_t& data, const options_t& options)
{
(void)options;
if (data.size() != dataByteCount)
{
throw invalid_argument("Empty data array. Needs to contain " + to_string(dataByteCount) + " bytes");
}
string coeff = TH_0x00::makeStringFromData(data);
string str = ((coeff != "0") ? (coeff + "*Ï€") : "");
// Improve final display
str = regex_replace(str, regex("\\+1\\*"), "+"); str = regex_replace(str, regex("\\(1\\*"), "(");
str = regex_replace(str, regex("-1\\*"), "-"); str = regex_replace(str, regex("\\(-1\\*"), "(-");
str = regex_replace(str, regex("\\+-"), "-");
// Shouldn't happen - I don't believe the calc generate such files.
if (str == "")
{
str = "0";
}
return str;
}
file_offset_t bin_index_t::file_node::append_data(const data_t& res)
{
open_data_file();
file_offset_t ret=fd->append(res);
data_size=ret+res.size();
return ret;
}
string TH_0x01::makeStringFromData(const data_t& data, const options_t& options)
{
(void)options;
string str;
size_t byteCount = data.size();
size_t numCount = (size_t) ((data[0] & 0xFF) + ((data[1] << 8) & 0xFF00));
if (byteCount < 2+TH_0x00::dataByteCount || ((byteCount - 2) % TH_0x00::dataByteCount != 0)
|| (numCount != (size_t)((byteCount - 2) / TH_0x00::dataByteCount)) || numCount > 999)
{
std::cerr << "Invalid data array. Needs to contain 2+" + to_string(TH_0x00::dataByteCount) + "*n bytes" << endl;
return "";
}
str = "{";
for (size_t i = 2, num = 0; i < byteCount; i += TH_0x00::dataByteCount, num++)
{
str += TH_0x00::makeStringFromData(data_t(data.begin()+i, data.begin()+i+TH_0x00::dataByteCount));
if (num < numCount - 1) // not last num
{
str += ',';
}
}
str += "}";
return str;
}
std::string STH_ExactFractionPi::makeStringFromData(const data_t& data, const options_t& options)
{
if (data.size() != 9)
{
throw std::invalid_argument("Invalid data array. Needs to contain 9 bytes");
}
return dec2frac(stod(STH_FP::makeStringFromData(data, options)), "Ï€");
}
void bin2points(const data_t& bin,points_t& pts)
{
if((bin.size()%2)!=0)
throw std::runtime_error("bin2points(): (bin.size()%2)!=0");
pts.resize(bin.size()/2);
for(unsigned i=0;i<pts.size();i++)
{
point& p=pts[i];
char x=*reinterpret_cast<const unsigned char*>(&bin[i*2]);
char y=*reinterpret_cast<const unsigned char*>(&bin[i*2+1]);
p.x=x;
p.y=y;
}
}
void bin2points(const data_t& bin,steps_t& pts)
{
if((bin.size()%3)!=0)
throw std::runtime_error("bin2points(): (bin.size()%3)!=0");
pts.resize(bin.size()/3);
for(unsigned i=0;i<pts.size();i++)
{
step_t& p=pts[i];
char x=*reinterpret_cast<const unsigned char*>(&bin[i*3]);
char y=*reinterpret_cast<const unsigned char*>(&bin[i*3+1]);
p.x=x;
p.y=y;
p.step=(Step)bin[i*3+2];
}
}
void construct(point &root, data_t dataset){
if (dataset.size() == 0)
return;
root = new point_t();
vector< double > buff;
for (size_t i = 0; i < dataset.at(0).size(); i++){
buff.push_back(variation(dataset, i));
}
size_t split = 0;
double min;
for (size_t i = 0; i < buff.size(); i++){
if (i == 0){
min = buff.at(i);
}
else{
if (min < buff.at(i)){
min = buff.at(i);
split = i;
}
}
}
sort(dataset.begin(), dataset.end(), [split](vec_t a, vec_t b){ return a.at(split) < b.at(split); });
size_t middle = dataset.size() / 2;
root->split = split;
for (size_t i = 0; i < dataset.at(middle).size(); i++){
root->node.push_back(dataset.at(middle).at(i));
}
root->dim = root->node.size();
data_t lDataset, rDataset;
for (size_t i = 0; i < middle; i++){
lDataset.push_back(dataset.at(i));
}
for (size_t i = middle + 1; i < dataset.size(); i++){
rDataset.push_back(dataset.at(i));
}
construct(root->left, lDataset);
construct(root->right, rDataset);
if (root->left)
root->left->parent = root;
if (root->right)
root->right->parent = root;
return;
}
bool bin_index_t::file_node::set(const data_t& key,const data_t& val)
{
validate_key_len(key);
open_index_file();
if(!root_page)
{
root_page=create_page();
append_page(*root_page);
save_index_data(0,root_page->page_offset);
add_page(root_page);
}
index_t it;
align_key(key,it);
if(get_item(it))
{
index_t old_i=it;
if(it.data_len>=val.size())save_data(it.data_offset,val);
else it.data_offset=append_data(val);
it.data_len=val.size();
if(old_i.data_offset==it.data_offset&&
old_i.data_len==it.data_len)
return false;
update_page(it);
return false;
}
index_t v;
v.key=key;
align_key(key,v);
v.data_len=val.size();
v.data_offset=append_data(val);
add_item(v);
return true;
}
void CTCPProfile::CreateIR(const ROHC::iphdr *ip, const ROHC::tcphdr *tcp, data_t &output) {
size_t headerStartIdx = output.size();
if (!largeCID && cid)
{
output.push_back(CreateShortCID(cid));
}
output.push_back(IRv2Packet);
if (largeCID)
{
SDVLEncode(back_inserter(output), cid);
}
output.push_back(static_cast<uint8_t>(ProfileID()));
size_t crcPos = output.size();
// Add zero crc for now
output.push_back(0);
create_ipv4_static(ip, output);
create_tcp_static(tcp, output);
create_ipv4_regular_innermost_dynamic(ip, output);
//create_tcp_dynamic(msn, reorder_ratio, udp, output);
// Calculate CRC
uint8_t crc = CRC8(output.begin() + headerStartIdx, output.end());
output[crcPos] = crc;
IncreasePacketCount(PT_IR);
++numberOfIRPacketsSent;
++numberOfIRPacketsSinceReset;
}
string TH_0x02::makeStringFromData(const data_t& data, const options_t& options)
{
(void)options;
size_t byteCount = data.size();
size_t colCount = data[0];
size_t rowCount = data[1];
if (data.size() < 2+TH_0x00::dataByteCount || colCount < 1 || rowCount < 1 || colCount > 255 || rowCount > 255
|| ((byteCount - 2) % TH_0x00::dataByteCount != 0) || (colCount*rowCount != (byteCount - 2) / TH_0x00::dataByteCount))
{
std::cerr << "Invalid data array. Needs to contain 1+1+" << TH_0x00::dataByteCount << "*n bytes" << std::endl;
return "";
}
string str = "[";
for (uint i = 2, num = 0; i < byteCount; i += TH_0x00::dataByteCount, num++)
{
if (num % colCount == 0) // first column
{
str += "[";
}
str += TH_0x00::makeStringFromData(data_t(data.begin()+i, data.begin()+i+TH_0x00::dataByteCount));
if (num % colCount < colCount - 1) // not last column
{
str += ",";
} else {
str += "]";
}
}
str += "]";
// TODO: prettified option
return str;
}
int RansacFitLine::GetInliers(const data_t& data,
const std::vector<model_t>& models,
std::vector<int>* inliers) const {
for (size_t i = 0; i < data.size() / 2; i++) {
double distance = fabs(data[2*i] * models[0][0] +
data[2*i+1] * models[0][1] -
models[0][2]);
if (distance <= threshold_) {
inliers->push_back(i);
}
}
return 0;
}
/** Sets the filter.
* The length of the impulse response is arbitrary. It automatically
* performs zero padding if necessary and creates the required number of
* partitions. It is not very efficient since it calls
* \b Convolver::prepare_impulse_response(). It is provided for convenience.
* If you have the filter's transfer function in halfcomplex format use
* \b Convolver::set_filter_f() instead.
* @param filter impulse response of the filter
*/
void Convolver::set_filter_t(const data_t& filter)
{
if (filter.empty())
{
WARNING("You are trying to use an empty filter.");
return;
}
data_t buffer;
// TODO: It would be more efficient to use _fft_plan and _fft_buffer etc.
prepare_impulse_response(buffer, &filter[0], filter.size(), _frame_size);
set_filter_f(buffer);
}
string TH_0x1B::makeStringFromData(const data_t& data, const options_t& options)
{
(void)options;
if (data.size() != dataByteCount)
{
throw invalid_argument("Empty data array. Needs to contain " + to_string(dataByteCount) + " bytes");
}
string coeffR = TH_0x00::makeStringFromData(data_t(data.begin(), data.begin() + TH_0x00::dataByteCount));
string coeffI = TH_0x00::makeStringFromData(data_t(data.begin() + TH_0x00::dataByteCount, data.begin() + 2 * TH_0x00::dataByteCount));
string str = dec2frac(atof(coeffR.c_str())) + "+" + dec2frac(atof(coeffI.c_str())) + "i";
str = regex_replace(str, regex("\\+-"), "-");
return str;
}
secure_comparator_t(const data_t& shared_secret)
: comparator_(NULL)
{
if (shared_secret.empty()) {
throw themispp::exception_t("Secure Comparator must have non-empty shared secret");
}
res_.reserve(512);
comparator_ = secure_comparator_create();
if (!comparator_) {
throw themispp::exception_t("Secure Comparator construction failed");
}
themis_status_t status = secure_comparator_append_secret(comparator_,
&shared_secret[0],
shared_secret.size());
if (THEMIS_SUCCESS != status) {
throw themispp::exception_t("Secure Comparator failed to append secret", status);
}
}
string TH_0x00::makeStringFromData(const data_t& data, const options_t& options)
{
(void)options;
if (data.size() != dataByteCount)
{
throw invalid_argument("Empty data array. Needs to contain " + to_string(dataByteCount) + " bytes");
}
uint flags = data[0];
bool isNegative = (flags >> 7 == 1);
// bool isSeqInit = (flags & 1 == 1); // if true, "used for initial sequence values"
int exponent = data[1] - 0x80;
string number = "";
for (uint i = 2; i < TH_0x00::dataByteCount; i++)
{
number += (data[i] < 0x10 ? "0" : "") + dechex(data[i]); // zero left pad
}
number = number.substr(0, 1) + "." + number.substr(1);
char buf[35] = {0};
sprintf(buf, "%.*f", DECIMAL_DIG, pow(10, exponent) * atof(number.c_str()));
string str(buf);
// Cleanup
if (str.length() > 12)
{
str.erase(str.begin() + 12, str.end());
}
if (str.find('.') != std::string::npos)
{
while (str.back() == '0') str.pop_back();
}
if (str.back() == '.')
{
str.pop_back();
}
str = (isNegative ? "-" : "") + str;
return str;
}
//min-max splits based on impurity measure
bool impurity_splitW_noMiss(data_t data, int& f_split, double& v_split, double imp, boost::numeric::ublas::vector<int>& c_total,args_t& myargs, double (*impurityHandle)(int, boost::numeric::ublas::vector<int>&,double)) {
int NF=myargs.features;
int num_c=myargs.num_c;
double alpha=myargs.alpha;
f_split = -1;
double min = MY_DBL_MAX, cf;
int n = data.size(), i,j ;
double imp_l=0.0, imp_r=0.0, imp_m=0.0, imp_max=0.0; //impurity on the left
double mind;
double v_split_d;
for (int f = 1; f < NF; f++) { //for each feature
cf=myargs.Costs[f];
sort(data.begin(), data.end(), boost::bind(mysortf, _1,_2, f));
boost::numeric::ublas::vector<int> c_l(num_c,0); //number of examples in each class on the left
boost::numeric::ublas::vector<int> c_r(c_total); //number of examples in each class on the right
mind = MY_DBL_MAX;
//assume no missing data
for( i=0;i<n-1;i++){
c_l[data[i]->label]++;
c_r[data[i]->label]--;
// do not consider splitting here if data is the same as next
if (data[i]->features[f] == data[i+1]->features[f])
continue;
imp_l=(*impurityHandle)(num_c, c_l, alpha);
imp_r=(*impurityHandle)(num_c, c_r, alpha);
imp_max=(imp_l < imp_r) ? imp_r: imp_l;
if(imp_max<mind){
mind=imp_max;
v_split_d = (data[i]->features[f] + data[i+1]->features[f])/2;
}
}
if ((imp-mind>0.0000001) && (cf/(imp-mind) < min)) {
min = cf/(imp-mind);
f_split = f;
v_split = v_split_d;
}
}
return min != MY_DBL_MAX;
}
std::string encode(const data_t& data) const {
auto end = reinterpret_cast<const uint8_t*>(&*data.end());
uint8_t tmp[base_t::digits_per_unit];
char out[base_t::digits_per_unit];
std::string text;
text.reserve(this->estimate(data.size()));
auto in = reinterpret_cast<const uint8_t*>(data.data());
size_t n = 0;
while (in < end) {
n = this->stretch(in, tmp, end);
for (size_t i = 0; i < n; ++i) {
out[i] = this->digit(tmp[i]);
}
text.append(out, n);
}
if (this->padding and n != 0 and n != sizeof(out)) {
text.append(sizeof(out) - n, '=');
}
return text;
}
void send(const data_t& data){
ssize_t send_size=secure_session_send(_session, &data[0], data.size());
if(send_size<=0)
throw themispp::exception_t("Secure Session failed sending");
}
bool dt_node::entropy_split(data_t data, vector<int> dataCount, vector<int> invertIdx, int NF, int& f_split, double& v_split, int K, bool par) {
f_split = -1;
double min = MY_DBL_MAX;
int n = data.size(), i;
vector<bool> skip;
//min E(i=1..k) pi * log(1/pi)
for (i = 0; i <= NF; i++)
skip.push_back( (K > 0) ? true : false);
for (i = 0; i < K; i++) {
int f;
do
f = rand() % (NF-2) + 1;
while (!skip[f]);
skip[f] = false;
}
//if (K <= 0)
//return find_split_p(data, NF, f_split, v_split, skip);
vector<int> location(n, -1);
for (int f = 1; f < NF; f++) {
if (skip[f]) continue;
sort_data_by_feature(location, dataCount, invertIdx, f);
//sort_data_by_feature(data,f);
//if (skip[f]) continue;
/*
vector< pair<tuple*, int> > tk;
int z;
for (z = 0; z < data.size(); z++)
tk.push_back( pair<tuple*,int>(data[z], f) );
sort(tk.begin(), tk.end(), mysortpred2);
for (z = 0; z < data.size(); z++)
data[z] = tk[z].first;
*/
int num_c = 7;
vector<double> c_miss, c_left, c_right;
//vector<double> p_miss, p_left, p_right;
//vector<int> freq;
//int n_left = 0, n_right = 0;
for (i=0;i<num_c;i++) {
c_miss.push_back(0.0);
// p_miss.push_back(0.0);
c_left.push_back(0.0);
c_right.push_back(0.0);
// p_left.push_back(0.0);
// p_right.push_back(0.0);
// freq.push_back(0);
}
// get impurity (entropy) for missing data
double M = 0.0, L, R, W_miss = 0.0, W_left = 0.0, W_right = 0.0;
int missing = 0;
int loc = location[missing];
while (data[loc]->features[f] == UNKNOWN && missing < n-1) {
c_miss[(int)data[loc]->target] += data[loc]->weight;
W_miss += data[loc]->weight;
missing++;
loc = location[missing];
}
if (missing == n-1) // all data is missing
continue;
int nn = n - missing; // number of data points that arent missing
// entropy
// put all data into right side R
int start = missing;
for (i = start; i < n; i++) {
loc = location[i];
c_right[(int)data[loc]->target]+=data[loc]->weight;
W_right+=data[loc]->weight;
}
if (W_right){
for (i = 0; i < num_c; i++) {
if (c_right[i])
R += c_right[i]/W_right * log(W_right / c_right[i]);
}
}
if (missing && W_miss) {
for (i = 0; i < num_c; i ++){
if (c_miss[i])
M += c_miss[i]/W_miss * log(W_miss/c_miss[i]);
}
}
L = 0.0;
// for every i
// put yi into left side, remove it from the right side, and calculate squared lost
for (i = start; i < n-1; i++) {
int j = i - missing;
int yn = (int)data[loc]->target;
W_right -= data[loc]->weight;
W_left += data[loc]->weight;
//n_right--;
//n_left++;
c_left[yn] += data[loc]->weight;
c_right[yn] -= data[loc]->weight;
//c_left[yn]++;
//c_right[yn]--;
//p_left[yn] += 1.0 / freq[yn];
//p_right[yn] -= 1.0 / freq[yn];
// do not consider splitting here if data is the same as next
if (data[location[i]]->features[f] == data[location[i+1]]->features[f])
continue;
L = 0.0;
int k;
for (k = 0; k < num_c; k++) {
if (c_left[k])
L += c_left[k]/W_left * log(W_left / c_left[k]);
}
R = 0.0;
for (k = 0; k < num_c; k++) {
if (c_right[k])
R += c_right[k]/W_right * log(W_right / c_right[k]);
}
double ssum = W_left+W_right+W_miss;
double I = W_left/ssum * L + W_right/ssum * R + W_miss/ssum * M;
//I = 1.0*n_left/n *
/*
L = 0.0, R = 0.0;
for (i = 0; i < num_c; i++)
L += p_left;
*/
if (I < min) {
min = I;
f_split = f;
v_split = (data[location[i]]->features[f] + data[location[i+1]]->features[f])/2;
}
}
}
return min != MY_DBL_MAX;
}
void bin_index_t::inode::validate_key_len(const data_t& key) const
{
if(key.size()!=key_len)throw std::runtime_error("invalid key length");
}
int getSize() const
{
return m_verts.size();
}
bool io::uncompress_gzip(const data_t& istream, data_t& data)
{
stream_t stream(istream.data(), istream.size());
return io_uncompress_gzip(stream, stream.size(), data);
}