int whiteSpaces () {
Collector c;
c.write (" \t \n <?xml \t \n version=\"1.0\" ?> \t \n <opener > \t \n <element/> </opener>");
tcheck (c.state() == XMLStreamDecoder::XS_Closed, "Was valid!");
tcheck (c.received.size() == 1 && c.received[0].name() == "element", "");
return 0;
}
/**
* Function that inits and runs the collector
* @param config_file Destination of the collectors configuration file
*/
int run_collector(const std::string& configFile)
{
Collector collector;
/* read the configuration */
try {
collector.readConfig(configFile);
}catch (exceptions::ConfigError e) {
msg(MSG_FATAL, "Couldn't configure collector: %s", e.what());
return error_states::CONFIG_ERROR;
}
/* start the detection modules */
try {
collector.startModules();
}catch (exceptions::DetectionModuleError e) {
msg(MSG_FATAL, "Couldn't start detection modules: %s", e.what());
return error_states::DETECTION_MODULE_ERROR;
}
/* wait for detection modules to do their initialisation work */
msg(MSG_INFO, "Waiting for detection modules to init their stuff....");
sleep(2);
msg(MSG_INFO, "Starting up collector...");
try {
collector.run();
} catch (std::exception e) {
msg(MSG_FATAL, "Error while running the collector: %s", e.what());
return error_states::RUN_ERROR;
}
msg(MSG_DEBUG, "Leaving run_collector()");
return 0;
}
translation_table_t PhraseTable::GetAllTranslationOptions(const vector<wid_t> &sentence, context_t *context) {
translation_table_t ttable;
for (size_t start = 0; start < sentence.size(); ++start) {
Collector *collector = self->index->NewCollector(context);
vector<wid_t> phrase;
vector<wid_t> phraseDelta;
for (size_t end = start; end < sentence.size(); ++end) {
wid_t word = sentence[end];
phrase.push_back(word);
phraseDelta.push_back(word);
if (ttable.find(phrase) == ttable.end()) {
vector<sample_t> samples;
collector->Extend(phraseDelta, self->numberOfSamples, samples);
phraseDelta.clear();
if (samples.empty())
break;
vector<TranslationOption> options;
MakeTranslationOptions(self->index, self->aligner, phrase, samples, options);
ttable[phrase] = options;
}
}
delete collector;
}
return ttable;
}
void add(Synt synt,const Element *base)
{
Element T(synt);
ulen K=synt.getIndex();
collector.append_fill(K,N,T);
collector.append_fill(K,N,Range(base+K,1));
}
int strangeProtocol () {
Collector c;
c.write ("brevlmlm");
tcheck (c.state() == XMLStreamDecoder::XS_Error, "");
Collector c2;
c2.write ("<?xml version=\"1.0\"?> renterntknt4kntkntkln");
tcheck (c2.state() == XMLStreamDecoder::XS_Error, "");
return 0;
}
TEST(Collector, CanRegister)
{
Collector c;
auto t = std::make_shared<Timer>();
auto uc = std::make_shared<UnsignedCounter>();
c.register_metric(t);
c.register_metric(uc);
std::cout << c.report();
}
bool PageRankReducer::reduce(const void* key, KeyValueIterator& values, Collector& collector) const
{
//<url, pagerank, linkoutlist> as reducer intput
//<url, newpagerank, linkoutlist> as reducer output
//the formula of pagerank:
//pr(a) - (1-d) = d * (pr(b)/l(b) + pr(c)/l(c) + ...)
//d is the damping factor
PageRankValue value;
int count = 0;
const double damping = 0.85;
while(values.hasMore()){
PageRankValue* pValue = (PageRankValue*)(values.next());
value.pagerank += damping * pValue->pagerank;
if(pValue->linkoutURLs.size() > 0){
for(int i = 0; i < pValue->linkoutURLs.size(); i ++){
value.linkoutURLs.push_back(pValue->linkoutURLs[i]);
}
}
count ++;
}
collector.collect(key, &value);
return true;
}
ulen TopLang::makeRules(Collector<RuleRec> &collector,const CondLangBase::RuleDesc &rule)
{
ulen ret=0;
DynArray<ElementRecExt> temp(DoCast(rule.args.len),rule.args.ptr);
{
ulen index=0;
for(auto &rec : temp ) rec.element_index=index++;
}
auto args=Range(temp);
do
{
if( TestCond(Range_const(args),rule.cond) )
{
collector.append_fill(makeRuleName(rule.name,Range_const(args)),rule.index,Range_const(args));
ret++;
}
}
while( Next(args) );
return ret;
}
TEST(Collector, CanReport)
{
Collector c;
auto t = std::make_shared<Timer>();
auto uc = std::make_shared<UnsignedCounter>();
c.register_metric(t);
c.register_metric(uc);
uc->add();
size_t bs;
uint8_t* bf = c.buffer(bs);
std::cout << "size " << bs << ": " << std::hex << (char*)bf << std::endl;
delete [] bf;
}
CollectorArray::iterator find_connecting( CollectorArray::iterator& target, CollectorArray& input )
{
CollectorArray::iterator ret = input.end();
Collector* pp = *target;
if ( pp->checkClosed() ) {
return ret;
}
if (overlap( pp->_points.back(), pp->_points.front() ) ) {
// poly2tri only accept polyline with non repeating points
pp->_points.pop_back();
pp->_closed = true;
return ret;
}
// Look for another unclosed path of the same style,
// which could join our begin or end point.
for (CollectorArray::iterator it = input.begin(); it != input.end(); ++it) {
Collector* po = (*it);
if ( pp == po )
continue;
if ( po->checkClosed() )
continue;
// Can we join?
PointDataArray::iterator pStart = pp->_points.begin();
PointDataArray::iterator oStart = po->_points.begin();
if (overlap( *oStart, pp->_points.back() ) ) {
// Yes, po can be appended to pp.
pp->_points.insert( pp->_points.end(), ++oStart, po->_points.end());
po->_right_style = Path::kINVALID;
MergeRectangle(pp->_bound, po->_bound);
ret = it; // remove this iterator and restart
return ret;
} else if (overlap( *pStart, po->_points.back() ) ) {
// Yes, pp can be appended to po.
po->_points.insert( po->_points.end(), ++pStart, pp->_points.end());
pp->_right_style = Path::kINVALID;
MergeRectangle(po->_bound, pp->_bound);
ret = target; // remove this iterator and restart
return ret;
}
}
return ret;
}
extern "C" int startApp(thread_arg_t* arg) {
int argc = arg->argc;
char** argv = arg->argv;
create_vm_t func = arg->func;
free(arg);
#ifndef MULTIPLE_GC
Collector::inject_my_thread(&argc);
VirtualMachine* VM = func();
VM->runApplication(argc, argv);
Collector::remove_my_thread();
Collector::collect();
#else
Collector* GC = Collector::allocate();
GC->inject_my_thread(&argc);
func(argc, argv);
GC->remove_my_thread();
GC->collect();
#endif
return 0;
}
void add(Rule rule)
{
ulen K=rule.getRet().getIndex();
PtrLen<const Element> args=rule.getArgs();
if( +args )
{
Vertex cur(K);
Element E=*args;
++args;
E.applyForSynt( [=] (Synt synt) { collector.append_fill(cur,synt.getIndex(),args); } );
if( +args )
for(;;)
{
collector.append_fill(cur,N,E,args);
Element F=*args;
++args;
F.applyForSynt( [=] (Synt synt) { collector.append_fill(cur,synt.getIndex(),E,args); } );
if( !args ) break;
Vertex v=extra.get();
collector.append_fill(cur,v,E);
cur=v;
E=F;
}
}
}
void RunTest(SuffixArray &index, const context_t *context,
const unordered_map<vector<wid_t>, size_t, phrase_hash> &ngrams, vector<speed_perf_t> &speedData) {
size_t queryCount = 0;
for (auto entry = ngrams.begin(); entry != ngrams.end(); ++entry) {
Collector *collector = index.NewCollector(context, true);
for (size_t i = 0; i < entry->first.size(); ++i) {
double begin = GetTime();
vector<sample_t> samples;
collector->Extend(entry->first[i], 1000, samples);
speedData[i].seconds += GetElapsedTime(begin);
speedData[i].requests++;
queryCount++;
if (queryCount % 10000 == 0)
cout << "." << flush;
}
delete collector;
}
}
bool WordFrequencyMapper::map(const void* key, const void* value, Collector& collector) const
{
//<char*, char*> as mapper intput
//<char*, int> as mapper output
int outValue = 1;
int outputCnt = 0;
int bodyLen = strlen(*(char**)value);
//do parsing
m_parser->init_page(*(char**)value);
m_parser->convert_charset("skip");
m_parser->delete_tags();
m_parser->cut_words(1);
m_parser->end_page();
//processing with the keywords list
char *p = m_parser->get_keywords();
//cout << p << endl;
p=strtok(p," ");
while(p){
switch(*p){
case DELI_WEIGHT:
case DELI_LOC:
case DELI_PARA:
//skip these tokens
break;
case '\0':
cout << "+";
default:{
char** pData = &p;
collector.collect( pData, &outValue);
outputCnt ++;
}
}
p=strtok(NULL," ");
}
//cout << *(char**)key << " : " << bodyLen << " : " << outputCnt << endl;
return true;
}
bool PageRankReducer::reduce(const void* key, KeyValueIterator& values, Collector& collector) const
{
//<url, pagerank, linkoutlist> as reducer intput
//<url, newpagerank, linkoutlist> as reducer output
//the formula of pagerank:
//pr(a) - (1-d) = d * (pr(b)/l(b) + pr(c)/l(c) + ...)
//d is the damping factor
PageRankValue value;
int count = 0;
const double damping = 0.85;
while(values.hasMore()){
PageRankValue* pValue = (PageRankValue*)(values.next());
value.pagerank += damping * pValue->pagerank;
if(pValue->urlNumber > 0){
if(value.linkoutURLs != NULL && value.urlNumber > 0){
char** temp = new char*[value.urlNumber + pValue->urlNumber];
memcpy(temp, value.linkoutURLs, value.urlNumber * sizeof(char*) );
memcpy(temp + value.urlNumber, pValue->linkoutURLs, pValue->urlNumber * sizeof(char*) );
value.urlNumber += pValue->urlNumber;
delete[] value.linkoutURLs;
value.linkoutURLs = temp;
}
else{
value.urlNumber = pValue->urlNumber;
value.linkoutURLs = pValue->linkoutURLs;
}
}
count ++;
}
collector.collect(key, &value);
return true;
}
void OnMouse(int event, int x, int y, int flags, void *userdata) {
Collector *c = (Collector *)userdata;
c->OnClick(event, x, y, flags, NULL);
}
int start() { return subscriber.start(collector.bind()); }
int call(uavcan::NodeID node_id, const typename DataType::Request& request)
{
client.setCallback(collector.bind());
return client.call(node_id, request);
}
TopLang::TopLang(const CondLang &clang)
{
Collector<RuleRec> collector;
DynArray<ulen> map(DoRaw(clang.getSyntCount()));
// atoms
{
auto range=clang.getAtoms();
auto atoms=createAtoms(range.len);
for(; +atoms ;++atoms,++range)
{
atoms->index=range->index;
atoms->name=pool.dup(range->name);
atoms->map_index=range->index;
}
}
// synts
{
auto range=clang.getSynts();
ulen len=0;
for(auto &synt : range ) len=LenAdd(len, Max<ulen>(synt.kinds.len,1) );
auto synts=createSynts(len);
ulen index=0;
ulen map_index=0;
for(; +range ;++range,++map_index)
if( +range->kinds )
{
map[map_index]=index;
StrLen name=range->name;
bool is_lang=range->is_lang;
ulen desc_map_index=range->index;
for(auto &kind : range->kinds )
{
synts->rules.len=makeRules(collector,*range,kind.index);
synts->index=index++;
synts->name=pool.cat(name,StrLen(".",1),kind.name);
synts->is_lang=is_lang;
synts->map_index=desc_map_index;
synts->kind_index=kind.index;
++synts;
}
}
else
{
map[map_index]=index;
synts->rules.len=makeRules(collector,*range);
synts->index=index++;
synts->name=pool.dup(range->name);
synts->is_lang=range->is_lang;
synts->map_index=range->index;
++synts;
}
}
// rules
{
auto range=collector.flat();
auto rules=createRules(range.len);
auto atoms=getAtoms();
auto synts=getSynts();
ulen index=0;
for(; +rules ;++rules,++range)
{
rules->index=index++;
rules->name=range->name;
rules->map_index=range->map_index;
auto arange=Range_const(range->args);
auto args=createElements(*rules,arange.len);
for(; +args ;++args,++arange)
{
arange->element.apply( [=] (const CondLangBase::AtomDesc *atom) { args->ptr=&(atoms[atom->index]); } ,
[=,&map] (const CondLangBase::SyntDesc *synt) { args->ptr=&(synts[map[synt->index]+arange->kind_index]); } );
}
}
}
// synt.rules rules.ret
{
auto synts=this->synts;
auto *ptr=rules.ptr;
for(; +synts ;++synts)
{
ulen len=synts->rules.len;
synts->rules.ptr=ptr;
for(auto &rule : Range(ptr,len) ) rule.ret=synts.ptr;
ptr+=len;
}
}
pool.shrink_extra();
}
void CircuitsWidget::deserialize(QString fileName) {
std::string fn = fileName.toUtf8().constData();
// n.b. Use this to convert QString in std::string on Windows!
//std::string fn = qs.toLocal8Bit().constData();
string line;
ifstream file;
file.open(fn.c_str());
// test
while(getline(file,line)){
if(line.compare("#") == 0){
getline(file,line); // reading type
int t = atoi(line.c_str());
getline(file,line); // reading id
int id = atoi(line.c_str());
getline(file,line); // reading x position
int x = atoi(line.c_str());
getline(file,line); // reading y position
int y = atoi(line.c_str());
getline(file,line); // reading width
int w = atoi(line.c_str());
getline(file,line); // reading height
int h = atoi(line.c_str());
getline(file,line); // reading value
string sval = line;
int val = atoi(line.c_str());
switch(t) {
case(8):{
//muler
Muler *m = new Muler(this);
drawComponent(m);
m->setId(id);
m->move(x,y);
m->resize(w,h);
m->setValue(val);
break;
}
case(7):{
//diver
Diver *d = new Diver(this);
drawComponent(d);
d->setId(id);
d->move(x,y);
d->resize(w,h);
d->setValue(val);
// NOTE: this is a particular case, please see under in "restoring connections"
break;
}
case(6):{
//noter
Noter *n = new Noter(this);
drawComponent(n);
n->setId(id);
n->move(x,y);
n->resize(w,h);
n->setValue(val);
break;
}
case(5):{
//orer
Orer *o = new Orer(this);
drawComponent(o);
o->setId(id);
o->move(x,y);
o->resize(w,h);
o->setValue(val);
break;
}
case(4):{
// ander
Ander *a = new Ander(this);
drawComponent(a);
a->setId(id);
a->move(x,y);
a->resize(w,h);
a->setValue(val);
break;
}
case(3):{
// variable
Variable *v = new Variable(this);
drawComponent(v);
v->setId(id);
v->move(x,y);
v->resize(w,h);
v->setValue(val);
v->var->setPlaceholderText(sval.c_str());
break;
}
case(2):{
// collector
Collector *c = new Collector(this);
drawComponent(c);
c->setId(id);
c->move(x,y);
c->resize(w,h);
c->setValue(val);
break;
}
case(1):{
// adder
Adder *a = new Adder(this);
drawComponent(a);
a->setId(id);
a->move(x,y);
a->resize(w,h);
a->setValue(val);
break;
}
default:{
printf("FATAL ERROR: bad format\n");
exit(1);
}
}
}
if(line.compare("@") == 0){
getline(file,line); // reading id
int id = atoi(line.c_str());
getline(file,line); // reading incoming components number
int nin = atoi(line.c_str());
getline(file,line); // reading outcoming components number
int nou = atoi(line.c_str());
// restoring connections
int i = 0;
while(componentList[i]->getId() != id)
i++;
for(int j = 0; j < nin; j++){
getline(file,line);
int z = atoi(line.c_str());
int q = 0;
while(componentList[q]->getId() != z)
q++;
componentList[i]->inComponentList.append(componentList[q]);
if(componentList[i]->getComponentType()==7){
// restoring correct divider if the component is a diver
Diver* d = (Diver*) componentList[i].data(); // pay attenction
d->setDivider(componentList[q]->getValue() / componentList[i]->getValue());
}
Wire *w = new Wire(componentList[i], componentList[q]);
wireList.append(w);
}
for(int j = 0; j < nou; j++){
getline(file,line);
int z = atoi(line.c_str());
int q = 0;
while(componentList[q]->getId() != z)
q++;
componentList[i]->outComponentList.append(componentList[q]);
Wire *w = new Wire(componentList[i], componentList[q]);
wireList.append(w);
}
}
}
file.close();
repaint();
setFocus();
}
void finish(DynArray<Arrow> &ret)
{
collector.copyTo(ret);
}
static void
on_read (gpointer data)
{
counter_t *counter = data;
int mask = (N_PAGES * get_page_size() - 1);
int n_bytes = mask + 1;
gboolean skip_samples;
Collector *collector;
uint64_t head, tail;
collector = counter->collector;
tail = counter->tail;
head = counter->mmap_page->data_head;
rmb();
if (head < tail)
{
g_warning ("sysprof fails at ring buffers (head "FMT64", tail "FMT64"\n", head, tail);
tail = head;
}
#if 0
/* Verify that the double mapping works */
x = g_random_int() & mask;
g_assert (*(counter->data + x) == *(counter->data + x + n_bytes));
#endif
skip_samples = in_dead_period (collector);
#if 0
g_print ("n bytes %d\n", head - tail);
#endif
while (head - tail >= sizeof (struct perf_event_header))
{
struct perf_event_header *header;
guint8 buffer[4096];
guint8 *free_me;
free_me = NULL;
/* Note that:
*
* - perf events are a multiple of 64 bits
* - the perf event header is 64 bits
* - the data area is a multiple of 64 bits
*
* which means there will always be space for one header, which means we
* can safely dereference the size field.
*/
header = (struct perf_event_header *)(counter->data + (tail & mask));
if (header->size > head - tail)
{
/* The kernel did not generate a complete event.
* I don't think that can happen, but we may as well
* be paranoid.
*/
break;
}
if (counter->data + (tail & mask) + header->size > counter->data + n_bytes)
{
int n_before, n_after;
guint8 *b;
if (header->size > sizeof (buffer))
free_me = b = g_malloc (header->size);
else
b = buffer;
n_after = (tail & mask) + header->size - n_bytes;
n_before = header->size - n_after;
memcpy (b, counter->data + (tail & mask), n_before);
memcpy (b + n_before, counter->data, n_after);
header = (struct perf_event_header *)b;
}
if (!skip_samples || header->type != PERF_RECORD_SAMPLE)
{
if (header->type == PERF_RECORD_SAMPLE)
collector->n_samples++;
process_event (collector, counter, (counter_event_t *)header);
}
if (free_me)
g_free (free_me);
tail += header->size;
}
counter->tail = tail;
counter->mmap_page->data_tail = tail;
if (collector->callback)
{
if (collector->n_samples - collector->prev_samples >= N_WAKEUP_EVENTS)
{
gboolean first_sample = collector->prev_samples == 0;
collector->callback (first_sample, collector->data);
collector->prev_samples = collector->n_samples;
}
}
}
int validStart () {
Collector c;
tcheck (c.state() == XMLStreamDecoder::XS_Start, "Start State");
c.write ("<?xml version=\"");
tcheck (c.state() == XMLStreamDecoder::XS_Start, "");
c.write ("1.0\"?>");
tcheck (c.state() == XMLStreamDecoder::XS_ReadXmlBegin, "");
c.write ("<open");
tcheck (c.state() == XMLStreamDecoder::XS_ReadXmlBegin, "");
c.write ("er>");
tcheck (c.state() == XMLStreamDecoder::XS_ReadOpener, "");
tcheck (c.opener().name() == "opener", "");
c.write ("<element>Hi!</element>");
tcheck (c.received.size() == 1 && c.received[0].name() == "element", "Could not read first element");
c.write ("<and>Bla<sub>yeah</sub></and>");
tcheck (c.received.size() == 2 && c.received[1].name() == "and", "Could not read 2nd element");
c.write ("</opener>");
tcheck (c.state() == XMLStreamDecoder::XS_Closed, "Did not detect closing");
return 0;
}
