static std::shared_ptr<planner_node> make_planner_node(
std::shared_ptr<planner_node> source,
group_aggregate_value& aggregator,
flex_type_enum output_type) {
std::shared_ptr<group_aggregate_value> agg(aggregator.new_instance());
return planner_node::make_shared(planner_node_type::REDUCE_NODE,
{{"output_type", (int)(output_type)}},
{{"aggregator", any(agg)}},
{source});
}
void TestExpertInterface(int nThreads) {
pq_t my_pq;
tbb::aggregator_ext<my_handler> agg((my_handler(&my_pq)));
for (int i=0; i<MaxThread; ++i) shared_data[i] = 0;
REMARK("Testing aggregator expert interface.\n");
NativeParallelFor(nThreads, ExpertBody(my_pq, agg));
for (int i=0; i<nThreads; ++i)
ASSERT(shared_data[i] == N, "wrong number of elements pushed");
REMARK("Done testing aggregator expert interface.\n");
}
static std::shared_ptr<query_operator> from_planner_node(
std::shared_ptr<planner_node> pnode) {
ASSERT_EQ((int)pnode->operator_type, (int)planner_node_type::REDUCE_NODE);
ASSERT_EQ(pnode->inputs.size(), 1);
ASSERT_TRUE(pnode->operator_parameters.count("output_type"));
ASSERT_TRUE(pnode->any_operator_parameters.count("aggregator"));
std::shared_ptr<group_aggregate_value> aggregator =
pnode->any_operator_parameters["aggregator"].as<std::shared_ptr<group_aggregate_value>>();
flex_type_enum output_type =
(flex_type_enum)(flex_int)(pnode->operator_parameters["output_type"]);
std::shared_ptr<group_aggregate_value> agg(aggregator->new_instance());
return std::make_shared<operator_impl>(agg, output_type);
}
Test::TestResult AggregationPerfTest::execTest()
{
// create a packet sampler which lets only half of the packets through
// NOTICE: the sampler will be destroyed by the d'tor of FilterModule
ConnectionQueue<Packet*> queue1(10);
TestQueue<IpfixRecord*> tqueue;
PacketAggregator agg(1);
Rules* rules = createRules();
agg.buildAggregator(rules, 0, 0, 16);
queue1.connectTo(&agg);
agg.connectTo(&tqueue);
agg.start();
queue1.start();
struct timeval starttime;
REQUIRE(gettimeofday(&starttime, 0) == 0);
sendPacketsTo(&queue1, numPackets);
// check that at least one record was received
IpfixRecord* rec;
ASSERT(tqueue.pop(1000, &rec), "received timeout when should have received flow!");
struct timeval stoptime;
REQUIRE(gettimeofday(&stoptime, 0) == 0);
struct timeval difftime;
REQUIRE(timeval_subtract(&difftime, &stoptime, &starttime) == 0);
printf("Aggregator: needed time for processing %d packets: %d.%06d seconds\n", numPackets, (int)difftime.tv_sec, (int)difftime.tv_usec);
queue1.shutdown();
agg.shutdown();
return PASSED;
}
int main(int /*argc*/, char* /*argv*/[])
{
HuboCan::HuboDescription desc;
desc.parseFile("../HuboCan/misc/Hubo2Plus.dd");
HuboCmd::Aggregator agg(desc);
agg.run();
while(true)
{
sleep(1);
const HuboCmd::JointCmdArray& cmds = agg.update();
size_t stop = simple_min(10, desc.getJointCount());
for(size_t i=0; i < stop; ++i)
{
std::cout.width(12);
std::cout << cmds[i].position;
}
std::cout << std::endl;
}
return 0;
}
inline std::shared_ptr<query_operator> clone() const {
std::shared_ptr<group_aggregate_value> agg(m_aggregator->new_instance());
return std::make_shared<operator_impl>(agg, m_output_type);
}
void partial_aggregate_stage_t::process_packet() {
partial_aggregate_packet_t* packet;
packet = (partial_aggregate_packet_t*) _adaptor->get_packet();
tuple_fifo* input_buffer = packet->_input_buffer;
dispatcher_t::dispatch_packet(packet->_input);
_aggregate = packet->_aggregate;
key_extractor_t* agg_key = _aggregate->key_extractor();
key_extractor_t* tup_key = packet->_extractor;
key_compare_t* compare = packet->_compare;
// create a set to hold the sorted run
tuple_less_t less(agg_key, compare);
tuple_set_t run(less);
// read in the tuples and aggregate them in the set
while(!input_buffer->eof()) {
_page_list.clear();
_page_count = 0;
_agg_page = NULL;
while(_page_count < MAX_RUN_PAGES && !input_buffer->eof()) {
guard<qpipe::page> page = NULL;
tuple_t in;
while(1) {
// out of pages?
if(!input_buffer->get_tuple(in))
break;
int hint = tup_key->extract_hint(in);
// fool the aggregate's key extractor into thinking
// the tuple is an aggregate. Use pointer math to put
// the tuple's key bits where the aggregate's key bits
// are supposed to go. (the search only touches the
// key bits anyway, which are guaranteed to be the
// same for both...)
size_t offset = agg_key->key_offset();
char* key_data = tup_key->extract_key(in);
hint_tuple_pair_t key(hint, key_data - offset);
// Supposedly, insertion with a proper hint is
// amortized O(1) time. However, the definition of
// "proper" is ambiguous:
// - SGI STL reference: insertion point *before* hint
// - Solaris STL source: insert point *after* hint
// - Dinkum STL reference: insertion point *adjacent* to hint
// - GNU STL: no documentation (as usual); adjacent?
// - Solaris profiler: same performance as vanilla insert either way
// find the lowest aggregate such that candidate >=
// key. This is either the aggregate we want or a good
// hint for the insertion that otherwise follows
tuple_set_t::iterator candidate = run.find(key);
if(candidate == run.end()) {
// initialize a blank aggregate tuple
hint_tuple_pair_t agg(hint, NULL);
if(alloc_agg(agg, key_data))
break;
// insert the new aggregate tuple
candidate = run.insert(agg).first;
}
else {
TRACE(TRACE_DEBUG, "Merging a tuple\n");
}
// update an existing aggregate tuple (which may have
// just barely been inserted)
_aggregate->aggregate(candidate->data, in);
}
}
// TODO: handle cases where the run doesn't fit in memory
assert(input_buffer->eof());
// write out the result
size_t out_size = packet->_output_filter->input_tuple_size();
array_guard_t<char> out_data = new char[out_size];
tuple_t out(out_data, out_size);
for(tuple_set_t::iterator it=run.begin(); it != run.end(); ++it) {
// convert the aggregate tuple to an output tuple
_aggregate->finish(out, it->data);
_adaptor->output(out);
}
}
}
static std::vector< typename sparse::matrix_index<spmat>::type >
aggregates( const spmat &A, const std::vector<char> &S ) {
typedef typename sparse::matrix_index<spmat>::type index_t;
typedef typename sparse::matrix_value<spmat>::type value_t;
const index_t n = sparse::matrix_rows(A);
const index_t undefined = static_cast<index_t>(-1);
const index_t removed = static_cast<index_t>(-2);
std::vector<index_t> agg(n);
const index_t *Arow = sparse::matrix_outer_index(A);
const index_t *Acol = sparse::matrix_inner_index(A);
// Remove nodes without neighbours
index_t max_neib = 0;
for(index_t i = 0; i < n; ++i) {
index_t j = Arow[i];
index_t e = Arow[i + 1];
max_neib = std::max(e - j, max_neib);
index_t state = removed;
for(; j < e; ++j)
if (S[j]) {
state = undefined;
break;
}
agg[i] = state;
}
std::vector<index_t> neib;
neib.reserve(max_neib);
index_t last_g = static_cast<index_t>(-1);
// Perform plain aggregation
for(index_t i = 0; i < n; ++i) {
if (agg[i] != undefined) continue;
// The point is not adjacent to a core of any previous aggregate:
// so its a seed of a new aggregate.
agg[i] = ++last_g;
neib.clear();
// Include its neighbors as well.
for(index_t j = Arow[i], e = Arow[i + 1]; j < e; ++j) {
index_t c = Acol[j];
if (S[j] && agg[c] != removed) {
agg[c] = last_g;
neib.push_back(c);
}
}
// Temporarily mark undefined points adjacent to the new aggregate as
// beloning to the aggregate. If nobody claims them later, they will
// stay here.
for(typename std::vector<index_t>::const_iterator nb = neib.begin(); nb != neib.end(); ++nb)
for(index_t j = Arow[*nb], e = Arow[*nb + 1]; j < e; ++j)
if (S[j] && agg[Acol[j]] == undefined) agg[Acol[j]] = last_g;
}
assert( std::count(agg.begin(), agg.end(), undefined) == 0 );
return agg;
}
bool
SuifPrinterModule::parse_and_print(ostream& output, const ObjectWrapper &obj,
const LString &name, const String &str,
int _indent, int deref = 0)
{
const Address what = obj.get_address();
const MetaClass *type = obj.get_meta_class();
// ObjectWrapper obj(what, type);
//output << "str:deref = " << deref << endl;
int l_sep = list_separator;
if (str.length() == 0) {
return false;
}
if (deref)
_indent -= istep;
int str_length = str.length();
bool need_indent = false;
bool first_field = true;
for (int i = 0; i < str_length; ++i) {
if (str[i] != '%') {
// If there are less than 2 extra stars don't print anything but the
// fields.
if (deref < 2) {
// Need to check for \n and take care of indentation here
switch(str[i]) {
case '\n':
output << str[i];
if (str[i+1]) {
need_indent = true;
indent(output, _indent);
}
break;
case '\t': indent(output, istep); break;
case '\b': _indent -= istep; break;
default: output << str[i];
}
}
}
else {
++i;
if (str[i] == '%') {
// Double % means print out a '%'
output << '%';
}
else {
// This has to be cleaned up a bit ...
//int field_deref = deref?deref-1:0;
int field_deref = 0;
char buff[256];
int j = 0;
char c = str[i++];
if (c == '*') {
++field_deref;
while ((c = str[i++]) == '*')
++ field_deref;
}
while (isalnum(c) || c == '_') {
buff[j++] = c;
c = str[i++];
}
i -= 2;
buff[j] = 0;
// Now retrieve the particular field and print it.
if (!strcmp(buff, "Cl")) {
output << type->get_instance_name() << '('
<< type->get_meta_class_id() << ") ";
}
else if (!strcmp(buff, "ii")) { // Deal with printing IInteger
IInteger *ii = (IInteger *)what;
output << ii->to_String().c_str();
}
else if (!strcmp(buff, "i")) { // Deal with printing int
output << *(int*)what;
}
else if (!strcmp(buff, "f")) { // float
output << *(float*)what;
}
else if (!strcmp(buff, "d")) { // double
output << *(double*)what;
}
else if (!strcmp(buff, "c")) { // char
output << *(char*)what;
}
else if (!strcmp(buff, "b")) { // byte
output << (int)*(char*)what;
}
else if (!strcmp(buff, "B")) { // bool
output << *(bool*)what;
}
else if (!strcmp(buff, "ls")) { // Deal with printing LStrings
LString str = *(LString*) what;
output << str;
}
else if (!strcmp(buff, "s")) { // Deal with printing Strings
String str = *(String*) what;
output << str;
}
else if (!strcmp(buff, "n")) { // Deal with name of field
if (!deref)
output << name;
}
else if (!strcmp(buff, "P")) {
if (obj.is_null())
output << "NULL";
else {
PointerWrapper ptr_obj(obj);
ObjectWrapper base_obj = ptr_obj.dereference();
if (ptr_obj.get_meta_class()->is_owning_pointer()) {
size_t ref = retrieve_tag(obj.get_object());
output << "t" << ref<< ": ";
print2(output, base_obj, emptyLString,
_indent+istep, field_deref);
}
else {
print_pointer(output, ptr_obj, emptyLString, _indent, deref);
}
}
}
else if (!strcmp(buff, "R")) { // print the ref #
if (!what)
output << "NULL";
else {
// PointerMetaClass *p = (PointerMetaClass*) type;
// const Address baseAddr = *(Address*) type;
//ObjectWrapper obj(what, type);
size_t ref = retrieve_tag(obj);
output << "t" << ref<< ": ";
}
}
else if (!strcmp(buff, "LS")) {
list_separator = ' ';
}
else if (!strcmp(buff, "LN")) {
list_separator = '\n';
}
else if (!strcmp(buff, "LC")) {
list_separator = ',';
}
else if (!strcmp(buff, "ANNOTES")) {
// Special CASE for handling ANNOTATIONS
AggregateWrapper agg(obj);
LString field_name("_annotes");
FieldDescription *f = agg.get_field_description(field_name);
if (!f)
cerr << type->get_meta_class(what)->get_class_name()
<< ":No field '" << field_name << "' found to print!!!\n";
else {
// Now we need to get the field offset and increment 'what'
if (field_deref != 0)
cerr << "Extra '*' for %ANNOTES\n";
FieldWrapper field = agg.get_field(field_name);
if (need_indent) {
indent(output, istep);
need_indent = false;
}
char old_sep = list_separator;
list_separator = '\n';
print2(output,
field.get_object(),
field_name, _indent+istep,
1);
list_separator = old_sep;
}
}
else if (j) {
// Retrieve the field mentioned
// The following cast works as we should reach here only if it
// is not an elementary or pointer type.
AggregateWrapper agg(obj);
char *bf = buff;
LString field_name(bf);
FieldDescription *f = agg.get_field_description(field_name);
if (!f)
cerr << type->get_meta_class(what)->get_class_name()
<< ":No field '" << field_name << "' found to print!!!\n";
else {
// Now we need to get the field offset and increment 'what'
if (deref)
if (!first_field) output << ' ';
else first_field = false;
FieldWrapper field = agg.get_field(field_name);
//char *f_add = (char*)what + f->get_offset();
//indent(output, _indent+istep);
if (need_indent) {
indent(output, istep);
need_indent = false;
}
if (deref && !field_deref)
field_deref = deref - 1;
//output << "\tstr:field_deref = " << field_deref << endl;
print2(output,
field.get_object(),
field_name, _indent+istep,
field_deref);
}
}
}
}
}
list_separator = l_sep;
return true;
}
