size_t WHtreeProcesser::flattenSelection( std::list< size_t > selection, bool keepBaseNodes )
{
if( keepBaseNodes && !m_tree.testRootBaseNodes() )
{
std::cerr << "WARNING@ flattenSelection: base nodes have mixed nodes and leaves, flattening will be standard " << std::endl;
keepBaseNodes = false;
}
while( !selection.empty() )
{
size_t thisNode( selection.front() );
selection.pop_front();
std::vector< nodeID_t > kids( m_tree.getNode( thisNode ).getChildren() );
for( size_t i = 0; i < kids.size(); ++i )
{
if( kids[i].first )
{
if( keepBaseNodes && ( m_tree.getNode( kids[i] ).getHLevel() == 1 ) )
{
continue;
}
else
{
m_tree.fetchNode( kids[i].second )->setFlag( true );
selection.push_back( kids[i].second );
}
}
}
}
std::pair< size_t, size_t > pruned( m_tree.cleanup() );
return pruned.second;
} // end "flattenSelection()" -----------------------------------------------------------------
double SemiMarkov::GetBestPath(vector<int> *path) const {
assert(forward_);
// Back pointers to find path.
int n = num_timesteps_;
int m = num_states_;
int cur_time = n;
int cur_state = m;
double total_score = best_score_[cur_time][cur_state];
double check_score = 0.0;
path->clear();
path->push_back(n);
while (cur_state != 0) {
int last_time = cur_time;
cur_time = best_action_[cur_time][cur_state];
assert(cur_time != -1);
assert(!pruned(cur_time, last_time - 1));
check_score += score(cur_time, last_time -1, cur_state - 1);
path->push_back(cur_time);
cur_state--;
}
assert((int)path->size() == num_states_ + 1);
cerr << total_score << " " << check_score <<endl;
//assert(fabs(total_score - check_score) < 1e-4);
reverse(path->begin(), path->end());
return total_score;
}
void UniformCostSearch::print_frame_data( int frame_number, float elapsed, Action curr_action, std::ostream& output )
{
output << "frame=" << frame_number;
output << ",expanded=" << expanded_nodes();
output << ",generated=" << generated_nodes();
output << ",pruned=" << pruned();
output << ",depth_tree=" << max_depth();
output << ",tree_size=" << num_nodes();
output << ",best_action=" << action_to_string( curr_action );
output << ",branch_reward=" << get_root_value();
output << ",elapsed=" << elapsed << std::endl;
}
void SemiMarkov::ViterbiForward() {
clock_t start = clock();
int n = num_timesteps_;
int m = num_states_;
// Score the first round correctly.
for (int t = 0; t < n + 1; ++t) {
for (int i = 0; i < m + 1; ++i) {
best_action_[t][i] = -1;
best_score_[t][i] = INF;
}
best_action_[t][0] = 0;
}
best_score_[0][0] = 0.0;
// The main loop for each cell.
for (int t = 0; t < n + 1; ++t) {
for (int s = max(0, t - width_limit_ + 1); s < t; ++s) {
if (pruned(s, t - 1)) {
continue;
}
for (int i = 1; i < min(t + 1, m + 1); ++i) {
// Find best way to transition to time t at state i.
double local_score = score(s, t - 1, i - 1);
double trial = best_score_[s][i - 1] + local_score;
if (trial < best_score_[t][i]) {
best_score_[t][i] = trial;
best_action_[t][i] = s;
}
}
}
}
assert(best_score_[n][m] < INF);
// Forward is done running.
forward_ = true;
clock_t end = clock();
cerr << "Running semi-markov forward " << n << " " << m << " " << end - start << endl;
}
void SemiMarkov::ViterbiBackward() {
assert(forward_);
clock_t start = clock();
int n = num_timesteps_;
int m = num_states_;
// Score the last round correctly.
for (int t = 0; t < n + 1; ++t) {
for (int i = 0; i < m + 1; ++i) {
best_back_action_[t][i] = -1;
best_back_score_[t][i] = INF;
}
}
best_back_score_[n][m] = 0.0;
// The main loop for each cell. t is current time, s is previous time,
// i is current state.
for (int t = n; t >= 0; --t) {
for (int s = max(0, t - width_limit_ + 1); s < t; ++s) {
if (pruned(s, t - 1)) continue;
for (int i = m; i > 0; --i) {
// Find best way to transition to time t at state i.
double local_score = score(s, t - 1, i - 1);
double trial = best_back_score_[t][i] + local_score;
if (trial < best_back_score_[s][i - 1]) {
best_back_score_[s][i - 1] = trial;
best_back_action_[s][i - 1] = t;
}
}
}
}
// Backward is done running.
backward_ = true;
clock_t end = clock();
cerr << "Running semi-markov backward " << n << " " << m << " " << end - start <<endl;
}
std::pair< size_t, size_t > WHtreeProcesser::pruneRandom( const size_t numberPruned, unsigned int seed )
{
if( numberPruned >= m_tree.getNumLeaves() )
{
throw std::runtime_error(
"ERROR @ WHtreeProcesser::pruneRandom(): too many seeds to be pruned! (more than leaves in the tree)" );
}
std::vector< size_t > prunedIDs( m_tree.getNumLeaves(), 0 );
for( size_t i = 0; i < prunedIDs.size(); ++i )
{
prunedIDs[i] = i;
}
size_t prunedLeaves( 0 );
while( prunedLeaves < numberPruned )
{
size_t prunedPosition = ( rand_r( &seed ) % prunedIDs.size() );
if( m_tree.fetchLeaf( prunedIDs[prunedPosition] )->isFlagged() )
{
continue;
}
m_tree.fetchLeaf( prunedIDs[prunedPosition] )->setFlag( true );
++prunedLeaves;
prunedIDs.erase( prunedIDs.begin() + prunedPosition );
}
unsigned int perthousand( numberPruned * 1000. / m_tree.getNumLeaves() );
float perone( perthousand / 1000. );
m_tree.m_treeName += ( "_randpruned" + string_utils::toString( perone ) );
std::pair< size_t, size_t > pruned( m_tree.cleanup() );
pruned.second += m_tree.debinarize();
return pruned;
} // end "pruneRandom()" -----------------------------------------------------------------
void LogStatistics::format(char **buf, uid_t uid, unsigned int logMask) {
static const unsigned short spaces_total = 19;
if (*buf) {
free(*buf);
*buf = NULL;
}
// Report on total logging, current and for all time
android::String8 output("size/num");
size_t oldLength;
short spaces = 1;
log_id_for_each(id) {
if (!(logMask & (1 << id))) {
continue;
}
oldLength = output.length();
if (spaces < 0) {
spaces = 0;
}
output.appendFormat("%*s%s", spaces, "", android_log_id_to_name(id));
spaces += spaces_total + oldLength - output.length();
}
spaces = 4;
output.appendFormat("\nTotal");
log_id_for_each(id) {
if (!(logMask & (1 << id))) {
continue;
}
oldLength = output.length();
if (spaces < 0) {
spaces = 0;
}
output.appendFormat("%*s%zu/%zu", spaces, "",
sizesTotal(id), elementsTotal(id));
spaces += spaces_total + oldLength - output.length();
}
spaces = 6;
output.appendFormat("\nNow");
log_id_for_each(id) {
if (!(logMask & (1 << id))) {
continue;
}
size_t els = elements(id);
if (els) {
oldLength = output.length();
if (spaces < 0) {
spaces = 0;
}
output.appendFormat("%*s%zu/%zu", spaces, "", sizes(id), els);
spaces -= output.length() - oldLength;
}
spaces += spaces_total;
}
// Report on Chattiest
// Chattiest by application (UID)
static const size_t maximum_sorted_entries = 32;
log_id_for_each(id) {
if (!(logMask & (1 << id))) {
continue;
}
bool headerPrinted = false;
std::unique_ptr<const UidEntry *[]> sorted = sort(maximum_sorted_entries, id);
ssize_t index = -1;
while ((index = uidTable_t::next(index, sorted, maximum_sorted_entries)) >= 0) {
const UidEntry *entry = sorted[index];
uid_t u = entry->getKey();
if ((uid != AID_ROOT) && (u != uid)) {
continue;
}
if (!headerPrinted) {
if (uid == AID_ROOT) {
output.appendFormat(
"\n\nChattiest UIDs in %s:\n",
android_log_id_to_name(id));
} else {
output.appendFormat(
"\n\nLogging for your UID in %s:\n",
android_log_id_to_name(id));
}
android::String8 name("UID");
android::String8 size("Size");
android::String8 pruned("Pruned");
if (!worstUidEnabledForLogid(id)) {
pruned.setTo("");
}
format_line(output, name, size, pruned);
headerPrinted = true;
}
android::String8 name("");
name.appendFormat("%u", u);
char *n = uidToName(u);
if (n) {
name.appendFormat("%*s%s", (int)std::max(6 - name.length(), (size_t)1), "", n);
free(n);
}
android::String8 size("");
size.appendFormat("%zu", entry->getSizes());
android::String8 pruned("");
size_t dropped = entry->getDropped();
if (dropped) {
pruned.appendFormat("%zu", dropped);
}
format_line(output, name, size, pruned);
}
}
if (enable) {
bool headerPrinted = false;
std::unique_ptr<const PidEntry *[]> sorted = pidTable.sort(maximum_sorted_entries);
ssize_t index = -1;
while ((index = pidTable.next(index, sorted, maximum_sorted_entries)) >= 0) {
const PidEntry *entry = sorted[index];
uid_t u = entry->getUid();
if ((uid != AID_ROOT) && (u != uid)) {
continue;
}
if (!headerPrinted) {
if (uid == AID_ROOT) {
output.appendFormat("\n\nChattiest PIDs:\n");
} else {
output.appendFormat("\n\nLogging for this PID:\n");
}
android::String8 name(" PID/UID");
android::String8 size("Size");
android::String8 pruned("Pruned");
format_line(output, name, size, pruned);
headerPrinted = true;
}
android::String8 name("");
name.appendFormat("%5u/%u", entry->getKey(), u);
const char *n = entry->getName();
if (n) {
name.appendFormat("%*s%s", (int)std::max(12 - name.length(), (size_t)1), "", n);
} else {
char *un = uidToName(u);
if (un) {
name.appendFormat("%*s%s", (int)std::max(12 - name.length(), (size_t)1), "", un);
free(un);
}
}
android::String8 size("");
size.appendFormat("%zu", entry->getSizes());
android::String8 pruned("");
size_t dropped = entry->getDropped();
if (dropped) {
pruned.appendFormat("%zu", dropped);
}
format_line(output, name, size, pruned);
}
}
*buf = strdup(output.string());
}
void LogStatistics::format(char **buf, uid_t uid, unsigned int logMask) {
static const unsigned short spaces_total = 19;
if (*buf) {
free(*buf);
*buf = NULL;
}
// Report on total logging, current and for all time
android::String8 output("size/num");
size_t oldLength;
short spaces = 1;
log_id_for_each(id) {
if (!(logMask & (1 << id))) {
continue;
}
oldLength = output.length();
if (spaces < 0) {
spaces = 0;
}
output.appendFormat("%*s%s", spaces, "", android_log_id_to_name(id));
spaces += spaces_total + oldLength - output.length();
}
spaces = 4;
output.appendFormat("\nTotal");
log_id_for_each(id) {
if (!(logMask & (1 << id))) {
continue;
}
oldLength = output.length();
if (spaces < 0) {
spaces = 0;
}
output.appendFormat("%*s%zu/%zu", spaces, "",
sizesTotal(id), elementsTotal(id));
spaces += spaces_total + oldLength - output.length();
}
spaces = 6;
output.appendFormat("\nNow");
log_id_for_each(id) {
if (!(logMask & (1 << id))) {
continue;
}
size_t els = elements(id);
if (els) {
oldLength = output.length();
if (spaces < 0) {
spaces = 0;
}
output.appendFormat("%*s%zu/%zu", spaces, "", sizes(id), els);
spaces -= output.length() - oldLength;
}
spaces += spaces_total;
}
// Report on Chattiest
// Chattiest by application (UID)
static const size_t maximum_sorted_entries = 32;
log_id_for_each(id) {
if (!(logMask & (1 << id))) {
continue;
}
bool headerPrinted = false;
std::unique_ptr<const UidEntry *[]> sorted = sort(maximum_sorted_entries, id);
ssize_t index = -1;
while ((index = uidTable_t::next(index, sorted, maximum_sorted_entries)) >= 0) {
const UidEntry *entry = sorted[index];
uid_t u = entry->getKey();
if ((uid != AID_ROOT) && (u != uid)) {
continue;
}
if (!headerPrinted) {
output.appendFormat("\n\n");
android::String8 name("");
if (uid == AID_ROOT) {
name.appendFormat(
"Chattiest UIDs in %s log buffer:",
android_log_id_to_name(id));
} else {
name.appendFormat(
"Logging for your UID in %s log buffer:",
android_log_id_to_name(id));
}
android::String8 size("Size");
android::String8 pruned("Pruned");
if (!worstUidEnabledForLogid(id)) {
pruned.setTo("");
}
format_line(output, name, size, pruned);
name.setTo("UID PACKAGE");
size.setTo("BYTES");
pruned.setTo("LINES");
if (!worstUidEnabledForLogid(id)) {
pruned.setTo("");
}
format_line(output, name, size, pruned);
headerPrinted = true;
}
android::String8 name("");
name.appendFormat("%u", u);
char *n = uidToName(u);
if (n) {
name.appendFormat("%*s%s", (int)std::max(6 - name.length(), (size_t)1), "", n);
free(n);
}
android::String8 size("");
size.appendFormat("%zu", entry->getSizes());
android::String8 pruned("");
size_t dropped = entry->getDropped();
if (dropped) {
pruned.appendFormat("%zu", dropped);
}
format_line(output, name, size, pruned);
}
}
if (enable) {
// Pid table
bool headerPrinted = false;
std::unique_ptr<const PidEntry *[]> sorted = pidTable.sort(maximum_sorted_entries);
ssize_t index = -1;
while ((index = pidTable.next(index, sorted, maximum_sorted_entries)) >= 0) {
const PidEntry *entry = sorted[index];
uid_t u = entry->getUid();
if ((uid != AID_ROOT) && (u != uid)) {
continue;
}
if (!headerPrinted) {
output.appendFormat("\n\n");
android::String8 name("");
if (uid == AID_ROOT) {
name.appendFormat("Chattiest PIDs:");
} else {
name.appendFormat("Logging for this PID:");
}
android::String8 size("Size");
android::String8 pruned("Pruned");
format_line(output, name, size, pruned);
name.setTo(" PID/UID COMMAND LINE");
size.setTo("BYTES");
pruned.setTo("LINES");
format_line(output, name, size, pruned);
headerPrinted = true;
}
android::String8 name("");
name.appendFormat("%5u/%u", entry->getKey(), u);
const char *n = entry->getName();
if (n) {
name.appendFormat("%*s%s", (int)std::max(12 - name.length(), (size_t)1), "", n);
} else {
char *un = uidToName(u);
if (un) {
name.appendFormat("%*s%s", (int)std::max(12 - name.length(), (size_t)1), "", un);
free(un);
}
}
android::String8 size("");
size.appendFormat("%zu", entry->getSizes());
android::String8 pruned("");
size_t dropped = entry->getDropped();
if (dropped) {
pruned.appendFormat("%zu", dropped);
}
format_line(output, name, size, pruned);
}
}
if (enable) {
// Tid table
bool headerPrinted = false;
// sort() returns list of references, unique_ptr makes sure self-delete
std::unique_ptr<const TidEntry *[]> sorted = tidTable.sort(maximum_sorted_entries);
ssize_t index = -1;
while ((index = tidTable.next(index, sorted, maximum_sorted_entries)) >= 0) {
const TidEntry *entry = sorted[index];
uid_t u = entry->getUid();
if ((uid != AID_ROOT) && (u != uid)) {
continue;
}
if (!headerPrinted) { // Only print header if we have table to print
output.appendFormat("\n\n");
android::String8 name("Chattiest TIDs:");
android::String8 size("Size");
android::String8 pruned("Pruned");
format_line(output, name, size, pruned);
name.setTo(" TID/UID COMM");
size.setTo("BYTES");
pruned.setTo("LINES");
format_line(output, name, size, pruned);
headerPrinted = true;
}
android::String8 name("");
name.appendFormat("%5u/%u", entry->getKey(), u);
const char *n = entry->getName();
if (n) {
name.appendFormat("%*s%s", (int)std::max(12 - name.length(), (size_t)1), "", n);
} else {
// if we do not have a PID name, lets punt to try UID name?
char *un = uidToName(u);
if (un) {
name.appendFormat("%*s%s", (int)std::max(12 - name.length(), (size_t)1), "", un);
free(un);
}
// We tried, better to not have a name at all, we still
// have TID/UID by number to report in any case.
}
android::String8 size("");
size.appendFormat("%zu", entry->getSizes());
android::String8 pruned("");
size_t dropped = entry->getDropped();
if (dropped) {
pruned.appendFormat("%zu", dropped);
}
format_line(output, name, size, pruned);
}
}
if (enable && (logMask & (1 << LOG_ID_EVENTS))) {
// Tag table
bool headerPrinted = false;
std::unique_ptr<const TagEntry *[]> sorted = tagTable.sort(maximum_sorted_entries);
ssize_t index = -1;
while ((index = tagTable.next(index, sorted, maximum_sorted_entries)) >= 0) {
const TagEntry *entry = sorted[index];
uid_t u = entry->getUid();
if ((uid != AID_ROOT) && (u != uid)) {
continue;
}
android::String8 pruned("");
if (!headerPrinted) {
output.appendFormat("\n\n");
android::String8 name("Chattiest events log buffer TAGs:");
android::String8 size("Size");
format_line(output, name, size, pruned);
name.setTo(" TAG/UID TAGNAME");
size.setTo("BYTES");
format_line(output, name, size, pruned);
headerPrinted = true;
}
android::String8 name("");
if (u == (uid_t)-1) {
name.appendFormat("%7u", entry->getKey());
} else {
name.appendFormat("%7u/%u", entry->getKey(), u);
}
const char *n = entry->getName();
if (n) {
name.appendFormat("%*s%s", (int)std::max(14 - name.length(), (size_t)1), "", n);
}
android::String8 size("");
size.appendFormat("%zu", entry->getSizes());
format_line(output, name, size, pruned);
}
}
*buf = strdup(output.string());
}
std::pair< size_t, size_t > WHtreeProcesser::pruneTree( float condition, size_t safeSize, const HTPROC_MODE pruneType )
{
if( safeSize == 0 )
{
safeSize = m_tree.getNumLeaves(); // any cluster no matter what size may be pruned if he meets the conditions
}
if( pruneType == HTPR_SIZERATIO )
{
if( condition < 2 )
throw std::runtime_error( "ERROR @ WHtreeProcesser::pruneTree(): size pruning ratio must be equal or greater than 2" );
}
else if( pruneType == HTPR_JOINSIZE )
{
condition = std::floor( condition );
if( condition < safeSize )
throw std::runtime_error(
"ERROR @ WHtreeProcesser::pruneTree(): size pruning lower boundary must be smaller than greater boundary" );
}
else if( pruneType == HTPR_JOINLEVEL )
{
if( condition <= 0 || condition >= 1 )
{
throw std::runtime_error( "ERROR @ WHtreeProcesser::pruneTree(): condition is out of boundaries" );
}
if( safeSize >= m_tree.getNumLeaves() )
{
throw std::runtime_error(
"ERROR @ WHtreeProcesser::pruneTree(): when pruning by distance level a safe size smaller than the roi size must be entered" );
}
}
size_t prunedLeaves( 0 ), prunedNodes( 0 );
// loop through all leaves and set them to prune if they match discardidng conditions
for( std::vector< WHnode >::iterator leavesIter( m_tree.m_leaves.begin() ); leavesIter != m_tree.m_leaves.end(); ++leavesIter )
{
size_t parentID( leavesIter->getParent().second );
size_t parentLevel( m_tree.getNode( parentID ).getDistLevel() );
if( ( pruneType == HTPR_JOINLEVEL ) && ( parentLevel > condition ) )
{
leavesIter->setFlag( true );
}
else if( ( pruneType == HTPR_SIZERATIO ) || ( pruneType == HTPR_JOINSIZE ) )
{
size_t biggerSize( 0 );
std::vector< nodeID_t > kids( m_tree.getNode( parentID ).getChildren() );
for( size_t i = 0; i < kids.size(); ++i )
{
size_t brotherSize( m_tree.getNode( kids[i] ).getSize() );
if( brotherSize > biggerSize )
{
biggerSize = brotherSize;
}
}
if( biggerSize > condition )
{
leavesIter->setFlag( true );
}
}
}
// loop through all nodes and set them to prune if they match discarding conditions
for( std::vector< WHnode >::iterator nodesIter( m_tree.m_nodes.begin() ); nodesIter != m_tree.m_nodes.end() - 1; ++nodesIter )
{ // dont check last node
size_t parentID( nodesIter->getParent().second );
size_t nodeSize( nodesIter->getSize() );
size_t parentLevel( m_tree.getNode( parentID ).getDistLevel() );
bool pruneBranch( false );
if( nodeSize < safeSize )
{
if( ( pruneType == HTPR_JOINLEVEL ) && ( parentLevel > condition ) )
{
pruneBranch = true;
}
else if( ( pruneType == HTPR_SIZERATIO ) || ( pruneType == HTPR_JOINSIZE ) )
{
size_t biggerSize( 0 );
std::vector< nodeID_t > kids( m_tree.getNode( parentID ).getChildren() );
for( size_t i = 0; i < kids.size(); ++i )
{
if( kids[i] == nodesIter->getFullID() )
{
continue;
}
size_t brotherSize( m_tree.getNode( kids[i] ).getSize() );
if( brotherSize > biggerSize )
{
biggerSize = brotherSize;
}
}
if( ( pruneType == HTPR_SIZERATIO ) && ( biggerSize > ( nodeSize * condition ) ) )
{
pruneBranch = true;
}
if( ( pruneType == HTPR_JOINSIZE ) && ( biggerSize >= condition ) )
{
pruneBranch = true;
}
}
}
if( pruneBranch )
{
std::list< nodeID_t > worklist;
worklist.push_back( nodesIter->getFullID() );
while( !worklist.empty() )
{
WHnode* currentNode( m_tree.fetchNode( worklist.front() ) );
worklist.pop_front();
// if current node has already been pruned we continue with the next iteration
if( currentNode->isFlagged() )
{
continue;
}
currentNode->setFlag( true );
std::vector< nodeID_t > currentKids( currentNode->getChildren() );
worklist.insert( worklist.end(), currentKids.begin(), currentKids.end() );
}
}
}
// count total pruned leaves
for( std::vector< WHnode >::iterator leavesIter( m_tree.m_leaves.begin() ); leavesIter != m_tree.m_leaves.end(); ++leavesIter )
{
if( leavesIter->isFlagged() )
{
++prunedLeaves;
}
}
// count total pruned nodes
for( std::vector< WHnode >::iterator nodesIter( m_tree.m_nodes.begin() ); nodesIter != m_tree.m_nodes.end() - 1; ++nodesIter )
{
if( nodesIter->isFlagged() )
++prunedNodes;
}
if( pruneType == HTPR_SIZERATIO )
{
m_tree.m_treeName += ( "_prunedR" + string_utils::toString( safeSize ) + ":" + string_utils::toString( condition ) );
}
else if( pruneType == HTPR_JOINSIZE )
{
m_tree.m_treeName += ( "_prunedS" + string_utils::toString( condition ) + ":" + string_utils::toString( safeSize ) );
}
else if( pruneType == HTPR_JOINLEVEL )
{
m_tree.m_treeName += ( "_prunedL" + string_utils::toString( safeSize ) + ":" + string_utils::toString( condition ) );
}
std::pair< size_t, size_t > pruned( m_tree.cleanup() );
pruned.second += m_tree.debinarize();
return pruned;
} // end "pruneTree()" -----------------------------------------------------------------
std::pair< size_t, size_t > WHtreeProcesser::pruneSelection( const std::vector< nodeID_t > &selection )
{
flagSelection( selection );
std::pair< size_t, size_t > pruned( m_tree.cleanup() );
return pruned;
} // end "pruneSelection()" -----------------------------------------------------------------
