MetadataNode InfoKernel::dumpQuery(PointViewPtr inView) const
{
int count;
std::string location;
// See if there's a provided point count.
StringList parts = Utils::split2(m_queryPoint, '/');
if (parts.size() == 2)
{
location = parts[0];
count = atoi(parts[1].c_str());
}
else if (parts.size() == 1)
{
location = parts[0];
count = inView->size();
}
else
count = 0;
if (count == 0)
throw pdal_error("Invalid location specificiation. "
"--query=\"X,Y[/count]\"");
auto seps = [](char c){ return (c == ',' || c == '|' || c == ' '); };
std::vector<std::string> tokens = Utils::split2(location, seps);
std::vector<double> values;
for (auto ti = tokens.begin(); ti != tokens.end(); ++ti)
{
double d;
if (Utils::fromString(*ti, d))
values.push_back(d);
}
if (values.size() != 2 && values.size() != 3)
throw pdal_error("--points must be two or three values");
PointViewPtr outView = inView->makeNew();
std::vector<PointId> ids;
if (values.size() >= 3)
{
KD3Index kdi(*inView);
kdi.build();
ids = kdi.neighbors(values[0], values[1], values[2], count);
}
else
{
KD2Index kdi(*inView);
kdi.build();
ids = kdi.neighbors(values[0], values[1], count);
}
for (auto i = ids.begin(); i != ids.end(); ++i)
outView->appendPoint(*inView.get(), *i);
return outView->toMetadata();
}
void HAGFilter::filter(PointView& view)
{
PointViewPtr gView = view.makeNew();
PointViewPtr ngView = view.makeNew();
std::vector<PointId> gIdx, ngIdx;
// First pass: Separate into ground and non-ground views.
for (PointId i = 0; i < view.size(); ++i)
{
double c = view.getFieldAs<double>(Dimension::Id::Classification, i);
if (c == 2)
{
gView->appendPoint(view, i);
gIdx.push_back(i);
}
else
{
ngView->appendPoint(view, i);
ngIdx.push_back(i);
}
}
// Bail if there weren't any points classified as ground.
if (gView->size() == 0)
throwError("Input PointView does not have any points classified "
"as ground");
// Build the 2D KD-tree.
KD2Index kdi(*gView);
kdi.build();
// Second pass: Find Z difference between non-ground points and the nearest
// neighbor (2D) in the ground view.
for (PointId i = 0; i < ngView->size(); ++i)
{
PointRef point = ngView->point(i);
double z0 = point.getFieldAs<double>(Dimension::Id::Z);
auto ids = kdi.neighbors(point, 1);
double z1 = gView->getFieldAs<double>(Dimension::Id::Z, ids[0]);
view.setField(Dimension::Id::HeightAboveGround, ngIdx[i], z0 - z1);
}
// Final pass: Ensure that all ground points have height value pegged at 0.
for (auto const& i : gIdx)
view.setField(Dimension::Id::HeightAboveGround, i, 0.0);
}
/* traverse page by page */
void findReadOnlyPages(Mem * mem) {
int i;
int pageSize = 4 * 1024; //4k
int totalPageNumber = mem->mem_size / (4 * 1024); //assume that every page has 4k
int calledPages[totalPageNumber];
int dsmPages[totalPageNumber];
//record virtual address
unsigned virtualAddrs[totalPageNumber];
for (i = 0; i < totalPageNumber; i++) {
calledPages[i] = 0;
dsmPages[i] = 0;
virtualAddrs[i] = 0;
}
unsigned cr3Pages[100];
for (i = 0; i < 100; i++) {
cr3Pages[i] = 0;
}
//start address
unsigned startVirtualAddr = 0x80000000;
//step 1. kdi
int kdi_time =0;
int allPages=0;
int cluster_index= kdi(startVirtualAddr, mem,pageSize,cr3Pages, &kdi_time, &allPages);
//step 2. signature generation
struct timeval earlier;
struct timeval later;
if (gettimeofday(&earlier, NULL)) {
perror("gettimeofday() error");
exit(1);
}
int cr3ClusterNo = 0;
int cr3PageNo = 0;
ranges[0].end = 0;
//find the cluster which has max cr3 number
int maxcr3Index = findClusterHasMaxCr3(cluster_index, clusters, cr3Pages, &cr3ClusterNo);
if (maxcr3Index == -1) {
puts("Cannot find clusters have cr3.");
// return ;
}
unsigned codePageNo = 0;
newstart = 1;
unsigned vAddr;
for (vAddr = clusters[maxcr3Index].start; vAddr < clusters[maxcr3Index].end; vAddr += 0x1000) {
cr3PageNo++;
unsigned pAddr = vtop(mem->mem, mem->mem_size, mem->pgd, vAddr);
if (vAddr == out_pc)
code_init(mem, vAddr, pageSize, dsmPages, virtualAddrs, 1, calledPages, &codePageNo);
else
code_init(mem, vAddr, pageSize, dsmPages, virtualAddrs, 0, calledPages, &codePageNo);
}
ranges[range_index].end = clusters[maxcr3Index].end;
ranges[range_index].len = ranges[range_index].end - ranges[range_index].start;
//find the max range
int max_len = 0, max_index = 0;
for (i = 1; i <= range_index; i++) {
if (containKernelAddresForRange(ranges[i], cr3Pages) != 0) {
continue;
}
printf("start:%x, end:%x: len:%x kernel\n", ranges[i].start, ranges[i].end, ranges[i].len);
if (ranges[i].len > max_len) {
max_index = i;
max_len = ranges[i].len;
}
}
unsigned pAddr = vtop(mem->mem, mem->mem_size, mem->pgd, ranges[max_index].start);
int pageIndex = pAddr / pageSize;
char *page = (char*) ((unsigned) mem->mem + pAddr);
code_preprocess(mem, page, ranges[max_index].len, 0x1000, ranges + max_index,
dsmPages + pageIndex);
printf("step2: cluster: %d\n", range_index);
//print md5 of pages that can be disassembled
startVirtualAddr = ranges[max_index].start;
unsigned disasPageNo = 0;
unsigned totalPageNo = 0;
for (; startVirtualAddr <= ranges[max_index].end; startVirtualAddr += 0x1000) {
totalPageNo++;
unsigned pAddr = vtop(mem->mem, mem->mem_size, mem->pgd, startVirtualAddr);
if (pAddr == -1 || pAddr > mem->mem_size)
continue;
int pageIndex = pAddr / pageSize;
if (dsmPages[pageIndex] == 1) {
unsigned offset = startVirtualAddr - ranges[max_index].start;
void *startAdress = (void*) ((unsigned) mem->mem + pageIndex * pageSize);
genMd5WithOffset(startAdress, pageSize, startVirtualAddr, offset);
disasPageNo++;
}
}
if (gettimeofday(&later, NULL)) {
perror("gettimeofday() error");
exit(1);
}
int sigGen_time = timeval_diff(NULL, &later, &earlier) / 1000;
printf("step2: time cost is %d milliseconds\n", sigGen_time);
float byerate = (float) ranges[max_index].disasBytes / (4096 * disasPageNo);
printf("Success pages: %u/%u disassembled bytes rate: %f, page rate: %f\n", disasPageNo,
totalPageNo, (float) ranges[max_index].disasBytes / (4096 * disasPageNo),
(float) disasPageNo / totalPageNo);
//record data;
recordData(allPages, cluster_index, cr3ClusterNo, cr3PageNo, totalPageNo, disasPageNo, byerate);
//begin match
int match_time = 0;
sigMatch(ranges[max_index], mem, pageSize, dsmPages, &match_time);
//record performance
recordPerformance(kdi_time, sigGen_time, match_time);
return;
}
std::vector<std::vector<PointId>> extractClusters(PointView& view,
uint64_t min_points,
uint64_t max_points,
double tolerance)
{
// Index the incoming PointView for subsequent radius searches.
KD3Index kdi(view);
kdi.build();
// Create variables to track PointIds that have already been added to
// clusters and to build the list of cluster indices.
std::vector<PointId> processed(view.size(), 0);
std::vector<std::vector<PointId>> clusters;
for (PointId i = 0; i < view.size(); ++i)
{
// Points can only belong to a single cluster.
if (processed[i])
continue;
// Initialize list of indices belonging to current cluster, marking the
// seed point as processed.
std::vector<PointId> seed_queue;
size_t sq_idx = 0;
seed_queue.push_back(i);
processed[i] = 1;
// Check each point in the cluster for additional neighbors within the
// given tolerance, remembering that the list can grow if we add points
// to the cluster.
while (sq_idx < seed_queue.size())
{
// Find neighbors of the next cluster point.
PointId j = seed_queue[sq_idx];
std::vector<PointId> ids = kdi.radius(j, tolerance);
// The case where the only neighbor is the query point.
if (ids.size() == 1)
{
sq_idx++;
continue;
}
// Skip neighbors that already belong to a cluster and add the rest
// to this cluster.
for (auto const& k : ids)
{
if (processed[k])
continue;
seed_queue.push_back(k);
processed[k] = 1;
}
sq_idx++;
}
// Keep clusters that are within the min/max number of points.
if (seed_queue.size() >= min_points && seed_queue.size() <= max_points)
clusters.push_back(seed_queue);
}
return clusters;
}
