MetadataNode InfoKernel::dumpQuery(PointViewPtr inView) const { int count; std::string location; // See if there's a provided point count. StringList parts = Utils::split2(m_queryPoint, '/'); if (parts.size() == 2) { location = parts[0]; count = atoi(parts[1].c_str()); } else if (parts.size() == 1) { location = parts[0]; count = inView->size(); } else count = 0; if (count == 0) throw pdal_error("Invalid location specificiation. " "--query=\"X,Y[/count]\""); auto seps = [](char c){ return (c == ',' || c == '|' || c == ' '); }; std::vector<std::string> tokens = Utils::split2(location, seps); std::vector<double> values; for (auto ti = tokens.begin(); ti != tokens.end(); ++ti) { double d; if (Utils::fromString(*ti, d)) values.push_back(d); } if (values.size() != 2 && values.size() != 3) throw pdal_error("--points must be two or three values"); PointViewPtr outView = inView->makeNew(); std::vector<PointId> ids; if (values.size() >= 3) { KD3Index kdi(*inView); kdi.build(); ids = kdi.neighbors(values[0], values[1], values[2], count); } else { KD2Index kdi(*inView); kdi.build(); ids = kdi.neighbors(values[0], values[1], count); } for (auto i = ids.begin(); i != ids.end(); ++i) outView->appendPoint(*inView.get(), *i); return outView->toMetadata(); }
void HAGFilter::filter(PointView& view) { PointViewPtr gView = view.makeNew(); PointViewPtr ngView = view.makeNew(); std::vector<PointId> gIdx, ngIdx; // First pass: Separate into ground and non-ground views. for (PointId i = 0; i < view.size(); ++i) { double c = view.getFieldAs<double>(Dimension::Id::Classification, i); if (c == 2) { gView->appendPoint(view, i); gIdx.push_back(i); } else { ngView->appendPoint(view, i); ngIdx.push_back(i); } } // Bail if there weren't any points classified as ground. if (gView->size() == 0) throwError("Input PointView does not have any points classified " "as ground"); // Build the 2D KD-tree. KD2Index kdi(*gView); kdi.build(); // Second pass: Find Z difference between non-ground points and the nearest // neighbor (2D) in the ground view. for (PointId i = 0; i < ngView->size(); ++i) { PointRef point = ngView->point(i); double z0 = point.getFieldAs<double>(Dimension::Id::Z); auto ids = kdi.neighbors(point, 1); double z1 = gView->getFieldAs<double>(Dimension::Id::Z, ids[0]); view.setField(Dimension::Id::HeightAboveGround, ngIdx[i], z0 - z1); } // Final pass: Ensure that all ground points have height value pegged at 0. for (auto const& i : gIdx) view.setField(Dimension::Id::HeightAboveGround, i, 0.0); }
/* traverse page by page */ void findReadOnlyPages(Mem * mem) { int i; int pageSize = 4 * 1024; //4k int totalPageNumber = mem->mem_size / (4 * 1024); //assume that every page has 4k int calledPages[totalPageNumber]; int dsmPages[totalPageNumber]; //record virtual address unsigned virtualAddrs[totalPageNumber]; for (i = 0; i < totalPageNumber; i++) { calledPages[i] = 0; dsmPages[i] = 0; virtualAddrs[i] = 0; } unsigned cr3Pages[100]; for (i = 0; i < 100; i++) { cr3Pages[i] = 0; } //start address unsigned startVirtualAddr = 0x80000000; //step 1. kdi int kdi_time =0; int allPages=0; int cluster_index= kdi(startVirtualAddr, mem,pageSize,cr3Pages, &kdi_time, &allPages); //step 2. signature generation struct timeval earlier; struct timeval later; if (gettimeofday(&earlier, NULL)) { perror("gettimeofday() error"); exit(1); } int cr3ClusterNo = 0; int cr3PageNo = 0; ranges[0].end = 0; //find the cluster which has max cr3 number int maxcr3Index = findClusterHasMaxCr3(cluster_index, clusters, cr3Pages, &cr3ClusterNo); if (maxcr3Index == -1) { puts("Cannot find clusters have cr3."); // return ; } unsigned codePageNo = 0; newstart = 1; unsigned vAddr; for (vAddr = clusters[maxcr3Index].start; vAddr < clusters[maxcr3Index].end; vAddr += 0x1000) { cr3PageNo++; unsigned pAddr = vtop(mem->mem, mem->mem_size, mem->pgd, vAddr); if (vAddr == out_pc) code_init(mem, vAddr, pageSize, dsmPages, virtualAddrs, 1, calledPages, &codePageNo); else code_init(mem, vAddr, pageSize, dsmPages, virtualAddrs, 0, calledPages, &codePageNo); } ranges[range_index].end = clusters[maxcr3Index].end; ranges[range_index].len = ranges[range_index].end - ranges[range_index].start; //find the max range int max_len = 0, max_index = 0; for (i = 1; i <= range_index; i++) { if (containKernelAddresForRange(ranges[i], cr3Pages) != 0) { continue; } printf("start:%x, end:%x: len:%x kernel\n", ranges[i].start, ranges[i].end, ranges[i].len); if (ranges[i].len > max_len) { max_index = i; max_len = ranges[i].len; } } unsigned pAddr = vtop(mem->mem, mem->mem_size, mem->pgd, ranges[max_index].start); int pageIndex = pAddr / pageSize; char *page = (char*) ((unsigned) mem->mem + pAddr); code_preprocess(mem, page, ranges[max_index].len, 0x1000, ranges + max_index, dsmPages + pageIndex); printf("step2: cluster: %d\n", range_index); //print md5 of pages that can be disassembled startVirtualAddr = ranges[max_index].start; unsigned disasPageNo = 0; unsigned totalPageNo = 0; for (; startVirtualAddr <= ranges[max_index].end; startVirtualAddr += 0x1000) { totalPageNo++; unsigned pAddr = vtop(mem->mem, mem->mem_size, mem->pgd, startVirtualAddr); if (pAddr == -1 || pAddr > mem->mem_size) continue; int pageIndex = pAddr / pageSize; if (dsmPages[pageIndex] == 1) { unsigned offset = startVirtualAddr - ranges[max_index].start; void *startAdress = (void*) ((unsigned) mem->mem + pageIndex * pageSize); genMd5WithOffset(startAdress, pageSize, startVirtualAddr, offset); disasPageNo++; } } if (gettimeofday(&later, NULL)) { perror("gettimeofday() error"); exit(1); } int sigGen_time = timeval_diff(NULL, &later, &earlier) / 1000; printf("step2: time cost is %d milliseconds\n", sigGen_time); float byerate = (float) ranges[max_index].disasBytes / (4096 * disasPageNo); printf("Success pages: %u/%u disassembled bytes rate: %f, page rate: %f\n", disasPageNo, totalPageNo, (float) ranges[max_index].disasBytes / (4096 * disasPageNo), (float) disasPageNo / totalPageNo); //record data; recordData(allPages, cluster_index, cr3ClusterNo, cr3PageNo, totalPageNo, disasPageNo, byerate); //begin match int match_time = 0; sigMatch(ranges[max_index], mem, pageSize, dsmPages, &match_time); //record performance recordPerformance(kdi_time, sigGen_time, match_time); return; }
std::vector<std::vector<PointId>> extractClusters(PointView& view, uint64_t min_points, uint64_t max_points, double tolerance) { // Index the incoming PointView for subsequent radius searches. KD3Index kdi(view); kdi.build(); // Create variables to track PointIds that have already been added to // clusters and to build the list of cluster indices. std::vector<PointId> processed(view.size(), 0); std::vector<std::vector<PointId>> clusters; for (PointId i = 0; i < view.size(); ++i) { // Points can only belong to a single cluster. if (processed[i]) continue; // Initialize list of indices belonging to current cluster, marking the // seed point as processed. std::vector<PointId> seed_queue; size_t sq_idx = 0; seed_queue.push_back(i); processed[i] = 1; // Check each point in the cluster for additional neighbors within the // given tolerance, remembering that the list can grow if we add points // to the cluster. while (sq_idx < seed_queue.size()) { // Find neighbors of the next cluster point. PointId j = seed_queue[sq_idx]; std::vector<PointId> ids = kdi.radius(j, tolerance); // The case where the only neighbor is the query point. if (ids.size() == 1) { sq_idx++; continue; } // Skip neighbors that already belong to a cluster and add the rest // to this cluster. for (auto const& k : ids) { if (processed[k]) continue; seed_queue.push_back(k); processed[k] = 1; } sq_idx++; } // Keep clusters that are within the min/max number of points. if (seed_queue.size() >= min_points && seed_queue.size() <= max_points) clusters.push_back(seed_queue); } return clusters; }