void ReportManager::assignDkeys(const RoseBuild *rose) { unique_ptr<RoseDedupeAux> dedupe = rose->generateDedupeAux(); DEBUG_PRINTF("assigning...\n"); map<u32, set<ReportID>> ext_to_int; for (u32 i = 0; i < reportIds.size(); i++) { const Report &ir = reportIds[i]; /* need to populate dkey */ if (isExternalReport(ir)) { ext_to_int[ir.onmatch].insert(i); } } for (const auto &m : ext_to_int) { u32 ext = m.first; if (!dedupe->requiresDedupeSupport(m.second)) { DEBUG_PRINTF("%u does not require dedupe\n", ext); continue; /* no dedupe required for this set */ } u32 dkey = reportIdToDedupeKey.size(); reportIdToDedupeKey[ext] = dkey; DEBUG_PRINTF("ext=%u -> dkey=%u\n", ext, dkey); } }
u32 ReportManager::getDkey(const Report &r) const { if (!isExternalReport(r)) { return ~u32{0}; } auto it = reportIdToDedupeKey.find(r.onmatch); if (it == reportIdToDedupeKey.end()) { return ~u32{0}; } return it->second; }
/** Remove any edges from vertices that generate accepts (for Highlander * graphs). */ void pruneHighlanderAccepts(NGHolder &g, const ReportManager &rm) { // Safety check: all reports must be simple exhaustible reports, or this is // not safe. This optimisation should be called early enough that no // internal reports have been added. for (auto report_id : all_reports(g)) { const Report &ir = rm.getReport(report_id); if (ir.ekey == INVALID_EKEY || ir.hasBounds() || !isExternalReport(ir)) { DEBUG_PRINTF("report %u is not external highlander with " "no bounds\n", report_id); return; } } vector<NFAEdge> dead; for (auto u : inv_adjacent_vertices_range(g.accept, g)) { if (is_special(u, g)) { continue; } // We can prune any out-edges that aren't accepts for (const auto &e : out_edges_range(u, g)) { if (!is_any_accept(target(e, g), g)) { dead.push_back(e); } } } if (dead.empty()) { return; } DEBUG_PRINTF("found %zu removable edges due to single match\n", dead.size()); remove_edges(dead, g); pruneUseless(g); }
static really_inline int roseAdaptor_i(u64a offset, ReportID id, void *context, char is_simple, char do_som) { assert(id != MO_INVALID_IDX); // Should never get an invalid ID. struct hs_scratch *scratch = (struct hs_scratch *)context; struct core_info *ci = &scratch->core_info; const struct RoseEngine *rose = ci->rose; DEBUG_PRINTF("internal report %u\n", id); const struct internal_report *ri = getInternalReport(rose, id); assert(isExternalReport(ri)); /* only external reports should reach here */ s32 offset_adj = ri->offsetAdjust; UNUSED u32 dkey = ri->dkey; u64a to_offset = offset; u64a from_offset = 0; UNUSED u32 dkeyCount = rose->dkeyCount; u32 flags = 0; #ifndef RELEASE_BUILD if (offset_adj) { // alert testing tools that we've got adjusted matches flags |= HS_MATCH_FLAG_ADJUSTED; } #endif DEBUG_PRINTF("internal match at %llu: IID=%u type=%hhu RID=%u " "offsetAdj=%d\n", offset, id, ri->type, ri->onmatch, offset_adj); if (unlikely(can_stop_matching(scratch))) { /* ok - we are from rose */ DEBUG_PRINTF("pre broken - halting\n"); return MO_HALT_MATCHING; } if (!is_simple && ri->hasBounds) { assert(ri->minOffset || ri->minLength || ri->maxOffset < MAX_OFFSET); assert(ri->minOffset <= ri->maxOffset); if (offset < ri->minOffset || offset > ri->maxOffset) { DEBUG_PRINTF("match fell outside valid range %llu !: [%llu,%llu]\n", offset, ri->minOffset, ri->maxOffset); return ROSE_CONTINUE_MATCHING_NO_EXHAUST; } } if (!is_simple && unlikely(isExhausted(ci->exhaustionVector, ri->ekey))) { DEBUG_PRINTF("ate exhausted match\n"); return MO_CONTINUE_MATCHING; } if (ri->type == EXTERNAL_CALLBACK) { from_offset = 0; } else if (do_som) { from_offset = handleSomExternal(scratch, ri, to_offset); } to_offset += offset_adj; assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset); if (do_som && ri->minLength) { if (from_offset != HS_OFFSET_PAST_HORIZON && (to_offset - from_offset < ri->minLength)) { return ROSE_CONTINUE_MATCHING_NO_EXHAUST; } if (ri->quashSom) { from_offset = 0; } } DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n", from_offset, to_offset, ri->onmatch, ci->userContext); int halt = 0; if (do_som || dkey != MO_INVALID_IDX) { if (offset != scratch->deduper.current_report_offset) { assert(scratch->deduper.current_report_offset == ~0ULL || scratch->deduper.current_report_offset < offset); if (offset == scratch->deduper.current_report_offset + 1) { fatbit_clear(scratch->deduper.log[offset % 2]); } else { fatbit_clear(scratch->deduper.log[0]); fatbit_clear(scratch->deduper.log[1]); } DEBUG_PRINTF("adj dedupe offset %hhd\n", do_som); if (do_som) { halt = flushStoredSomMatches(scratch, offset); if (halt) { goto exit; } } scratch->deduper.current_report_offset = offset; } } #ifdef DEDUPE_MATCHES if (dkey != MO_INVALID_IDX) { if (ri->type == EXTERNAL_CALLBACK || ri->quashSom) { DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset); assert(offset_adj == 0 || offset_adj == -1); if (fatbit_set(scratch->deduper.log[to_offset % 2], dkeyCount, dkey)) { /* we have already raised this report at this offset, squash dupe * match. */ DEBUG_PRINTF("dedupe\n"); goto exit; } } else if (do_som) { /* SOM external event */ DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset); assert(offset_adj == 0 || offset_adj == -1); u64a *starts = scratch->deduper.som_start_log[to_offset % 2]; if (fatbit_set(scratch->deduper.som_log[to_offset % 2], dkeyCount, dkey)) { starts[dkey] = MIN(starts[dkey], from_offset); } else { starts[dkey] = from_offset; } if (offset_adj) { scratch->deduper.som_log_dirty |= 1; } else { scratch->deduper.som_log_dirty |= 2; } goto exit; } } #endif halt = ci->userCallback((unsigned int)ri->onmatch, from_offset, to_offset, flags, ci->userContext); #ifdef DEDUPE_MATCHES exit: #endif if (halt) { DEBUG_PRINTF("callback requested to terminate matches\n"); setBroken(ci->state, BROKEN_FROM_USER); ci->broken = BROKEN_FROM_USER; return MO_HALT_MATCHING; } if (!is_simple && ri->ekey != END_EXHAUST) { markAsMatched(ci->exhaustionVector, ri->ekey); return MO_CONTINUE_MATCHING; } else { return ROSE_CONTINUE_MATCHING_NO_EXHAUST; } }