void setSomFromSomAware(struct hs_scratch *scratch,
const struct som_operation *ri, u64a from_offset,
u64a to_offset) {
assert(scratch);
assert(ri);
assert(to_offset);
assert(ri->type == SOM_INTERNAL_LOC_SET_FROM
|| ri->type == SOM_INTERNAL_LOC_SET_FROM_IF_WRITABLE);
struct core_info *ci = &scratch->core_info;
const struct RoseEngine *rose = ci->rose;
assert(rose->hasSom);
const u32 som_store_count = rose->somLocationCount;
u8 *som_store_valid = (u8 *)ci->state + rose->stateOffsets.somValid;
u8 *som_store_writable = (u8 *)ci->state + rose->stateOffsets.somWritable;
struct fatbit *som_set_now = scratch->som_set_now;
struct fatbit *som_attempted_set = scratch->som_attempted_set;
u64a *som_store = scratch->som_store;
u64a *som_failed_store = scratch->som_attempted_store;
if (to_offset != scratch->som_set_now_offset) {
DEBUG_PRINTF("setting som_set_now_offset=%llu\n", to_offset);
fatbit_clear(som_set_now);
fatbit_clear(som_attempted_set);
scratch->som_set_now_offset = to_offset;
}
if (ri->type == SOM_INTERNAL_LOC_SET_FROM) {
DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_FROM\n");
mmbit_set(som_store_valid, som_store_count, ri->onmatch);
setSomLoc(som_set_now, som_store, som_store_count, ri, from_offset);
} else {
DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_FROM_IF_WRITABLE\n");
if (ok_and_mark_if_write(som_store_valid, som_set_now,
som_store_writable, som_store_count,
ri->onmatch)) {
setSomLoc(som_set_now, som_store, som_store_count, ri, from_offset);
mmbit_unset(som_store_writable, som_store_count, ri->onmatch);
} else {
/* not writable, stash as an attempted write in case we are
* racing our escape. */
DEBUG_PRINTF("not writable, stashing attempt\n");
assert(to_offset >= ri->aux.somDistance);
u32 som_loc = ri->onmatch;
if (!fatbit_set(som_attempted_set, som_store_count, ri->onmatch)) {
som_failed_store[som_loc] = from_offset;
} else {
LIMIT_TO_AT_MOST(&som_failed_store[som_loc], from_offset);
}
DEBUG_PRINTF("som_failed_store[%u] = %llu\n", som_loc,
som_failed_store[som_loc]);
}
}
}
static really_inline
void setSomLoc(struct fatbit *som_set_now, u64a *som_store, u32 som_store_count,
const struct som_operation *ri, u64a to_offset) {
/* validity handled by callers */
assert(to_offset >= ri->aux.somDistance);
u64a start_offset = to_offset - ri->aux.somDistance;
u32 som_loc = ri->onmatch;
/* resolve any races for matches at this point in favour of the earliest som
*/
if (!fatbit_set(som_set_now, som_store_count, som_loc)) {
som_store[som_loc] = start_offset;
} else {
LIMIT_TO_AT_MOST(&som_store[som_loc], start_offset);
}
DEBUG_PRINTF("som_store[%u] set to %llu\n", som_loc, som_store[som_loc]);
}
static really_inline
void setSomLocRevNfa(struct hs_scratch *scratch, struct fatbit *som_set_now,
u64a *som_store, u32 som_store_count,
const struct som_operation *ri, u64a to_offset) {
/* validity handled by callers */
u64a from_offset = 0;
runRevNfa(scratch, ri, to_offset, &from_offset);
u32 som_loc = ri->onmatch;
/* resolve any races for matches at this point in favour of the earliest som
*/
if (!fatbit_set(som_set_now, som_store_count, som_loc)) {
som_store[som_loc] = from_offset;
} else {
LIMIT_TO_AT_MOST(&som_store[som_loc], from_offset);
}
DEBUG_PRINTF("som_store[%u] set to %llu\n", som_loc, som_store[som_loc]);
}
static rose_inline
void recordAnchoredLiteralMatch(const struct RoseEngine *t,
struct hs_scratch *scratch, u32 literal_id,
u64a end) {
assert(end);
struct fatbit **anchoredLiteralRows = getAnchoredLiteralLog(scratch);
DEBUG_PRINTF("record %u @ %llu\n", literal_id, end);
if (!bf64_set(&scratch->al_log_sum, end - 1)) {
// first time, clear row
DEBUG_PRINTF("clearing %llu/%u\n", end - 1, t->anchored_count);
fatbit_clear(anchoredLiteralRows[end - 1]);
}
u32 rel_idx = literal_id - t->anchored_base_id;
DEBUG_PRINTF("record %u @ %llu index %u/%u\n", literal_id, end, rel_idx,
t->anchored_count);
assert(rel_idx < t->anchored_count);
fatbit_set(anchoredLiteralRows[end - 1], t->anchored_count, rel_idx);
}
static rose_inline
void runEagerPrefixesStream(const struct RoseEngine *t,
struct hs_scratch *scratch) {
if (!t->eagerIterOffset
|| scratch->core_info.buf_offset >= EAGER_STOP_OFFSET) {
return;
}
char *state = scratch->core_info.state;
u8 *ara = getActiveLeftArray(t, state); /* indexed by offsets into
* left_table */
const u32 arCount = t->activeLeftCount;
const u32 qCount = t->queueCount;
const struct LeftNfaInfo *left_table = getLeftTable(t);
const struct mmbit_sparse_iter *it = getByOffset(t, t->eagerIterOffset);
struct mmbit_sparse_state si_state[MAX_SPARSE_ITER_STATES];
u32 idx = 0;
u32 ri = mmbit_sparse_iter_begin(ara, arCount, &idx, it, si_state);
for (; ri != MMB_INVALID;
ri = mmbit_sparse_iter_next(ara, arCount, ri, &idx, it, si_state)) {
const struct LeftNfaInfo *left = left_table + ri;
u32 qi = ri + t->leftfixBeginQueue;
DEBUG_PRINTF("leftfix %u of %u, maxLag=%u\n", ri, arCount, left->maxLag);
assert(!fatbit_isset(scratch->aqa, qCount, qi));
assert(left->eager);
assert(!left->infix);
struct mq *q = scratch->queues + qi;
const struct NFA *nfa = getNfaByQueue(t, qi);
s64a loc = MIN(scratch->core_info.len,
EAGER_STOP_OFFSET - scratch->core_info.buf_offset);
fatbit_set(scratch->aqa, qCount, qi);
initRoseQueue(t, qi, left, scratch);
if (scratch->core_info.buf_offset) {
s64a sp = left->transient ? -(s64a)scratch->core_info.hlen
: -(s64a)loadRoseDelay(t, state, left);
pushQueueAt(q, 0, MQE_START, sp);
if (scratch->core_info.buf_offset + sp > 0) {
loadStreamState(nfa, q, sp);
/* if the leftfix fix is currently in a match state, we cannot
* advance it. */
if (nfaInAnyAcceptState(nfa, q)) {
continue;
}
pushQueueAt(q, 1, MQE_END, loc);
} else {
pushQueueAt(q, 1, MQE_TOP, sp);
pushQueueAt(q, 2, MQE_END, loc);
nfaQueueInitState(q->nfa, q);
}
} else {
pushQueueAt(q, 0, MQE_START, 0);
pushQueueAt(q, 1, MQE_TOP, 0);
pushQueueAt(q, 2, MQE_END, loc);
nfaQueueInitState(nfa, q);
}
char alive = nfaQueueExecToMatch(q->nfa, q, loc);
if (!alive) {
DEBUG_PRINTF("queue %u dead, squashing\n", qi);
mmbit_unset(ara, arCount, ri);
fatbit_unset(scratch->aqa, qCount, qi);
scratch->tctxt.groups &= left->squash_mask;
} else if (q->cur == q->end) {
assert(alive != MO_MATCHES_PENDING);
/* unlike in block mode we cannot squash groups if there is no match
* in this block as we need the groups on for later stream writes */
/* TODO: investigate possibility of a method to suppress groups for
* a single stream block. */
DEBUG_PRINTF("queue %u finished, nfa lives\n", qi);
q->cur = q->end = 0;
pushQueueAt(q, 0, MQE_START, loc);
} else {
assert(alive == MO_MATCHES_PENDING);
DEBUG_PRINTF("queue %u unfinished, nfa lives\n", qi);
q->end--; /* remove end item */
}
}
}
static really_inline
char roseCatchUpLeftfix(const struct RoseEngine *t, char *state,
struct hs_scratch *scratch, u32 qi,
const struct LeftNfaInfo *left) {
assert(!left->transient); // active roses only
struct core_info *ci = &scratch->core_info;
const u32 qCount = t->queueCount;
struct mq *q = scratch->queues + qi;
const struct NFA *nfa = getNfaByQueue(t, qi);
if (nfaSupportsZombie(nfa)
&& ci->buf_offset /* prefix can be alive with no q */
&& !fatbit_isset(scratch->aqa, qCount, qi)
&& isZombie(t, state, left)) {
DEBUG_PRINTF("yawn - zombie\n");
return 1;
}
if (left->stopTable) {
enum MiracleAction mrv =
roseScanForMiracles(t, state, scratch, qi, left, nfa);
switch (mrv) {
case MIRACLE_DEAD:
return 0;
case MIRACLE_SAVED:
return 1;
default:
assert(mrv == MIRACLE_CONTINUE);
break;
}
}
if (!fatbit_set(scratch->aqa, qCount, qi)) {
initRoseQueue(t, qi, left, scratch);
s32 sp;
if (ci->buf_offset) {
sp = -(s32)loadRoseDelay(t, state, left);
} else {
sp = 0;
}
DEBUG_PRINTF("ci->len=%zu, sp=%d, historyRequired=%u\n", ci->len, sp,
t->historyRequired);
if ( ci->len - sp + 1 < t->historyRequired) {
// we'll end up safely in the history region.
DEBUG_PRINTF("safely in history, skipping\n");
storeRoseDelay(t, state, left, (s64a)ci->len - sp);
return 1;
}
pushQueueAt(q, 0, MQE_START, sp);
if (left->infix || ci->buf_offset + sp > 0) {
loadStreamState(nfa, q, sp);
} else {
pushQueueAt(q, 1, MQE_TOP, sp);
nfaQueueInitState(nfa, q);
}
} else {
DEBUG_PRINTF("queue already active\n");
if (q->end - q->cur == 1 && q_cur_type(q) == MQE_START) {
DEBUG_PRINTF("empty queue, start loc=%lld\n", q_cur_loc(q));
s64a last_loc = q_cur_loc(q);
if (ci->len - last_loc + 1 < t->historyRequired) {
// we'll end up safely in the history region.
DEBUG_PRINTF("safely in history, saving state and skipping\n");
saveStreamState(nfa, q, last_loc);
storeRoseDelay(t, state, left, (s64a)ci->len - last_loc);
return 1;
}
}
}
// Determine whether the byte before last_loc will be in the history
// buffer on the next stream write.
s64a last_loc = q_last_loc(q);
s64a leftovers = ci->len - last_loc;
if (leftovers + 1 >= t->historyRequired) {
u32 catchup_offset = left->maxLag ? left->maxLag - 1 : 0;
last_loc = (s64a)ci->len - catchup_offset;
}
if (left->infix) {
if (infixTooOld(q, last_loc)) {
DEBUG_PRINTF("infix died of old age\n");
return 0;
}
reduceInfixQueue(q, last_loc, left->maxQueueLen, q->nfa->maxWidth);
}
DEBUG_PRINTF("end scan at %lld\n", last_loc);
pushQueueNoMerge(q, MQE_END, last_loc);
#ifdef DEBUG
debugQueue(q);
#endif
char rv = nfaQueueExecRose(nfa, q, MO_INVALID_IDX);
if (!rv) { /* nfa is dead */
DEBUG_PRINTF("died catching up to stream boundary\n");
return 0;
} else {
DEBUG_PRINTF("alive, saving stream state\n");
if (nfaSupportsZombie(nfa) &&
nfaGetZombieStatus(nfa, q, last_loc) == NFA_ZOMBIE_ALWAYS_YES) {
DEBUG_PRINTF("not so fast - zombie\n");
setAsZombie(t, state, left);
} else {
saveStreamState(nfa, q, last_loc);
storeRoseDelay(t, state, left, (s64a)ci->len - last_loc);
}
}
return 1;
}
static really_inline
enum MiracleAction roseScanForMiracles(const struct RoseEngine *t, char *state,
struct hs_scratch *scratch, u32 qi,
const struct LeftNfaInfo *left,
const struct NFA *nfa) {
struct core_info *ci = &scratch->core_info;
const u32 qCount = t->queueCount;
struct mq *q = scratch->queues + qi;
const char q_active = fatbit_isset(scratch->aqa, qCount, qi);
DEBUG_PRINTF("q_active=%d\n", q_active);
const s64a begin_loc = q_active ? q_cur_loc(q) : 0;
const s64a end_loc = ci->len;
s64a miracle_loc;
if (roseMiracleOccurs(t, left, ci, begin_loc, end_loc, &miracle_loc)) {
goto found_miracle;
}
if (roseCountingMiracleOccurs(t, left, ci, begin_loc, end_loc,
&miracle_loc)) {
goto found_miracle;
}
DEBUG_PRINTF("no miracle\n");
return MIRACLE_CONTINUE;
found_miracle:
DEBUG_PRINTF("miracle at %lld\n", miracle_loc);
if (left->infix) {
if (!q_active) {
DEBUG_PRINTF("killing infix\n");
return MIRACLE_DEAD;
}
DEBUG_PRINTF("skip q forward, %lld to %lld\n", begin_loc, miracle_loc);
q_skip_forward_to(q, miracle_loc);
if (q_last_type(q) == MQE_START) {
DEBUG_PRINTF("miracle caused infix to die\n");
return MIRACLE_DEAD;
}
DEBUG_PRINTF("re-init infix state\n");
assert(q->items[q->cur].type == MQE_START);
q->items[q->cur].location = miracle_loc;
nfaQueueInitState(q->nfa, q);
} else {
if (miracle_loc > end_loc - t->historyRequired) {
char *streamState = state + getNfaInfoByQueue(t, qi)->stateOffset;
u64a offset = ci->buf_offset + miracle_loc;
u8 key = offset ? getByteBefore(ci, miracle_loc) : 0;
DEBUG_PRINTF("init state, key=0x%02x, offset=%llu\n", key, offset);
if (!nfaInitCompressedState(nfa, offset, streamState, key)) {
return MIRACLE_DEAD;
}
storeRoseDelay(t, state, left, (s64a)ci->len - miracle_loc);
return MIRACLE_SAVED;
}
DEBUG_PRINTF("re-init prefix (skip %lld->%lld)\n", begin_loc,
miracle_loc);
if (!q_active) {
fatbit_set(scratch->aqa, qCount, qi);
initRoseQueue(t, qi, left, scratch);
}
q->cur = q->end = 0;
pushQueueAt(q, 0, MQE_START, miracle_loc);
pushQueueAt(q, 1, MQE_TOP, miracle_loc);
nfaQueueInitState(q->nfa, q);
}
return MIRACLE_CONTINUE;
}
hwlmcb_rv_t roseHandleChainMatch(const struct RoseEngine *t,
struct hs_scratch *scratch, u32 event,
u64a top_squash_distance, u64a end,
char in_catchup) {
assert(event == MQE_TOP || event >= MQE_TOP_FIRST);
struct core_info *ci = &scratch->core_info;
u8 *aa = getActiveLeafArray(t, scratch->core_info.state);
u32 aaCount = t->activeArrayCount;
struct fatbit *activeQueues = scratch->aqa;
u32 qCount = t->queueCount;
const u32 qi = 0; /* MPV is always queue 0 if it exists */
struct mq *q = &scratch->queues[qi];
const struct NfaInfo *info = getNfaInfoByQueue(t, qi);
s64a loc = (s64a)end - ci->buf_offset;
assert(loc <= (s64a)ci->len && loc >= -(s64a)ci->hlen);
if (!mmbit_set(aa, aaCount, qi)) {
initQueue(q, qi, t, scratch);
nfaQueueInitState(q->nfa, q);
pushQueueAt(q, 0, MQE_START, loc);
fatbit_set(activeQueues, qCount, qi);
} else if (info->no_retrigger) {
DEBUG_PRINTF("yawn\n");
/* nfa only needs one top; we can go home now */
return HWLM_CONTINUE_MATCHING;
} else if (!fatbit_set(activeQueues, qCount, qi)) {
initQueue(q, qi, t, scratch);
loadStreamState(q->nfa, q, 0);
pushQueueAt(q, 0, MQE_START, 0);
} else if (isQueueFull(q)) {
DEBUG_PRINTF("queue %u full -> catching up nfas\n", qi);
/* we know it is a chained nfa and the suffixes/outfixes must already
* be known to be consistent */
if (ensureMpvQueueFlushed(t, scratch, qi, loc, in_catchup)
== HWLM_TERMINATE_MATCHING) {
DEBUG_PRINTF("terminating...\n");
return HWLM_TERMINATE_MATCHING;
}
}
if (top_squash_distance) {
assert(q->cur != q->end);
struct mq_item *last = &q->items[q->end - 1];
if (last->type == event
&& last->location >= loc - (s64a)top_squash_distance) {
last->location = loc;
goto event_enqueued;
}
}
pushQueue(q, event, loc);
event_enqueued:
if (q_cur_loc(q) == (s64a)ci->len) {
/* we may not run the nfa; need to ensure state is fine */
DEBUG_PRINTF("empty run\n");
pushQueueNoMerge(q, MQE_END, loc);
char alive = nfaQueueExec(q->nfa, q, loc);
if (alive) {
scratch->tctxt.mpv_inactive = 0;
q->cur = q->end = 0;
pushQueueAt(q, 0, MQE_START, loc);
} else {
mmbit_unset(aa, aaCount, qi);
fatbit_unset(scratch->aqa, qCount, qi);
}
}
DEBUG_PRINTF("added mpv event at %lld\n", loc);
scratch->tctxt.next_mpv_offset = 0; /* the top event may result in matches
* earlier than expected */
return HWLM_CONTINUE_MATCHING;
}
static rose_inline
int roseEodRunIterator(const struct RoseEngine *t, u8 *state, u64a offset,
struct hs_scratch *scratch) {
if (!t->eodIterOffset) {
return MO_CONTINUE_MATCHING;
}
const struct RoseRole *roleTable = getRoleTable(t);
const struct RosePred *predTable = getPredTable(t);
const struct RoseIterMapping *iterMapBase
= getByOffset(t, t->eodIterMapOffset);
const struct mmbit_sparse_iter *it = getByOffset(t, t->eodIterOffset);
assert(ISALIGNED(iterMapBase));
assert(ISALIGNED(it));
// Sparse iterator state was allocated earlier
struct mmbit_sparse_state *s = scratch->sparse_iter_state;
struct fatbit *handled_roles = scratch->handled_roles;
const u32 numStates = t->rolesWithStateCount;
void *role_state = getRoleState(state);
u32 idx = 0;
u32 i = mmbit_sparse_iter_begin(role_state, numStates, &idx, it, s);
fatbit_clear(handled_roles);
for (; i != MMB_INVALID;
i = mmbit_sparse_iter_next(role_state, numStates, i, &idx, it, s)) {
DEBUG_PRINTF("pred state %u (iter idx=%u) is on\n", i, idx);
const struct RoseIterMapping *iterMap = iterMapBase + idx;
const struct RoseIterRole *roles = getByOffset(t, iterMap->offset);
assert(ISALIGNED(roles));
DEBUG_PRINTF("%u roles to consider\n", iterMap->count);
for (u32 j = 0; j != iterMap->count; j++) {
u32 role = roles[j].role;
assert(role < t->roleCount);
DEBUG_PRINTF("checking role %u, pred %u:\n", role, roles[j].pred);
const struct RoseRole *tr = roleTable + role;
if (fatbit_isset(handled_roles, t->roleCount, role)) {
DEBUG_PRINTF("role %u already handled by the walk, skip\n",
role);
continue;
}
// Special case: if this role is a trivial case (pred type simple)
// we don't need to check any history and we already know the pred
// role is on.
if (tr->flags & ROSE_ROLE_PRED_SIMPLE) {
DEBUG_PRINTF("pred type is simple, no need for checks\n");
} else {
assert(roles[j].pred < t->predCount);
const struct RosePred *tp = predTable + roles[j].pred;
if (!roseCheckPredHistory(tp, offset)) {
continue;
}
}
/* mark role as handled so we don't touch it again in this walk */
fatbit_set(handled_roles, t->roleCount, role);
DEBUG_PRINTF("fire report for role %u, report=%u\n", role,
tr->reportId);
int rv = scratch->tctxt.cb(offset, tr->reportId,
scratch->tctxt.userCtx);
if (rv == MO_HALT_MATCHING) {
return MO_HALT_MATCHING;
}
}
}
return MO_CONTINUE_MATCHING;
}
static really_inline
int roseSomAdaptor_i(u64a from_offset, u64a to_offset, ReportID id,
void *context, char is_simple) {
assert(id != MO_INVALID_IDX); // Should never get an invalid ID.
u32 flags = 0;
struct hs_scratch *scratch = (struct hs_scratch *)context;
struct core_info *ci = &scratch->core_info;
const struct RoseEngine *rose = ci->rose;
const struct internal_report *ri = getInternalReport(rose, id);
/* internal events should be handled by rose directly */
assert(ri->type == EXTERNAL_CALLBACK);
DEBUG_PRINTF("internal match at %llu: IID=%u type=%hhu RID=%u "
"offsetAdj=%d\n", to_offset, id, ri->type, ri->onmatch,
ri->offsetAdjust);
if (unlikely(can_stop_matching(scratch))) {
DEBUG_PRINTF("pre broken - halting\n");
return MO_HALT_MATCHING;
}
if (!is_simple && ri->hasBounds) {
assert(ri->minOffset || ri->minLength || ri->maxOffset < MAX_OFFSET);
if (to_offset < ri->minOffset || to_offset > ri->maxOffset) {
DEBUG_PRINTF("match fell outside valid range %llu !: [%llu,%llu]\n",
to_offset, ri->minOffset, ri->maxOffset);
return MO_CONTINUE_MATCHING;
}
}
int halt = 0;
if (!is_simple && unlikely(isExhausted(ci->exhaustionVector, ri->ekey))) {
DEBUG_PRINTF("ate exhausted match\n");
goto do_return;
}
#ifdef DEDUPE_MATCHES
u64a offset = to_offset;
#endif
to_offset += ri->offsetAdjust;
assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset);
if (!is_simple && ri->minLength) {
if (from_offset != HS_OFFSET_PAST_HORIZON &&
(to_offset - from_offset < ri->minLength)) {
return MO_CONTINUE_MATCHING;
}
if (ri->quashSom) {
from_offset = 0;
}
}
DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n",
from_offset, to_offset, ri->onmatch, ci->userContext);
#ifndef RELEASE_BUILD
if (ri->offsetAdjust != 0) {
// alert testing tools that we've got adjusted matches
flags |= HS_MATCH_FLAG_ADJUSTED;
}
#endif
#ifdef DEDUPE_MATCHES
u32 dkeyCount = rose->dkeyCount;
if (offset != scratch->deduper.current_report_offset) {
assert(scratch->deduper.current_report_offset == ~0ULL
|| scratch->deduper.current_report_offset < offset);
if (offset == scratch->deduper.current_report_offset + 1) {
fatbit_clear(scratch->deduper.log[offset % 2]);
} else {
fatbit_clear(scratch->deduper.log[0]);
fatbit_clear(scratch->deduper.log[1]);
}
halt = flushStoredSomMatches(scratch, offset);
if (halt) {
goto do_return;
}
scratch->deduper.current_report_offset = offset;
}
u32 dkey = ri->dkey;
if (dkey != MO_INVALID_IDX) {
if (ri->quashSom) {
DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset);
assert(ri->offsetAdjust == 0 || ri->offsetAdjust == -1);
if (fatbit_set(scratch->deduper.log[to_offset % 2], dkeyCount,
dkey)) {
/* we have already raised this report at this offset, squash
* dupe match. */
DEBUG_PRINTF("dedupe\n");
goto do_return;
}
} else {
/* SOM external event */
DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset);
assert(ri->offsetAdjust == 0 || ri->offsetAdjust == -1);
u64a *starts = scratch->deduper.som_start_log[to_offset % 2];
if (fatbit_set(scratch->deduper.som_log[to_offset % 2], dkeyCount,
dkey)) {
starts[dkey] = MIN(starts[dkey], from_offset);
} else {
starts[dkey] = from_offset;
}
if (ri->offsetAdjust) {
scratch->deduper.som_log_dirty |= 1;
} else {
scratch->deduper.som_log_dirty |= 2;
}
goto do_return;
}
}
#endif
halt = ci->userCallback((unsigned int)ri->onmatch, from_offset, to_offset,
flags, ci->userContext);
if (!is_simple) {
markAsMatched(ci->exhaustionVector, ri->ekey);
}
do_return:
if (halt) {
DEBUG_PRINTF("callback requested to terminate matches\n");
setBroken(ci->state, BROKEN_FROM_USER);
ci->broken = BROKEN_FROM_USER;
return MO_HALT_MATCHING;
}
return MO_CONTINUE_MATCHING;
}
void handleSomInternal(struct hs_scratch *scratch,
const struct som_operation *ri, const u64a to_offset) {
assert(scratch);
assert(ri);
DEBUG_PRINTF("-->som action required at %llu\n", to_offset);
// SOM handling at scan time operates on data held in scratch. In
// streaming mode, this data is read from / written out to stream state at
// stream write boundaries.
struct core_info *ci = &scratch->core_info;
const struct RoseEngine *rose = ci->rose;
assert(rose->hasSom);
const u32 som_store_count = rose->somLocationCount;
u8 *som_store_valid = (u8 *)ci->state + rose->stateOffsets.somValid;
u8 *som_store_writable = (u8 *)ci->state + rose->stateOffsets.somWritable;
struct fatbit *som_set_now = scratch->som_set_now;
struct fatbit *som_attempted_set = scratch->som_attempted_set;
u64a *som_store = scratch->som_store;
u64a *som_failed_store = scratch->som_attempted_store;
if (to_offset != scratch->som_set_now_offset) {
assert(scratch->som_set_now_offset == ~0ULL
|| to_offset > scratch->som_set_now_offset);
DEBUG_PRINTF("setting som_set_now_offset=%llu\n", to_offset);
fatbit_clear(som_set_now);
fatbit_clear(som_attempted_set);
scratch->som_set_now_offset = to_offset;
}
switch (ri->type) {
case SOM_INTERNAL_LOC_SET:
DEBUG_PRINTF("SOM_INTERNAL_LOC_SET\n");
mmbit_set(som_store_valid, som_store_count, ri->onmatch);
setSomLoc(som_set_now, som_store, som_store_count, ri, to_offset);
return;
case SOM_INTERNAL_LOC_SET_IF_UNSET:
DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_IF_UNSET\n");
if (ok_and_mark_if_unset(som_store_valid, som_set_now, som_store_count,
ri->onmatch)) {
setSomLoc(som_set_now, som_store, som_store_count, ri, to_offset);
}
return;
case SOM_INTERNAL_LOC_SET_IF_WRITABLE: {
u32 slot = ri->onmatch;
DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_IF_WRITABLE\n");
if (ok_and_mark_if_write(som_store_valid, som_set_now,
som_store_writable, som_store_count, slot)) {
setSomLoc(som_set_now, som_store, som_store_count, ri, to_offset);
mmbit_unset(som_store_writable, som_store_count, slot);
} else {
/* not writable, stash as an attempted write in case we are
* racing our escape. */
DEBUG_PRINTF("not writable, stashing attempt\n");
assert(to_offset >= ri->aux.somDistance);
u64a start_offset = to_offset - ri->aux.somDistance;
if (!fatbit_set(som_attempted_set, som_store_count, slot)) {
som_failed_store[slot] = start_offset;
} else {
LIMIT_TO_AT_MOST(&som_failed_store[slot], start_offset);
}
DEBUG_PRINTF("som_failed_store[%u] = %llu\n", slot,
som_failed_store[slot]);
}
return;
}
case SOM_INTERNAL_LOC_SET_REV_NFA:
DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_REV_NFA\n");
mmbit_set(som_store_valid, som_store_count, ri->onmatch);
setSomLocRevNfa(scratch, som_set_now, som_store, som_store_count, ri,
to_offset);
return;
case SOM_INTERNAL_LOC_SET_REV_NFA_IF_UNSET:
DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_REV_NFA_IF_UNSET\n");
if (ok_and_mark_if_unset(som_store_valid, som_set_now, som_store_count,
ri->onmatch)) {
setSomLocRevNfa(scratch, som_set_now, som_store, som_store_count,
ri, to_offset);
}
return;
case SOM_INTERNAL_LOC_SET_REV_NFA_IF_WRITABLE: {
u32 slot = ri->onmatch;
DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_IF_WRITABLE\n");
if (ok_and_mark_if_write(som_store_valid, som_set_now,
som_store_writable, som_store_count, slot)) {
setSomLocRevNfa(scratch, som_set_now, som_store, som_store_count,
ri, to_offset);
mmbit_unset(som_store_writable, som_store_count, slot);
} else {
/* not writable, stash as an attempted write in case we are
* racing our escape. */
DEBUG_PRINTF("not writable, stashing attempt\n");
u64a from_offset = 0;
runRevNfa(scratch, ri, to_offset, &from_offset);
if (!fatbit_set(som_attempted_set, som_store_count, slot)) {
som_failed_store[slot] = from_offset;
} else {
LIMIT_TO_AT_MOST(&som_failed_store[slot], from_offset);
}
DEBUG_PRINTF("som_failed_store[%u] = %llu\n", slot,
som_failed_store[slot]);
}
return;
}
case SOM_INTERNAL_LOC_COPY: {
u32 slot_in = ri->aux.somDistance;
u32 slot_out = ri->onmatch;
DEBUG_PRINTF("SOM_INTERNAL_LOC_COPY S[%u] = S[%u]\n", slot_out,
slot_in);
assert(mmbit_isset(som_store_valid, som_store_count, slot_in));
mmbit_set(som_store_valid, som_store_count, slot_out);
fatbit_set(som_set_now, som_store_count, slot_out);
som_store[slot_out] = som_store[slot_in];
return;
}
case SOM_INTERNAL_LOC_COPY_IF_WRITABLE: {
u32 slot_in = ri->aux.somDistance;
u32 slot_out = ri->onmatch;
DEBUG_PRINTF("SOM_INTERNAL_LOC_COPY_IF_WRITABLE S[%u] = S[%u]\n",
slot_out, slot_in);
assert(mmbit_isset(som_store_valid, som_store_count, slot_in));
if (ok_and_mark_if_write(som_store_valid, som_set_now,
som_store_writable, som_store_count,
slot_out)) {
DEBUG_PRINTF("copy, set som_store[%u]=%llu\n", slot_out,
som_store[slot_in]);
som_store[slot_out] = som_store[slot_in];
fatbit_set(som_set_now, som_store_count, slot_out);
mmbit_unset(som_store_writable, som_store_count, slot_out);
} else {
/* not writable, stash as an attempted write in case we are
* racing our escape */
DEBUG_PRINTF("not writable, stashing attempt\n");
fatbit_set(som_attempted_set, som_store_count, slot_out);
som_failed_store[slot_out] = som_store[slot_in];
DEBUG_PRINTF("som_failed_store[%u] = %llu\n", slot_out,
som_failed_store[slot_out]);
}
return;
}
case SOM_INTERNAL_LOC_MAKE_WRITABLE: {
u32 slot = ri->onmatch;
DEBUG_PRINTF("SOM_INTERNAL_LOC_MAKE_WRITABLE\n");
/* if just written to the loc, ignore the racing escape */
if (fatbit_isset(som_set_now, som_store_count, slot)) {
DEBUG_PRINTF("just written\n");
return;
}
if (fatbit_isset(som_attempted_set, som_store_count, slot)) {
/* writes were waiting for an escape to arrive */
DEBUG_PRINTF("setting som_store[%u] = %llu from "
"som_failed_store[%u]\n", slot, som_failed_store[slot],
slot);
som_store[slot] = som_failed_store[slot];
fatbit_set(som_set_now, som_store_count, slot);
return;
}
mmbit_set(som_store_writable, som_store_count, slot);
return;
}
default:
DEBUG_PRINTF("unknown report type!\n");
break;
}
// All valid som_operation types should be handled and returned above.
assert(0);
return;
}
