void setSomFromSomAware(struct hs_scratch *scratch, const struct som_operation *ri, u64a from_offset, u64a to_offset) { assert(scratch); assert(ri); assert(to_offset); assert(ri->type == SOM_INTERNAL_LOC_SET_FROM || ri->type == SOM_INTERNAL_LOC_SET_FROM_IF_WRITABLE); struct core_info *ci = &scratch->core_info; const struct RoseEngine *rose = ci->rose; assert(rose->hasSom); const u32 som_store_count = rose->somLocationCount; u8 *som_store_valid = (u8 *)ci->state + rose->stateOffsets.somValid; u8 *som_store_writable = (u8 *)ci->state + rose->stateOffsets.somWritable; struct fatbit *som_set_now = scratch->som_set_now; struct fatbit *som_attempted_set = scratch->som_attempted_set; u64a *som_store = scratch->som_store; u64a *som_failed_store = scratch->som_attempted_store; if (to_offset != scratch->som_set_now_offset) { DEBUG_PRINTF("setting som_set_now_offset=%llu\n", to_offset); fatbit_clear(som_set_now); fatbit_clear(som_attempted_set); scratch->som_set_now_offset = to_offset; } if (ri->type == SOM_INTERNAL_LOC_SET_FROM) { DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_FROM\n"); mmbit_set(som_store_valid, som_store_count, ri->onmatch); setSomLoc(som_set_now, som_store, som_store_count, ri, from_offset); } else { DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_FROM_IF_WRITABLE\n"); if (ok_and_mark_if_write(som_store_valid, som_set_now, som_store_writable, som_store_count, ri->onmatch)) { setSomLoc(som_set_now, som_store, som_store_count, ri, from_offset); mmbit_unset(som_store_writable, som_store_count, ri->onmatch); } else { /* not writable, stash as an attempted write in case we are * racing our escape. */ DEBUG_PRINTF("not writable, stashing attempt\n"); assert(to_offset >= ri->aux.somDistance); u32 som_loc = ri->onmatch; if (!fatbit_set(som_attempted_set, som_store_count, ri->onmatch)) { som_failed_store[som_loc] = from_offset; } else { LIMIT_TO_AT_MOST(&som_failed_store[som_loc], from_offset); } DEBUG_PRINTF("som_failed_store[%u] = %llu\n", som_loc, som_failed_store[som_loc]); } } }
static really_inline void setSomLoc(struct fatbit *som_set_now, u64a *som_store, u32 som_store_count, const struct som_operation *ri, u64a to_offset) { /* validity handled by callers */ assert(to_offset >= ri->aux.somDistance); u64a start_offset = to_offset - ri->aux.somDistance; u32 som_loc = ri->onmatch; /* resolve any races for matches at this point in favour of the earliest som */ if (!fatbit_set(som_set_now, som_store_count, som_loc)) { som_store[som_loc] = start_offset; } else { LIMIT_TO_AT_MOST(&som_store[som_loc], start_offset); } DEBUG_PRINTF("som_store[%u] set to %llu\n", som_loc, som_store[som_loc]); }
static really_inline void setSomLocRevNfa(struct hs_scratch *scratch, struct fatbit *som_set_now, u64a *som_store, u32 som_store_count, const struct som_operation *ri, u64a to_offset) { /* validity handled by callers */ u64a from_offset = 0; runRevNfa(scratch, ri, to_offset, &from_offset); u32 som_loc = ri->onmatch; /* resolve any races for matches at this point in favour of the earliest som */ if (!fatbit_set(som_set_now, som_store_count, som_loc)) { som_store[som_loc] = from_offset; } else { LIMIT_TO_AT_MOST(&som_store[som_loc], from_offset); } DEBUG_PRINTF("som_store[%u] set to %llu\n", som_loc, som_store[som_loc]); }
static rose_inline void recordAnchoredLiteralMatch(const struct RoseEngine *t, struct hs_scratch *scratch, u32 literal_id, u64a end) { assert(end); struct fatbit **anchoredLiteralRows = getAnchoredLiteralLog(scratch); DEBUG_PRINTF("record %u @ %llu\n", literal_id, end); if (!bf64_set(&scratch->al_log_sum, end - 1)) { // first time, clear row DEBUG_PRINTF("clearing %llu/%u\n", end - 1, t->anchored_count); fatbit_clear(anchoredLiteralRows[end - 1]); } u32 rel_idx = literal_id - t->anchored_base_id; DEBUG_PRINTF("record %u @ %llu index %u/%u\n", literal_id, end, rel_idx, t->anchored_count); assert(rel_idx < t->anchored_count); fatbit_set(anchoredLiteralRows[end - 1], t->anchored_count, rel_idx); }
static rose_inline void runEagerPrefixesStream(const struct RoseEngine *t, struct hs_scratch *scratch) { if (!t->eagerIterOffset || scratch->core_info.buf_offset >= EAGER_STOP_OFFSET) { return; } char *state = scratch->core_info.state; u8 *ara = getActiveLeftArray(t, state); /* indexed by offsets into * left_table */ const u32 arCount = t->activeLeftCount; const u32 qCount = t->queueCount; const struct LeftNfaInfo *left_table = getLeftTable(t); const struct mmbit_sparse_iter *it = getByOffset(t, t->eagerIterOffset); struct mmbit_sparse_state si_state[MAX_SPARSE_ITER_STATES]; u32 idx = 0; u32 ri = mmbit_sparse_iter_begin(ara, arCount, &idx, it, si_state); for (; ri != MMB_INVALID; ri = mmbit_sparse_iter_next(ara, arCount, ri, &idx, it, si_state)) { const struct LeftNfaInfo *left = left_table + ri; u32 qi = ri + t->leftfixBeginQueue; DEBUG_PRINTF("leftfix %u of %u, maxLag=%u\n", ri, arCount, left->maxLag); assert(!fatbit_isset(scratch->aqa, qCount, qi)); assert(left->eager); assert(!left->infix); struct mq *q = scratch->queues + qi; const struct NFA *nfa = getNfaByQueue(t, qi); s64a loc = MIN(scratch->core_info.len, EAGER_STOP_OFFSET - scratch->core_info.buf_offset); fatbit_set(scratch->aqa, qCount, qi); initRoseQueue(t, qi, left, scratch); if (scratch->core_info.buf_offset) { s64a sp = left->transient ? -(s64a)scratch->core_info.hlen : -(s64a)loadRoseDelay(t, state, left); pushQueueAt(q, 0, MQE_START, sp); if (scratch->core_info.buf_offset + sp > 0) { loadStreamState(nfa, q, sp); /* if the leftfix fix is currently in a match state, we cannot * advance it. */ if (nfaInAnyAcceptState(nfa, q)) { continue; } pushQueueAt(q, 1, MQE_END, loc); } else { pushQueueAt(q, 1, MQE_TOP, sp); pushQueueAt(q, 2, MQE_END, loc); nfaQueueInitState(q->nfa, q); } } else { pushQueueAt(q, 0, MQE_START, 0); pushQueueAt(q, 1, MQE_TOP, 0); pushQueueAt(q, 2, MQE_END, loc); nfaQueueInitState(nfa, q); } char alive = nfaQueueExecToMatch(q->nfa, q, loc); if (!alive) { DEBUG_PRINTF("queue %u dead, squashing\n", qi); mmbit_unset(ara, arCount, ri); fatbit_unset(scratch->aqa, qCount, qi); scratch->tctxt.groups &= left->squash_mask; } else if (q->cur == q->end) { assert(alive != MO_MATCHES_PENDING); /* unlike in block mode we cannot squash groups if there is no match * in this block as we need the groups on for later stream writes */ /* TODO: investigate possibility of a method to suppress groups for * a single stream block. */ DEBUG_PRINTF("queue %u finished, nfa lives\n", qi); q->cur = q->end = 0; pushQueueAt(q, 0, MQE_START, loc); } else { assert(alive == MO_MATCHES_PENDING); DEBUG_PRINTF("queue %u unfinished, nfa lives\n", qi); q->end--; /* remove end item */ } } }
static really_inline char roseCatchUpLeftfix(const struct RoseEngine *t, char *state, struct hs_scratch *scratch, u32 qi, const struct LeftNfaInfo *left) { assert(!left->transient); // active roses only struct core_info *ci = &scratch->core_info; const u32 qCount = t->queueCount; struct mq *q = scratch->queues + qi; const struct NFA *nfa = getNfaByQueue(t, qi); if (nfaSupportsZombie(nfa) && ci->buf_offset /* prefix can be alive with no q */ && !fatbit_isset(scratch->aqa, qCount, qi) && isZombie(t, state, left)) { DEBUG_PRINTF("yawn - zombie\n"); return 1; } if (left->stopTable) { enum MiracleAction mrv = roseScanForMiracles(t, state, scratch, qi, left, nfa); switch (mrv) { case MIRACLE_DEAD: return 0; case MIRACLE_SAVED: return 1; default: assert(mrv == MIRACLE_CONTINUE); break; } } if (!fatbit_set(scratch->aqa, qCount, qi)) { initRoseQueue(t, qi, left, scratch); s32 sp; if (ci->buf_offset) { sp = -(s32)loadRoseDelay(t, state, left); } else { sp = 0; } DEBUG_PRINTF("ci->len=%zu, sp=%d, historyRequired=%u\n", ci->len, sp, t->historyRequired); if ( ci->len - sp + 1 < t->historyRequired) { // we'll end up safely in the history region. DEBUG_PRINTF("safely in history, skipping\n"); storeRoseDelay(t, state, left, (s64a)ci->len - sp); return 1; } pushQueueAt(q, 0, MQE_START, sp); if (left->infix || ci->buf_offset + sp > 0) { loadStreamState(nfa, q, sp); } else { pushQueueAt(q, 1, MQE_TOP, sp); nfaQueueInitState(nfa, q); } } else { DEBUG_PRINTF("queue already active\n"); if (q->end - q->cur == 1 && q_cur_type(q) == MQE_START) { DEBUG_PRINTF("empty queue, start loc=%lld\n", q_cur_loc(q)); s64a last_loc = q_cur_loc(q); if (ci->len - last_loc + 1 < t->historyRequired) { // we'll end up safely in the history region. DEBUG_PRINTF("safely in history, saving state and skipping\n"); saveStreamState(nfa, q, last_loc); storeRoseDelay(t, state, left, (s64a)ci->len - last_loc); return 1; } } } // Determine whether the byte before last_loc will be in the history // buffer on the next stream write. s64a last_loc = q_last_loc(q); s64a leftovers = ci->len - last_loc; if (leftovers + 1 >= t->historyRequired) { u32 catchup_offset = left->maxLag ? left->maxLag - 1 : 0; last_loc = (s64a)ci->len - catchup_offset; } if (left->infix) { if (infixTooOld(q, last_loc)) { DEBUG_PRINTF("infix died of old age\n"); return 0; } reduceInfixQueue(q, last_loc, left->maxQueueLen, q->nfa->maxWidth); } DEBUG_PRINTF("end scan at %lld\n", last_loc); pushQueueNoMerge(q, MQE_END, last_loc); #ifdef DEBUG debugQueue(q); #endif char rv = nfaQueueExecRose(nfa, q, MO_INVALID_IDX); if (!rv) { /* nfa is dead */ DEBUG_PRINTF("died catching up to stream boundary\n"); return 0; } else { DEBUG_PRINTF("alive, saving stream state\n"); if (nfaSupportsZombie(nfa) && nfaGetZombieStatus(nfa, q, last_loc) == NFA_ZOMBIE_ALWAYS_YES) { DEBUG_PRINTF("not so fast - zombie\n"); setAsZombie(t, state, left); } else { saveStreamState(nfa, q, last_loc); storeRoseDelay(t, state, left, (s64a)ci->len - last_loc); } } return 1; }
static really_inline enum MiracleAction roseScanForMiracles(const struct RoseEngine *t, char *state, struct hs_scratch *scratch, u32 qi, const struct LeftNfaInfo *left, const struct NFA *nfa) { struct core_info *ci = &scratch->core_info; const u32 qCount = t->queueCount; struct mq *q = scratch->queues + qi; const char q_active = fatbit_isset(scratch->aqa, qCount, qi); DEBUG_PRINTF("q_active=%d\n", q_active); const s64a begin_loc = q_active ? q_cur_loc(q) : 0; const s64a end_loc = ci->len; s64a miracle_loc; if (roseMiracleOccurs(t, left, ci, begin_loc, end_loc, &miracle_loc)) { goto found_miracle; } if (roseCountingMiracleOccurs(t, left, ci, begin_loc, end_loc, &miracle_loc)) { goto found_miracle; } DEBUG_PRINTF("no miracle\n"); return MIRACLE_CONTINUE; found_miracle: DEBUG_PRINTF("miracle at %lld\n", miracle_loc); if (left->infix) { if (!q_active) { DEBUG_PRINTF("killing infix\n"); return MIRACLE_DEAD; } DEBUG_PRINTF("skip q forward, %lld to %lld\n", begin_loc, miracle_loc); q_skip_forward_to(q, miracle_loc); if (q_last_type(q) == MQE_START) { DEBUG_PRINTF("miracle caused infix to die\n"); return MIRACLE_DEAD; } DEBUG_PRINTF("re-init infix state\n"); assert(q->items[q->cur].type == MQE_START); q->items[q->cur].location = miracle_loc; nfaQueueInitState(q->nfa, q); } else { if (miracle_loc > end_loc - t->historyRequired) { char *streamState = state + getNfaInfoByQueue(t, qi)->stateOffset; u64a offset = ci->buf_offset + miracle_loc; u8 key = offset ? getByteBefore(ci, miracle_loc) : 0; DEBUG_PRINTF("init state, key=0x%02x, offset=%llu\n", key, offset); if (!nfaInitCompressedState(nfa, offset, streamState, key)) { return MIRACLE_DEAD; } storeRoseDelay(t, state, left, (s64a)ci->len - miracle_loc); return MIRACLE_SAVED; } DEBUG_PRINTF("re-init prefix (skip %lld->%lld)\n", begin_loc, miracle_loc); if (!q_active) { fatbit_set(scratch->aqa, qCount, qi); initRoseQueue(t, qi, left, scratch); } q->cur = q->end = 0; pushQueueAt(q, 0, MQE_START, miracle_loc); pushQueueAt(q, 1, MQE_TOP, miracle_loc); nfaQueueInitState(q->nfa, q); } return MIRACLE_CONTINUE; }
hwlmcb_rv_t roseHandleChainMatch(const struct RoseEngine *t, struct hs_scratch *scratch, u32 event, u64a top_squash_distance, u64a end, char in_catchup) { assert(event == MQE_TOP || event >= MQE_TOP_FIRST); struct core_info *ci = &scratch->core_info; u8 *aa = getActiveLeafArray(t, scratch->core_info.state); u32 aaCount = t->activeArrayCount; struct fatbit *activeQueues = scratch->aqa; u32 qCount = t->queueCount; const u32 qi = 0; /* MPV is always queue 0 if it exists */ struct mq *q = &scratch->queues[qi]; const struct NfaInfo *info = getNfaInfoByQueue(t, qi); s64a loc = (s64a)end - ci->buf_offset; assert(loc <= (s64a)ci->len && loc >= -(s64a)ci->hlen); if (!mmbit_set(aa, aaCount, qi)) { initQueue(q, qi, t, scratch); nfaQueueInitState(q->nfa, q); pushQueueAt(q, 0, MQE_START, loc); fatbit_set(activeQueues, qCount, qi); } else if (info->no_retrigger) { DEBUG_PRINTF("yawn\n"); /* nfa only needs one top; we can go home now */ return HWLM_CONTINUE_MATCHING; } else if (!fatbit_set(activeQueues, qCount, qi)) { initQueue(q, qi, t, scratch); loadStreamState(q->nfa, q, 0); pushQueueAt(q, 0, MQE_START, 0); } else if (isQueueFull(q)) { DEBUG_PRINTF("queue %u full -> catching up nfas\n", qi); /* we know it is a chained nfa and the suffixes/outfixes must already * be known to be consistent */ if (ensureMpvQueueFlushed(t, scratch, qi, loc, in_catchup) == HWLM_TERMINATE_MATCHING) { DEBUG_PRINTF("terminating...\n"); return HWLM_TERMINATE_MATCHING; } } if (top_squash_distance) { assert(q->cur != q->end); struct mq_item *last = &q->items[q->end - 1]; if (last->type == event && last->location >= loc - (s64a)top_squash_distance) { last->location = loc; goto event_enqueued; } } pushQueue(q, event, loc); event_enqueued: if (q_cur_loc(q) == (s64a)ci->len) { /* we may not run the nfa; need to ensure state is fine */ DEBUG_PRINTF("empty run\n"); pushQueueNoMerge(q, MQE_END, loc); char alive = nfaQueueExec(q->nfa, q, loc); if (alive) { scratch->tctxt.mpv_inactive = 0; q->cur = q->end = 0; pushQueueAt(q, 0, MQE_START, loc); } else { mmbit_unset(aa, aaCount, qi); fatbit_unset(scratch->aqa, qCount, qi); } } DEBUG_PRINTF("added mpv event at %lld\n", loc); scratch->tctxt.next_mpv_offset = 0; /* the top event may result in matches * earlier than expected */ return HWLM_CONTINUE_MATCHING; }
static rose_inline int roseEodRunIterator(const struct RoseEngine *t, u8 *state, u64a offset, struct hs_scratch *scratch) { if (!t->eodIterOffset) { return MO_CONTINUE_MATCHING; } const struct RoseRole *roleTable = getRoleTable(t); const struct RosePred *predTable = getPredTable(t); const struct RoseIterMapping *iterMapBase = getByOffset(t, t->eodIterMapOffset); const struct mmbit_sparse_iter *it = getByOffset(t, t->eodIterOffset); assert(ISALIGNED(iterMapBase)); assert(ISALIGNED(it)); // Sparse iterator state was allocated earlier struct mmbit_sparse_state *s = scratch->sparse_iter_state; struct fatbit *handled_roles = scratch->handled_roles; const u32 numStates = t->rolesWithStateCount; void *role_state = getRoleState(state); u32 idx = 0; u32 i = mmbit_sparse_iter_begin(role_state, numStates, &idx, it, s); fatbit_clear(handled_roles); for (; i != MMB_INVALID; i = mmbit_sparse_iter_next(role_state, numStates, i, &idx, it, s)) { DEBUG_PRINTF("pred state %u (iter idx=%u) is on\n", i, idx); const struct RoseIterMapping *iterMap = iterMapBase + idx; const struct RoseIterRole *roles = getByOffset(t, iterMap->offset); assert(ISALIGNED(roles)); DEBUG_PRINTF("%u roles to consider\n", iterMap->count); for (u32 j = 0; j != iterMap->count; j++) { u32 role = roles[j].role; assert(role < t->roleCount); DEBUG_PRINTF("checking role %u, pred %u:\n", role, roles[j].pred); const struct RoseRole *tr = roleTable + role; if (fatbit_isset(handled_roles, t->roleCount, role)) { DEBUG_PRINTF("role %u already handled by the walk, skip\n", role); continue; } // Special case: if this role is a trivial case (pred type simple) // we don't need to check any history and we already know the pred // role is on. if (tr->flags & ROSE_ROLE_PRED_SIMPLE) { DEBUG_PRINTF("pred type is simple, no need for checks\n"); } else { assert(roles[j].pred < t->predCount); const struct RosePred *tp = predTable + roles[j].pred; if (!roseCheckPredHistory(tp, offset)) { continue; } } /* mark role as handled so we don't touch it again in this walk */ fatbit_set(handled_roles, t->roleCount, role); DEBUG_PRINTF("fire report for role %u, report=%u\n", role, tr->reportId); int rv = scratch->tctxt.cb(offset, tr->reportId, scratch->tctxt.userCtx); if (rv == MO_HALT_MATCHING) { return MO_HALT_MATCHING; } } } return MO_CONTINUE_MATCHING; }
static really_inline int roseSomAdaptor_i(u64a from_offset, u64a to_offset, ReportID id, void *context, char is_simple) { assert(id != MO_INVALID_IDX); // Should never get an invalid ID. u32 flags = 0; struct hs_scratch *scratch = (struct hs_scratch *)context; struct core_info *ci = &scratch->core_info; const struct RoseEngine *rose = ci->rose; const struct internal_report *ri = getInternalReport(rose, id); /* internal events should be handled by rose directly */ assert(ri->type == EXTERNAL_CALLBACK); DEBUG_PRINTF("internal match at %llu: IID=%u type=%hhu RID=%u " "offsetAdj=%d\n", to_offset, id, ri->type, ri->onmatch, ri->offsetAdjust); if (unlikely(can_stop_matching(scratch))) { DEBUG_PRINTF("pre broken - halting\n"); return MO_HALT_MATCHING; } if (!is_simple && ri->hasBounds) { assert(ri->minOffset || ri->minLength || ri->maxOffset < MAX_OFFSET); if (to_offset < ri->minOffset || to_offset > ri->maxOffset) { DEBUG_PRINTF("match fell outside valid range %llu !: [%llu,%llu]\n", to_offset, ri->minOffset, ri->maxOffset); return MO_CONTINUE_MATCHING; } } int halt = 0; if (!is_simple && unlikely(isExhausted(ci->exhaustionVector, ri->ekey))) { DEBUG_PRINTF("ate exhausted match\n"); goto do_return; } #ifdef DEDUPE_MATCHES u64a offset = to_offset; #endif to_offset += ri->offsetAdjust; assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset); if (!is_simple && ri->minLength) { if (from_offset != HS_OFFSET_PAST_HORIZON && (to_offset - from_offset < ri->minLength)) { return MO_CONTINUE_MATCHING; } if (ri->quashSom) { from_offset = 0; } } DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n", from_offset, to_offset, ri->onmatch, ci->userContext); #ifndef RELEASE_BUILD if (ri->offsetAdjust != 0) { // alert testing tools that we've got adjusted matches flags |= HS_MATCH_FLAG_ADJUSTED; } #endif #ifdef DEDUPE_MATCHES u32 dkeyCount = rose->dkeyCount; if (offset != scratch->deduper.current_report_offset) { assert(scratch->deduper.current_report_offset == ~0ULL || scratch->deduper.current_report_offset < offset); if (offset == scratch->deduper.current_report_offset + 1) { fatbit_clear(scratch->deduper.log[offset % 2]); } else { fatbit_clear(scratch->deduper.log[0]); fatbit_clear(scratch->deduper.log[1]); } halt = flushStoredSomMatches(scratch, offset); if (halt) { goto do_return; } scratch->deduper.current_report_offset = offset; } u32 dkey = ri->dkey; if (dkey != MO_INVALID_IDX) { if (ri->quashSom) { DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset); assert(ri->offsetAdjust == 0 || ri->offsetAdjust == -1); if (fatbit_set(scratch->deduper.log[to_offset % 2], dkeyCount, dkey)) { /* we have already raised this report at this offset, squash * dupe match. */ DEBUG_PRINTF("dedupe\n"); goto do_return; } } else { /* SOM external event */ DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset); assert(ri->offsetAdjust == 0 || ri->offsetAdjust == -1); u64a *starts = scratch->deduper.som_start_log[to_offset % 2]; if (fatbit_set(scratch->deduper.som_log[to_offset % 2], dkeyCount, dkey)) { starts[dkey] = MIN(starts[dkey], from_offset); } else { starts[dkey] = from_offset; } if (ri->offsetAdjust) { scratch->deduper.som_log_dirty |= 1; } else { scratch->deduper.som_log_dirty |= 2; } goto do_return; } } #endif halt = ci->userCallback((unsigned int)ri->onmatch, from_offset, to_offset, flags, ci->userContext); if (!is_simple) { markAsMatched(ci->exhaustionVector, ri->ekey); } do_return: if (halt) { DEBUG_PRINTF("callback requested to terminate matches\n"); setBroken(ci->state, BROKEN_FROM_USER); ci->broken = BROKEN_FROM_USER; return MO_HALT_MATCHING; } return MO_CONTINUE_MATCHING; }
void handleSomInternal(struct hs_scratch *scratch, const struct som_operation *ri, const u64a to_offset) { assert(scratch); assert(ri); DEBUG_PRINTF("-->som action required at %llu\n", to_offset); // SOM handling at scan time operates on data held in scratch. In // streaming mode, this data is read from / written out to stream state at // stream write boundaries. struct core_info *ci = &scratch->core_info; const struct RoseEngine *rose = ci->rose; assert(rose->hasSom); const u32 som_store_count = rose->somLocationCount; u8 *som_store_valid = (u8 *)ci->state + rose->stateOffsets.somValid; u8 *som_store_writable = (u8 *)ci->state + rose->stateOffsets.somWritable; struct fatbit *som_set_now = scratch->som_set_now; struct fatbit *som_attempted_set = scratch->som_attempted_set; u64a *som_store = scratch->som_store; u64a *som_failed_store = scratch->som_attempted_store; if (to_offset != scratch->som_set_now_offset) { assert(scratch->som_set_now_offset == ~0ULL || to_offset > scratch->som_set_now_offset); DEBUG_PRINTF("setting som_set_now_offset=%llu\n", to_offset); fatbit_clear(som_set_now); fatbit_clear(som_attempted_set); scratch->som_set_now_offset = to_offset; } switch (ri->type) { case SOM_INTERNAL_LOC_SET: DEBUG_PRINTF("SOM_INTERNAL_LOC_SET\n"); mmbit_set(som_store_valid, som_store_count, ri->onmatch); setSomLoc(som_set_now, som_store, som_store_count, ri, to_offset); return; case SOM_INTERNAL_LOC_SET_IF_UNSET: DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_IF_UNSET\n"); if (ok_and_mark_if_unset(som_store_valid, som_set_now, som_store_count, ri->onmatch)) { setSomLoc(som_set_now, som_store, som_store_count, ri, to_offset); } return; case SOM_INTERNAL_LOC_SET_IF_WRITABLE: { u32 slot = ri->onmatch; DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_IF_WRITABLE\n"); if (ok_and_mark_if_write(som_store_valid, som_set_now, som_store_writable, som_store_count, slot)) { setSomLoc(som_set_now, som_store, som_store_count, ri, to_offset); mmbit_unset(som_store_writable, som_store_count, slot); } else { /* not writable, stash as an attempted write in case we are * racing our escape. */ DEBUG_PRINTF("not writable, stashing attempt\n"); assert(to_offset >= ri->aux.somDistance); u64a start_offset = to_offset - ri->aux.somDistance; if (!fatbit_set(som_attempted_set, som_store_count, slot)) { som_failed_store[slot] = start_offset; } else { LIMIT_TO_AT_MOST(&som_failed_store[slot], start_offset); } DEBUG_PRINTF("som_failed_store[%u] = %llu\n", slot, som_failed_store[slot]); } return; } case SOM_INTERNAL_LOC_SET_REV_NFA: DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_REV_NFA\n"); mmbit_set(som_store_valid, som_store_count, ri->onmatch); setSomLocRevNfa(scratch, som_set_now, som_store, som_store_count, ri, to_offset); return; case SOM_INTERNAL_LOC_SET_REV_NFA_IF_UNSET: DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_REV_NFA_IF_UNSET\n"); if (ok_and_mark_if_unset(som_store_valid, som_set_now, som_store_count, ri->onmatch)) { setSomLocRevNfa(scratch, som_set_now, som_store, som_store_count, ri, to_offset); } return; case SOM_INTERNAL_LOC_SET_REV_NFA_IF_WRITABLE: { u32 slot = ri->onmatch; DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_IF_WRITABLE\n"); if (ok_and_mark_if_write(som_store_valid, som_set_now, som_store_writable, som_store_count, slot)) { setSomLocRevNfa(scratch, som_set_now, som_store, som_store_count, ri, to_offset); mmbit_unset(som_store_writable, som_store_count, slot); } else { /* not writable, stash as an attempted write in case we are * racing our escape. */ DEBUG_PRINTF("not writable, stashing attempt\n"); u64a from_offset = 0; runRevNfa(scratch, ri, to_offset, &from_offset); if (!fatbit_set(som_attempted_set, som_store_count, slot)) { som_failed_store[slot] = from_offset; } else { LIMIT_TO_AT_MOST(&som_failed_store[slot], from_offset); } DEBUG_PRINTF("som_failed_store[%u] = %llu\n", slot, som_failed_store[slot]); } return; } case SOM_INTERNAL_LOC_COPY: { u32 slot_in = ri->aux.somDistance; u32 slot_out = ri->onmatch; DEBUG_PRINTF("SOM_INTERNAL_LOC_COPY S[%u] = S[%u]\n", slot_out, slot_in); assert(mmbit_isset(som_store_valid, som_store_count, slot_in)); mmbit_set(som_store_valid, som_store_count, slot_out); fatbit_set(som_set_now, som_store_count, slot_out); som_store[slot_out] = som_store[slot_in]; return; } case SOM_INTERNAL_LOC_COPY_IF_WRITABLE: { u32 slot_in = ri->aux.somDistance; u32 slot_out = ri->onmatch; DEBUG_PRINTF("SOM_INTERNAL_LOC_COPY_IF_WRITABLE S[%u] = S[%u]\n", slot_out, slot_in); assert(mmbit_isset(som_store_valid, som_store_count, slot_in)); if (ok_and_mark_if_write(som_store_valid, som_set_now, som_store_writable, som_store_count, slot_out)) { DEBUG_PRINTF("copy, set som_store[%u]=%llu\n", slot_out, som_store[slot_in]); som_store[slot_out] = som_store[slot_in]; fatbit_set(som_set_now, som_store_count, slot_out); mmbit_unset(som_store_writable, som_store_count, slot_out); } else { /* not writable, stash as an attempted write in case we are * racing our escape */ DEBUG_PRINTF("not writable, stashing attempt\n"); fatbit_set(som_attempted_set, som_store_count, slot_out); som_failed_store[slot_out] = som_store[slot_in]; DEBUG_PRINTF("som_failed_store[%u] = %llu\n", slot_out, som_failed_store[slot_out]); } return; } case SOM_INTERNAL_LOC_MAKE_WRITABLE: { u32 slot = ri->onmatch; DEBUG_PRINTF("SOM_INTERNAL_LOC_MAKE_WRITABLE\n"); /* if just written to the loc, ignore the racing escape */ if (fatbit_isset(som_set_now, som_store_count, slot)) { DEBUG_PRINTF("just written\n"); return; } if (fatbit_isset(som_attempted_set, som_store_count, slot)) { /* writes were waiting for an escape to arrive */ DEBUG_PRINTF("setting som_store[%u] = %llu from " "som_failed_store[%u]\n", slot, som_failed_store[slot], slot); som_store[slot] = som_failed_store[slot]; fatbit_set(som_set_now, som_store_count, slot); return; } mmbit_set(som_store_writable, som_store_count, slot); return; } default: DEBUG_PRINTF("unknown report type!\n"); break; } // All valid som_operation types should be handled and returned above. assert(0); return; }