depth depth::operator=(const depth& leftOptr)
{
m_height = leftOptr.getHeight();
m_width = leftOptr.getWidth();
const ushort* dataPtr = leftOptr.getDataPtr();
m_uspDepthData = new ushort[m_height * m_width];
std::copy(dataPtr, dataPtr + m_height * m_width, m_uspDepthData);
return leftOptr;
}
static
u32 depth_to_u32(const depth &d) {
assert(d.is_reachable());
if (d.is_infinite()) {
return REPEAT_INF;
}
u32 d_val = d;
assert(d_val < REPEAT_INF);
return d_val;
}
template <class LbrStruct> static
void fillNfa(NFA *nfa, lbr_common *c, ReportID report, const depth &repeatMin,
const depth &repeatMax, u32 minPeriod, enum RepeatType rtype) {
assert(nfa);
RepeatStateInfo rsi(rtype, repeatMin, repeatMax, minPeriod);
DEBUG_PRINTF("selected %s model for {%s,%s} repeat\n",
repeatTypeName(rtype), repeatMin.str().c_str(),
repeatMax.str().c_str());
// Fill the lbr_common structure first. Note that the RepeatInfo structure
// directly follows the LbrStruct.
const u32 info_offset = sizeof(LbrStruct);
c->repeatInfoOffset = info_offset;
c->report = report;
RepeatInfo *info = (RepeatInfo *)((char *)c + info_offset);
info->type = verify_u8(rtype);
info->repeatMin = depth_to_u32(repeatMin);
info->repeatMax = depth_to_u32(repeatMax);
info->stateSize = rsi.stateSize;
info->packedCtrlSize = rsi.packedCtrlSize;
info->horizon = rsi.horizon;
info->minPeriod = minPeriod;
copy_bytes(&info->packedFieldSizes, rsi.packedFieldSizes);
info->patchCount = rsi.patchCount;
info->patchSize = rsi.patchSize;
info->encodingSize = rsi.encodingSize;
info->patchesOffset = rsi.patchesOffset;
// Fill the NFA structure.
nfa->nPositions = repeatMin;
nfa->streamStateSize = verify_u32(rsi.packedCtrlSize + rsi.stateSize);
nfa->scratchStateSize = (u32)sizeof(lbr_state);
nfa->minWidth = verify_u32(repeatMin);
nfa->maxWidth = repeatMax.is_finite() ? verify_u32(repeatMax) : 0;
// Fill the lbr table for sparse lbr model.
if (rtype == REPEAT_SPARSE_OPTIMAL_P) {
u64a *table = getTable<LbrStruct>(nfa);
// Adjust table length according to the optimal patch length.
size_t len = nfa->length;
assert((u32)repeatMax >= rsi.patchSize);
len -= sizeof(u64a) * ((u32)repeatMax - rsi.patchSize);
nfa->length = verify_u32(len);
info->length = verify_u32(sizeof(RepeatInfo)
+ sizeof(u64a) * (rsi.patchSize + 1));
copy_bytes(table, rsi.table);
}
}
enum RepeatType chooseRepeatType(const depth &repeatMin, const depth &repeatMax,
u32 minPeriod, bool is_reset) {
if (repeatMax.is_infinite()) {
return REPEAT_FIRST;
}
if (repeatMin == depth(0) || is_reset) {
return REPEAT_LAST;
}
// Cases with max < 64 can be handled with either bitmap or trailer. We use
// whichever has smaller packed state.
if (repeatMax < depth(64)) {
u32 bitmap_len =
packedSize(REPEAT_BITMAP, repeatMin, repeatMax, minPeriod);
u32 trailer_len =
packedSize(REPEAT_TRAILER, repeatMin, repeatMax, minPeriod);
return bitmap_len <= trailer_len ? REPEAT_BITMAP : REPEAT_TRAILER;
}
if (repeatMin <= depth(64)) {
return REPEAT_TRAILER;
}
u32 range_len = ~0U;
if (repeatMax > repeatMin &&
numRangeSlots(repeatMin, repeatMax) <= REPEAT_RANGE_MAX_SLOTS) {
assert(numRangeSlots(repeatMin, repeatMax) < 256); // stored in u8
range_len =
streamStateSize(REPEAT_RANGE, repeatMin, repeatMax, minPeriod);
}
assert(repeatMax.is_finite());
u32 sparse_len = ~0U;
if (minPeriod > 6) {
sparse_len =
streamStateSize(REPEAT_SPARSE_OPTIMAL_P, repeatMin, repeatMax, minPeriod);
}
if (range_len != ~0U || sparse_len != ~0U) {
return range_len < sparse_len ? REPEAT_RANGE : REPEAT_SPARSE_OPTIMAL_P;
}
return REPEAT_RING;
}
vector<u8> findLeftOffsetStopAlphabet(const CastleProto &castle,
UNUSED som_type som) {
const depth max_width = findMaxWidth(castle);
DEBUG_PRINTF("castle has reach %s and max width %s\n",
describeClass(castle.reach()).c_str(),
max_width.str().c_str());
const CharReach escape = ~castle.reach(); // invert reach for stop chars.
u32 d = min(max_width, depth(MAX_STOP_DEPTH));
const u8 mask = verify_u8((1U << d) - 1);
vector<u8> stop(N_CHARS, 0);
for (size_t c = escape.find_first(); c != escape.npos;
c = escape.find_next(c)) {
stop[c] |= mask;
}
return stop;
}
static
aligned_unique_ptr<NFA> constructLBR(const CharReach &cr,
const depth &repeatMin,
const depth &repeatMax, u32 minPeriod,
bool is_reset, ReportID report) {
DEBUG_PRINTF("bounds={%s,%s}, cr=%s (count %zu), report=%u\n",
repeatMin.str().c_str(), repeatMax.str().c_str(),
describeClass(cr, 20, CC_OUT_TEXT).c_str(), cr.count(),
report);
assert(repeatMin <= repeatMax);
assert(repeatMax.is_reachable());
aligned_unique_ptr<NFA> nfa
= buildLbrDot(cr, repeatMin, repeatMax, minPeriod, is_reset, report);
if (!nfa) {
nfa = buildLbrVerm(cr, repeatMin, repeatMax, minPeriod, is_reset,
report);
}
if (!nfa) {
nfa = buildLbrNVerm(cr, repeatMin, repeatMax, minPeriod, is_reset,
report);
}
if (!nfa) {
nfa = buildLbrShuf(cr, repeatMin, repeatMax, minPeriod, is_reset,
report);
}
if (!nfa) {
nfa = buildLbrTruf(cr, repeatMin, repeatMax, minPeriod, is_reset,
report);
}
if (!nfa) {
assert(0);
return nullptr;
}
return nfa;
}
/** If the pattern is unanchored, has a max_offset and has not asked for SOM,
* we can use that knowledge to anchor it which will limit its lifespan. Note
* that we can't use this transformation if there's a min_length, as it's
* currently handled using "sly SOM".
*
* Note that it is possible to handle graphs that have a combination of
* anchored and unanchored paths, but it's too tricky for the moment.
*/
static
bool anchorPatternWithBoundedRepeat(NGWrapper &g, const depth &minWidth,
const depth &maxWidth) {
assert(!g.som);
assert(g.max_offset != MAX_OFFSET);
assert(minWidth <= maxWidth);
assert(maxWidth.is_reachable());
DEBUG_PRINTF("widths=[%s,%s], min/max offsets=[%llu,%llu]\n",
minWidth.str().c_str(), maxWidth.str().c_str(), g.min_offset,
g.max_offset);
if (g.max_offset > MAX_MAXOFFSET_TO_ANCHOR) {
return false;
}
if (g.max_offset < minWidth) {
assert(0);
return false;
}
// If the pattern has virtual starts, we probably don't want to touch it.
if (hasVirtualStarts(g)) {
DEBUG_PRINTF("virtual starts, bailing\n");
return false;
}
// Similarly, bail if the pattern is vacuous. TODO: this could be done, we
// would just need to be a little careful with reports.
if (isVacuous(g)) {
DEBUG_PRINTF("vacuous, bailing\n");
return false;
}
u32 min_bound, max_bound;
if (maxWidth.is_infinite()) {
min_bound = 0;
max_bound = g.max_offset - minWidth;
} else {
min_bound = g.min_offset > maxWidth ? g.min_offset - maxWidth : 0;
max_bound = g.max_offset - minWidth;
}
DEBUG_PRINTF("prepending ^.{%u,%u}\n", min_bound, max_bound);
vector<NFAVertex> initials;
for (auto v : adjacent_vertices_range(g.startDs, g)) {
if (v == g.startDs) {
continue;
}
initials.push_back(v);
}
if (initials.empty()) {
DEBUG_PRINTF("no initial vertices\n");
return false;
}
// Wire up 'min_offset' mandatory dots from anchored start.
NFAVertex u = g.start;
for (u32 i = 0; i < min_bound; i++) {
NFAVertex v = add_vertex(g);
g[v].char_reach.setall();
add_edge(u, v, g);
u = v;
}
NFAVertex head = u;
// Wire up optional dots for (max_offset - min_offset).
for (u32 i = 0; i < max_bound - min_bound; i++) {
NFAVertex v = add_vertex(g);
g[v].char_reach.setall();
if (head != u) {
add_edge(head, v, g);
}
add_edge(u, v, g);
u = v;
}
// Remove edges from starts and wire both head and u to our initials.
for (auto v : initials) {
remove_edge(g.startDs, v, g);
remove_edge(g.start, v, g);
if (head != u) {
add_edge(head, v, g);
}
add_edge(u, v, g);
}
g.renumberVertices();
g.renumberEdges();
return true;
}
RepeatStateInfo::RepeatStateInfo(enum RepeatType type, const depth &repeatMin,
const depth &repeatMax, u32 minPeriod)
: stateSize(0), packedCtrlSize(0), horizon(0), patchCount(0),
patchSize(0), encodingSize(0), patchesOffset(0) {
assert(repeatMin <= repeatMax);
assert(repeatMax.is_reachable());
assert(minPeriod || type != REPEAT_SPARSE_OPTIMAL_P);
switch (type) {
case REPEAT_FIRST:
assert(repeatMin.is_finite());
stateSize = 0; // everything is in the control block.
horizon = repeatMin;
packedCtrlSize = calcPackedBytes(horizon + 1);
break;
case REPEAT_LAST:
assert(repeatMax.is_finite());
stateSize = 0; // everything is in the control block.
horizon = repeatMax + 1;
packedCtrlSize = calcPackedBytes(horizon + 1);
break;
case REPEAT_RING:
assert(repeatMax.is_finite());
stateSize = mmbit_size(repeatMax + 1);
horizon = repeatMax * 2 + 1; /* TODO: investigate tightening */
// Packed offset member, plus two bytes for each ring index, reduced to
// one byte each if they'll fit in eight bits.
{
u32 offset_len = calcPackedBytes(horizon + 1);
u32 ring_indices_len = repeatMax < depth(254) ? 2 : 4;
packedCtrlSize = offset_len + ring_indices_len;
}
break;
case REPEAT_RANGE:
assert(repeatMax.is_finite());
assert(repeatMin < repeatMax);
stateSize = numRangeSlots(repeatMin, repeatMax) * sizeof(u16);
horizon = repeatMax * 2 + 1;
// Packed offset member, plus one byte for the number of range
// elements.
packedCtrlSize = calcPackedBytes(horizon + 1) + 1;
break;
case REPEAT_BITMAP:
stateSize = 0; // everything is in the control block.
horizon = 0; // unused
packedCtrlSize = ROUNDUP_N(repeatMax + 1, 8) / 8;
break;
case REPEAT_SPARSE_OPTIMAL_P:
assert(minPeriod);
assert(repeatMax.is_finite());
{
u32 rv = repeatRecurTable(this, repeatMax, minPeriod);
u32 repeatTmp = 0;
if ((u32)repeatMax < minPeriod) {
repeatTmp = repeatMax;
patchCount = 1;
} else {
// find optimal patch size
repeatTmp =
findOptimalPatchSize(this, repeatMax, minPeriod, rv);
assert(patchCount < 65536);
}
DEBUG_PRINTF("repeat[%u %u], period=%u\n", (u32)repeatMin,
(u32)repeatMax, minPeriod);
u64a maxVal = table[repeatTmp];
encodingSize = calcPackedBytes(maxVal);
patchSize = repeatTmp;
assert(encodingSize <= 64);
patchesOffset = mmbit_size(patchCount);
stateSize = patchesOffset + encodingSize * patchCount;
horizon = (repeatTmp * patchCount) * 2 + 1;
u32 ring_indices_len = patchCount < depth(254) ? 2 : 4;
packedCtrlSize = calcPackedBytes(horizon + 1) + ring_indices_len;
}
break;
case REPEAT_TRAILER:
assert(repeatMax.is_finite());
assert(repeatMin <= depth(64));
stateSize = 0; // everything is in the control block.
horizon = repeatMax + 1;
packedFieldSizes.resize(2);
packedFieldSizes[0] = calcPackedBits(horizon + 1);
packedFieldSizes[1] = repeatMin;
packedCtrlSize = (packedFieldSizes[0] + packedFieldSizes[1] + 7U) / 8U;
break;
}
DEBUG_PRINTF("stateSize=%u, packedCtrlSize=%u, horizon=%u\n", stateSize,
packedCtrlSize, horizon);
assert(packedCtrlSize <= sizeof(RepeatControl));
}
