static UBool usearch_fwd_n(
UStringSearch *usearch,
const UString *subject,
DArray *array, /* NULL to skip n matches */
int32_t n,
int32_t *l,
UErrorCode *status
) {
int32_t u;
while (n > 0 && U_SUCCESS(*status) && USEARCH_DONE != (u = usearch_next(usearch, status))) {
--n;
if (NULL != array) {
add_match(array, subject, *l, u);
}
*l = u += usearch_getMatchedLength(usearch);
}
if (0 == n) {
return TRUE;
} else {
if (NULL != array) {
add_match(array, subject, *l, subject->len);
}
return FALSE;
}
}
UBool findPattern()
{
UErrorCode status = U_ZERO_ERROR;
int32_t offset = usearch_next(search, &status);
if (offset == USEARCH_DONE) {
fprintf(stdout, "Pattern not found in source\n");
}
while (offset != USEARCH_DONE) {
fprintf(stdout, "Pattern found at offset %d size %d\n", offset,
usearch_getMatchedLength(search));
offset = usearch_next(search, &status);
}
if (U_FAILURE(status)) {
fprintf(stderr, "Error in searching for pattern %d\n", status);
return FALSE;
}
fprintf(stdout, "End of search\n");
return TRUE;
}
static double process_item(MatchInfo *m, Stack *stack, int32_t *final_positions, UStringSearch **searches) {
UChar32 hc, lc;
double final_score = 0.0, score = 0.0, score_for_char = 0.0;
int32_t pos, i, j, hidx, nidx, last_idx, distance, *positions = final_positions + m->needle_len;
MemoryItem mem = {0};
UStringSearch *search = NULL;
UErrorCode status = U_ZERO_ERROR;
stack_push(stack, 0, 0, 0, 0.0, final_positions);
while (stack->pos >= 0) {
stack_pop(stack, &hidx, &nidx, &last_idx, &score, positions);
mem = m->memo[hidx][nidx][last_idx];
if (mem.score == DBL_MAX) {
// No memoized result, calculate the score
for (i = nidx; i < m->needle_len;) {
nidx = i;
U16_FWD_1(m->needle, i, m->needle_len);// i now points to next char in needle
search = searches[nidx];
if (search == NULL || m->haystack_len - hidx < m->needle_len - nidx) { score = 0.0; break; }
status = U_ZERO_ERROR; // We ignore any errors as we already know that hidx is correct
usearch_setOffset(search, hidx, &status);
status = U_ZERO_ERROR;
pos = usearch_next(search, &status);
if (pos == USEARCH_DONE) { score = 0.0; break; } // No matches found
distance = u_countChar32(m->haystack + last_idx, pos - last_idx);
if (distance <= 1) score_for_char = m->max_score_per_char;
else {
U16_GET(m->haystack, 0, pos, m->haystack_len, hc);
j = pos;
U16_PREV(m->haystack, 0, j, lc); // lc is the prev character
score_for_char = calc_score_for_char(m, lc, hc, distance);
}
j = pos;
U16_NEXT(m->haystack, j, m->haystack_len, hc);
hidx = j;
if (m->haystack_len - hidx >= m->needle_len - nidx) stack_push(stack, hidx, nidx, last_idx, score, positions);
last_idx = pos;
positions[nidx] = pos;
score += score_for_char;
} // for(i) iterate over needle
mem.score = score; memcpy(mem.positions, positions, sizeof(*positions) * m->needle_len);
} else {
score = mem.score; memcpy(positions, mem.positions, sizeof(*positions) * m->needle_len);
}
// We have calculated the score for this hidx, nidx, last_idx combination, update final_score and final_positions, if needed
if (score > final_score) {
final_score = score;
memcpy(final_positions, positions, sizeof(*positions) * m->needle_len);
}
}
return final_score;
}
bool TextSearcherICU::nextMatchResult(MatchResult& result) {
UErrorCode status = U_ZERO_ERROR;
const int matchStart = usearch_next(m_searcher, &status);
DCHECK_EQ(status, U_ZERO_ERROR);
// TODO(iceman): It is possible to use |usearch_getText| function
// to retrieve text length and not store it explicitly.
if (!(matchStart >= 0 && static_cast<size_t>(matchStart) < m_textLength)) {
DCHECK_EQ(matchStart, USEARCH_DONE);
result.start = 0;
result.length = 0;
return false;
}
result.start = static_cast<size_t>(matchStart);
result.length = usearch_getMatchedLength(m_searcher);
return true;
}
/**
* Count pattern occurcess in a string [with collation]
*
* @param str character vector
* @param pattern character vector
* @param opts_collator passed to stri__ucol_open()
* @return integer vector
*
* @version 0.1-?? (Marek Gagolewski)
*
* @version 0.1-?? (Marek Gagolewski)
* corrected behavior on empty str/pattern
*
* @version 0.1-?? (Marek Gagolewski, 2013-06-23)
* make StriException-friendly,
* use StriContainerUStringSearch
*
* @version 0.2-3 (Marek Gagolewski, 2014-05-08)
* new fun: stri_count_coll (opts_collator == NA not allowed)
*
* @version 0.3-1 (Marek Gagolewski, 2014-11-04)
* Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
*/
SEXP stri_count_coll(SEXP str, SEXP pattern, SEXP opts_collator)
{
PROTECT(str = stri_prepare_arg_string(str, "str"));
PROTECT(pattern = stri_prepare_arg_string(pattern, "pattern"));
// call stri__ucol_open after prepare_arg:
// if prepare_arg had failed, we would have a mem leak
UCollator* collator = NULL;
collator = stri__ucol_open(opts_collator);
STRI__ERROR_HANDLER_BEGIN(2)
R_len_t vectorize_length = stri__recycling_rule(true, 2, LENGTH(str), LENGTH(pattern));
StriContainerUTF16 str_cont(str, vectorize_length);
StriContainerUStringSearch pattern_cont(pattern, vectorize_length, collator); // collator is not owned by pattern_cont
SEXP ret;
STRI__PROTECT(ret = Rf_allocVector(INTSXP, vectorize_length));
int* ret_tab = INTEGER(ret);
for (R_len_t i = pattern_cont.vectorize_init();
i != pattern_cont.vectorize_end();
i = pattern_cont.vectorize_next(i))
{
STRI__CONTINUE_ON_EMPTY_OR_NA_STR_PATTERN(str_cont, pattern_cont,
ret_tab[i] = NA_INTEGER,
ret_tab[i] = 0)
UStringSearch *matcher = pattern_cont.getMatcher(i, str_cont.get(i));
usearch_reset(matcher);
UErrorCode status = U_ZERO_ERROR;
R_len_t found = 0;
while (!U_FAILURE(status) && ((int)usearch_next(matcher, &status) != USEARCH_DONE))
++found;
STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
ret_tab[i] = found;
}
if (collator) { ucol_close(collator); collator=NULL; }
STRI__UNPROTECT_ALL
return ret;
STRI__ERROR_HANDLER_END(
if (collator) ucol_close(collator);
)
}
/**
* Count pattern occurcess in a string [with collation]
*
* @param str character vector
* @param pattern character vector
* @param collator_opts passed to stri__ucol_open(),
* if \code{NA}, then \code{stri_detect_fixed_byte} is called
* @return integer vector
*
* @version 0.1 (Marek Gagolewski)
* @version 0.2 (Marek Gagolewski) - corrected behavior on empty str/pattern
* @version 0.3 (Marek Gagolewski, 2013-06-23) make StriException-friendly,
* use StriContainerUStringSearch
*/
SEXP stri_count_fixed(SEXP str, SEXP pattern, SEXP collator_opts)
{
str = stri_prepare_arg_string(str, "str");
pattern = stri_prepare_arg_string(pattern, "pattern");
// call stri__ucol_open after prepare_arg:
// if prepare_arg had failed, we would have a mem leak
UCollator* collator = stri__ucol_open(collator_opts);
if (!collator)
return stri__count_fixed_byte(str, pattern);
STRI__ERROR_HANDLER_BEGIN
R_len_t vectorize_length = stri__recycling_rule(true, 2, LENGTH(str), LENGTH(pattern));
StriContainerUTF16 str_cont(str, vectorize_length);
StriContainerUStringSearch pattern_cont(pattern, vectorize_length, collator); // collator is not owned by pattern_cont
SEXP ret;
PROTECT(ret = Rf_allocVector(INTSXP, vectorize_length));
int* ret_tab = INTEGER(ret);
for (R_len_t i = pattern_cont.vectorize_init();
i != pattern_cont.vectorize_end();
i = pattern_cont.vectorize_next(i))
{
STRI__CONTINUE_ON_EMPTY_OR_NA_STR_PATTERN(str_cont, pattern_cont,
ret_tab[i] = NA_INTEGER,
ret_tab[i] = 0)
UStringSearch *matcher = pattern_cont.getMatcher(i, str_cont.get(i));
usearch_reset(matcher);
UErrorCode status = U_ZERO_ERROR;
ret_tab[i] = 0;
while (((int)usearch_next(matcher, &status) != USEARCH_DONE) && !U_FAILURE(status))
++ret_tab[i];
if (U_FAILURE(status)) throw StriException(status);
}
if (collator) { ucol_close(collator); collator=NULL; }
UNPROTECT(1);
return ret;
STRI__ERROR_HANDLER_END(
if (collator) ucol_close(collator);
)
}
/* {{{ grapheme_strpos_utf16 - strrpos using utf16*/
int32_t grapheme_strpos_utf16(char *haystack, size_t haystack_len, char *needle, size_t needle_len, int32_t offset, int32_t *puchar_pos, int f_ignore_case, int last)
{
UChar *uhaystack = NULL, *uneedle = NULL;
int32_t uhaystack_len = 0, uneedle_len = 0, char_pos, ret_pos, offset_pos = 0;
unsigned char u_break_iterator_buffer[U_BRK_SAFECLONE_BUFFERSIZE];
UBreakIterator* bi = NULL;
UErrorCode status;
UStringSearch* src = NULL;
UCollator *coll;
if(puchar_pos) {
*puchar_pos = -1;
}
/* convert the strings to UTF-16. */
status = U_ZERO_ERROR;
intl_convert_utf8_to_utf16(&uhaystack, &uhaystack_len, haystack, haystack_len, &status );
STRPOS_CHECK_STATUS(status, "Error converting input string to UTF-16");
status = U_ZERO_ERROR;
intl_convert_utf8_to_utf16(&uneedle, &uneedle_len, needle, needle_len, &status );
STRPOS_CHECK_STATUS(status, "Error converting needle string to UTF-16");
/* get a pointer to the haystack taking into account the offset */
status = U_ZERO_ERROR;
bi = grapheme_get_break_iterator(u_break_iterator_buffer, &status );
STRPOS_CHECK_STATUS(status, "Failed to get iterator");
status = U_ZERO_ERROR;
ubrk_setText(bi, uhaystack, uhaystack_len, &status);
STRPOS_CHECK_STATUS(status, "Failed to set up iterator");
status = U_ZERO_ERROR;
src = usearch_open(uneedle, uneedle_len, uhaystack, uhaystack_len, "", bi, &status);
STRPOS_CHECK_STATUS(status, "Error creating search object");
if(f_ignore_case) {
coll = usearch_getCollator(src);
status = U_ZERO_ERROR;
ucol_setAttribute(coll, UCOL_STRENGTH, UCOL_SECONDARY, &status);
STRPOS_CHECK_STATUS(status, "Error setting collation strength");
usearch_reset(src);
}
if(offset != 0) {
offset_pos = grapheme_get_haystack_offset(bi, offset);
if(offset_pos == -1) {
status = U_ILLEGAL_ARGUMENT_ERROR;
STRPOS_CHECK_STATUS(status, "Invalid search offset");
}
status = U_ZERO_ERROR;
usearch_setOffset(src, offset_pos, &status);
STRPOS_CHECK_STATUS(status, "Invalid search offset");
}
if(last) {
char_pos = usearch_last(src, &status);
if(char_pos < offset_pos) {
/* last one is beyound our start offset */
char_pos = USEARCH_DONE;
}
} else {
char_pos = usearch_next(src, &status);
}
STRPOS_CHECK_STATUS(status, "Error looking up string");
if(char_pos != USEARCH_DONE && ubrk_isBoundary(bi, char_pos)) {
ret_pos = grapheme_count_graphemes(bi, uhaystack,char_pos);
if(puchar_pos) {
*puchar_pos = char_pos;
}
} else {
ret_pos = -1;
}
if (uhaystack) {
efree( uhaystack );
}
if (uneedle) {
efree( uneedle );
}
ubrk_close (bi);
usearch_close (src);
return ret_pos;
}
inline size_t SearchBuffer::search(size_t& start)
{
size_t size = m_buffer.size();
if (m_atBreak) {
if (!size)
return 0;
} else {
if (size != m_buffer.capacity())
return 0;
}
UStringSearch* searcher = blink::searcher();
UErrorCode status = U_ZERO_ERROR;
usearch_setText(searcher, m_buffer.data(), size, &status);
ASSERT(status == U_ZERO_ERROR);
usearch_setOffset(searcher, m_prefixLength, &status);
ASSERT(status == U_ZERO_ERROR);
int matchStart = usearch_next(searcher, &status);
ASSERT(status == U_ZERO_ERROR);
nextMatch:
if (!(matchStart >= 0 && static_cast<size_t>(matchStart) < size)) {
ASSERT(matchStart == USEARCH_DONE);
return 0;
}
// Matches that start in the overlap area are only tentative.
// The same match may appear later, matching more characters,
// possibly including a combining character that's not yet in the buffer.
if (!m_atBreak && static_cast<size_t>(matchStart) >= size - m_overlap) {
size_t overlap = m_overlap;
if (m_options & AtWordStarts) {
// Ensure that there is sufficient context before matchStart the next time around for
// determining if it is at a word boundary.
int wordBoundaryContextStart = matchStart;
U16_BACK_1(m_buffer.data(), 0, wordBoundaryContextStart);
wordBoundaryContextStart = startOfLastWordBoundaryContext(m_buffer.data(), wordBoundaryContextStart);
overlap = std::min(size - 1, std::max(overlap, size - wordBoundaryContextStart));
}
memcpy(m_buffer.data(), m_buffer.data() + size - overlap, overlap * sizeof(UChar));
m_prefixLength -= std::min(m_prefixLength, size - overlap);
m_buffer.shrink(overlap);
return 0;
}
size_t matchedLength = usearch_getMatchedLength(searcher);
ASSERT_WITH_SECURITY_IMPLICATION(matchStart + matchedLength <= size);
// If this match is "bad", move on to the next match.
if (isBadMatch(m_buffer.data() + matchStart, matchedLength) || ((m_options & AtWordStarts) && !isWordStartMatch(matchStart, matchedLength))) {
matchStart = usearch_next(searcher, &status);
ASSERT(status == U_ZERO_ERROR);
goto nextMatch;
}
size_t newSize = size - (matchStart + 1);
memmove(m_buffer.data(), m_buffer.data() + matchStart + 1, newSize * sizeof(UChar));
m_prefixLength -= std::min<size_t>(m_prefixLength, matchStart + 1);
m_buffer.shrink(newSize);
start = size - matchStart;
return matchedLength;
}
static UBool engine_fixed_split(error_t **error, void *data, const UString *subject, DArray *array, interval_list_t *intervals)
{
UErrorCode status;
int32_t l, lastU;
dlist_element_t *el;
FETCH_DATA(data, p, fixed_pattern_t);
lastU = l = 0;
status = U_ZERO_ERROR;
if (NULL != p->usearch) {
usearch_setText(p->usearch, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "usearch_setText");
return FALSE;
}
/* <X> */
if (NULL == intervals) {
int32_t u;
while (U_SUCCESS(status) && USEARCH_DONE != (u = usearch_next(p->usearch, &status))) {
add_match(array, subject, l, u);
l = u += usearch_getMatchedLength(p->usearch);
}
add_match(array, subject, l, subject->len);
} else {
/* </X> */
for (el = intervals->head; NULL != el; el = el->next) {
FETCH_DATA(el->data, i, interval_t);
if (i->lower_limit > 0) {
if (!usearch_fwd_n(p->usearch, subject, NULL, i->lower_limit - lastU, &l, &status)) {
break;
}
}
if (!usearch_fwd_n(p->usearch, subject, array, i->upper_limit - i->lower_limit, &l, &status)) {
break;
}
lastU = i->upper_limit;
}
/* <X> */
}
/* </X> */
usearch_unbindText(p->usearch);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "usearch_next");
return FALSE;
}
} else {
if (NULL != p->ubrk) {
ubrk_setText(p->ubrk, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "ubrk_setText");
return FALSE;
}
}
/* <X> */
if (NULL == intervals) {
UChar *m;
int32_t u;
u = 0;
while (NULL != (m = u_strFindFirst(subject->ptr + u, subject->len - u, p->pattern->ptr, p->pattern->len))) {
u = m - subject->ptr;
if (NULL == p->ubrk || (ubrk_isBoundary(p->ubrk, u) && ubrk_isBoundary(p->ubrk, u + p->pattern->len))) {
add_match(array, subject, l, u);
}
l = u = u + p->pattern->len;
}
add_match(array, subject, l, subject->len);
} else {
/* </X> */
for (el = intervals->head; NULL != el; el = el->next) {
FETCH_DATA(el->data, i, interval_t);
if (i->lower_limit > 0) {
if (!binary_fwd_n(p->ubrk, p->pattern, subject, NULL, i->lower_limit - lastU, &l)) {
break;
}
}
if (!binary_fwd_n(p->ubrk, p->pattern, subject, array, i->upper_limit - i->lower_limit, &l)) {
break;
}
lastU = i->upper_limit;
}
/* <X> */
}
/* </X> */
ubrk_unbindText(p->ubrk);
}
return TRUE;
}
static engine_return_t engine_fixed_match_all(error_t **error, void *data, const UString *subject, interval_list_t *intervals)
{
int32_t matches;
UErrorCode status;
FETCH_DATA(data, p, fixed_pattern_t);
matches = 0;
status = U_ZERO_ERROR;
if (ustring_empty(p->pattern)) {
if (IS_WORD_BOUNDED(p->flags)) {
if (ustring_empty(subject)) {
return ENGINE_MATCH_FOUND;
} else {
int32_t l, u, lastState, state;
ubrk_setText(p->ubrk, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "ubrk_setText");
return ENGINE_FAILURE;
}
if (UBRK_DONE != (l = ubrk_first(p->ubrk))) {
lastState = ubrk_getRuleStatus(p->ubrk);
while (UBRK_DONE != (u = ubrk_next(p->ubrk))) {
state = ubrk_getRuleStatus(p->ubrk);
if (UBRK_WORD_NONE == lastState && lastState == state) {
return ENGINE_MATCH_FOUND;
}
lastState = state;
l = u;
}
}
return ENGINE_NO_MATCH;
}
} else {
return ENGINE_MATCH_FOUND;
}
} else if (NULL != p->usearch) {
int32_t l, u;
if (subject->len > 0) {
usearch_setText(p->usearch, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "usearch_setText");
return ENGINE_FAILURE;
}
for (l = usearch_first(p->usearch, &status); U_SUCCESS(status) && USEARCH_DONE != l; l = usearch_next(p->usearch, &status)) {
matches++;
u = l + usearch_getMatchedLength(p->usearch);
if (interval_list_add(intervals, subject->len, l, u)) {
return ENGINE_WHOLE_LINE_MATCH;
}
}
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "usearch_[first|next]");
return ENGINE_FAILURE;
}
usearch_unbindText(p->usearch);
return (matches ? ENGINE_MATCH_FOUND : ENGINE_NO_MATCH);
} else {
return ENGINE_NO_MATCH;
}
} else {
UChar *m;
int32_t pos;
pos = 0;
if (NULL != p->ubrk) {
ubrk_setText(p->ubrk, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "ubrk_setText");
return ENGINE_FAILURE;
}
}
while (NULL != (m = u_strFindFirst(subject->ptr + pos, subject->len - pos, p->pattern->ptr, p->pattern->len))) {
pos = m - subject->ptr;
if (NULL == p->ubrk || (ubrk_isBoundary(p->ubrk, pos) && ubrk_isBoundary(p->ubrk, pos + p->pattern->len))) {
matches++;
if (interval_list_add(intervals, subject->len, pos, pos + p->pattern->len)) {
return ENGINE_WHOLE_LINE_MATCH;
}
}
pos += p->pattern->len;
}
ubrk_unbindText(p->ubrk);
return (matches ? ENGINE_MATCH_FOUND : ENGINE_NO_MATCH);
}
}
