static void TestBug11665(void) {
// The problem was with the incorrect breaking of Japanese text beginning
// with Katakana characters when no prior Japanese or Chinese text had been
// encountered.
//
// Tested here in cintltst, rather than in intltest, because only cintltst
// tests have the ability to reset ICU, which is needed to get the bug
// to manifest itself.
static UChar japaneseText[] = {0x30A2, 0x30EC, 0x30EB, 0x30AE, 0x30FC, 0x6027, 0x7D50, 0x819C, 0x708E};
int32_t boundaries[10] = {0};
UBreakIterator *bi = NULL;
int32_t brk;
int32_t brkIdx = 0;
int32_t totalBreaks = 0;
UErrorCode status = U_ZERO_ERROR;
ctest_resetICU();
bi = ubrk_open(UBRK_WORD, "en_US", japaneseText, UPRV_LENGTHOF(japaneseText), &status);
TEST_ASSERT_SUCCESS(status);
if (!bi) {
return;
}
for (brk=ubrk_first(bi); brk != UBRK_DONE; brk=ubrk_next(bi)) {
boundaries[brkIdx] = brk;
if (++brkIdx >= UPRV_LENGTHOF(boundaries) - 1) {
break;
}
}
if (brkIdx <= 2 || brkIdx >= UPRV_LENGTHOF(boundaries)) {
log_err("%s:%d too few or many breaks found.\n", __FILE__, __LINE__);
} else {
totalBreaks = brkIdx;
brkIdx = 0;
for (brk=ubrk_first(bi); brk != UBRK_DONE; brk=ubrk_next(bi)) {
if (brk != boundaries[brkIdx]) {
log_err("%s:%d Break #%d differs between first and second iteration.\n", __FILE__, __LINE__, brkIdx);
break;
}
if (++brkIdx >= UPRV_LENGTHOF(boundaries) - 1) {
log_err("%s:%d Too many breaks.\n", __FILE__, __LINE__);
break;
}
}
if (totalBreaks != brkIdx) {
log_err("%s:%d Number of breaks differ between first and second iteration.\n", __FILE__, __LINE__);
}
}
ubrk_close(bi);
}
// BreakIterator.split {{{
static PyObject *
icu_BreakIterator_split(icu_BreakIterator *self, PyObject *args, PyObject *kwargs) {
int32_t prev = 0, p = 0, sz = 0;
PyObject *ans = NULL, *token = NULL;
ans = PyList_New(0);
if (ans == NULL) return PyErr_NoMemory();
p = ubrk_first(self->break_iterator);
while (p != UBRK_DONE) {
prev = p; p = ubrk_next(self->break_iterator);
if (self->type == UBRK_WORD && ubrk_getRuleStatus(self->break_iterator) == UBRK_WORD_NONE)
continue; // We are not at the start of a word
sz = (p == UBRK_DONE) ? self->text_len - prev : p - prev;
if (sz > 0) {
token = icu_to_python(self->text + prev, sz);
if (token == NULL) {
Py_DECREF(ans); ans = NULL; break;
}
if (PyList_Append(ans, token) != 0) {
Py_DECREF(token); Py_DECREF(ans); ans = NULL; break;
}
Py_DECREF(token);
}
}
return ans;
} // }}}
std::vector<lstring> convert_split_words(const lstring <) {
std::vector<lstring> ret;
UBreakIterator* bi;
int prev = -1, pos;
UErrorCode err = U_ZERO_ERROR;
bi = ubrk_open(UBRK_WORD, get_locale(), (UChar *)lt.data(), lt.size(), &err);
if (U_FAILURE(err))
return ret;
pos = ubrk_first(bi);
while (pos != UBRK_DONE) {
int rules = ubrk_getRuleStatus(bi);
if ((rules == UBRK_WORD_NONE) || (prev == -1)) {
prev = pos;
} else {
ret.emplace_back(lt.substr(prev, pos - prev));
prev = -1;
}
pos = ubrk_next(bi);
}
ubrk_close(bi);
return ret;
}
/* Print each element in order: */
void printEachForward( UBreakIterator* boundary, UChar* str) {
int32_t end;
int32_t start = ubrk_first(boundary);
for (end = ubrk_next(boundary); end != UBRK_DONE; start = end, end =
ubrk_next(boundary)) {
printTextRange(str, start, end );
}
}
// BreakIterator.index {{{
static PyObject *
icu_BreakIterator_index(icu_BreakIterator *self, PyObject *token) {
#if PY_VERSION_HEX >= 0x03030000
#error Not implemented for python >= 3.3
#endif
UChar *buf = NULL, *needle = NULL;
int32_t word_start = 0, p = 0, sz = 0, ans = -1, leading_hyphen = 0, trailing_hyphen = 0;
buf = python_to_icu(token, &sz);
if (buf == NULL) return NULL;
if (sz < 1) goto end;
needle = buf;
if (sz > 1 && IS_HYPHEN_CHAR(buf[0])) { needle = buf + 1; leading_hyphen = 1; sz -= 1; }
if (sz > 1 && IS_HYPHEN_CHAR(buf[sz-1])) trailing_hyphen = 1;
Py_BEGIN_ALLOW_THREADS;
p = ubrk_first(self->break_iterator);
while (p != UBRK_DONE) {
word_start = p; p = ubrk_next(self->break_iterator);
if (self->type == UBRK_WORD && ubrk_getRuleStatus(self->break_iterator) == UBRK_WORD_NONE)
continue; // We are not at the start of a word
if (self->text_len >= word_start + sz && memcmp(self->text + word_start, needle, sz * sizeof(UChar)) == 0) {
if (word_start > 0 && (
(leading_hyphen && !IS_HYPHEN_CHAR(self->text[word_start-1])) ||
(!leading_hyphen && IS_HYPHEN_CHAR(self->text[word_start-1]))
)) continue;
if (!trailing_hyphen && IS_HYPHEN_CHAR(self->text[word_start + sz])) continue;
if (p == UBRK_DONE || self->text_len <= word_start + sz) { ans = word_start; break; }
if (
// Check that the found word is followed by a word boundary
ubrk_isBoundary(self->break_iterator, word_start + sz) &&
// If there is a leading hyphen check that the leading
// hyphen is preceded by a word boundary
(!leading_hyphen || (word_start > 1 && ubrk_isBoundary(self->break_iterator, word_start - 2))) &&
// Check that there is a word boundary *after* the trailing
// hyphen. We cannot rely on ubrk_isBoundary() as that
// always returns true because of the trailing hyphen.
(!trailing_hyphen || ubrk_following(self->break_iterator, word_start + sz) == UBRK_DONE || ubrk_getRuleStatus(self->break_iterator) == UBRK_WORD_NONE)
) { ans = word_start; break; }
if (p != UBRK_DONE) ubrk_isBoundary(self->break_iterator, p); // Reset the iterator to its position before the call to ubrk_isBoundary()
}
}
if (leading_hyphen && ans > -1) ans -= 1;
#ifdef Py_UNICODE_WIDE
if (ans > 0) ans = u_countChar32(self->text, ans);
#endif
Py_END_ALLOW_THREADS;
end:
free(buf);
return Py_BuildValue("l", (long)ans);
} // }}}
/*
* TestBreakIteratorRules - Verify that a break iterator can be created from
* a set of source rules.
*/
static void TestBreakIteratorRules() {
/* Rules will keep together any run of letters not including 'a', OR
* keep together 'abc', but only when followed by 'def', OTHERWISE
* just return one char at a time.
*/
char rules[] = "abc{666}/def;\n [\\p{L} - [a]]* {2}; . {1};";
/* 0123456789012345678 */
char data[] = "abcdex abcdefgh-def"; /* the test data string */
char breaks[] = "** ** * ** *"; /* * the expected break positions */
char tags[] = "01 21 6 21 2"; /* expected tag values at break positions */
int32_t tagMap[] = {0, 1, 2, 3, 4, 5, 666};
UChar *uData;
void *freeHook = NULL;
UErrorCode status = U_ZERO_ERROR;
int32_t pos;
int i;
UBreakIterator *bi = testOpenRules(rules);
if (bi == NULL) {return;}
uData = toUChar(data, &freeHook);
ubrk_setText(bi, uData, -1, &status);
pos = ubrk_first(bi);
for (i=0; i<sizeof(breaks); i++) {
if (pos == i && breaks[i] != '*') {
log_err("FAIL: unexpected break at position %d found\n", pos);
break;
}
if (pos != i && breaks[i] == '*') {
log_err("FAIL: expected break at position %d not found.\n", i);
break;
}
if (pos == i) {
int32_t tag, expectedTag;
tag = ubrk_getRuleStatus(bi);
expectedTag = tagMap[tags[i]&0xf];
if (tag != expectedTag) {
log_err("FAIL: incorrect tag value. Position = %d; expected tag %d, got %d",
pos, expectedTag, tag);
break;
}
pos = ubrk_next(bi);
}
}
freeToUCharStrings(&freeHook);
ubrk_close(bi);
}
MojErr MojDbTextTokenizer::tokenize(const MojString& text, MojDbTextCollator* collator, KeySet& keysOut) const
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(m_ubrk.get());
// convert to UChar from str
MojDbTextUtils::UnicodeVec unicodeStr;
MojErr err = MojDbTextUtils::strToUnicode(text, unicodeStr);
MojErrCheck(err);
// clone break iterator and set text
MojByte buf[U_BRK_SAFECLONE_BUFFERSIZE];
UErrorCode status = U_ZERO_ERROR;
MojInt32 size = sizeof(buf);
IterPtr ubrk(ubrk_safeClone(m_ubrk.get(), buf, &size, &status));
MojUnicodeErrCheck(status);
MojAssert(ubrk.get());
ubrk_setText(ubrk.get(), unicodeStr.begin(), (MojInt32) unicodeStr.size(), &status);
MojUnicodeErrCheck(status);
MojInt32 tokBegin = -1;
MojInt32 pos = ubrk_first(ubrk.get());
while (pos != UBRK_DONE) {
UWordBreak status = (UWordBreak) ubrk_getRuleStatus(ubrk.get());
if (status != UBRK_WORD_NONE) {
MojAssert(tokBegin != -1);
MojDbKey key;
const UChar* tokChars = unicodeStr.begin() + tokBegin;
MojSize tokSize = (MojSize) (pos - tokBegin);
if (collator) {
err = collator->sortKey(tokChars, tokSize, key);
MojErrCheck(err);
} else {
MojString tok;
err = MojDbTextUtils::unicodeToStr(tokChars, tokSize, tok);
MojErrCheck(err);
err = key.assign(tok);
MojErrCheck(err);
}
err = keysOut.put(key);
MojErrCheck(err);
}
tokBegin = pos;
pos = ubrk_next(ubrk.get());
}
return MojErrNone;
}
int tokenizer_next( tokenizer_t *t, char *word, size_t size ) {
UChar savedEndChar;
int k;
// start iterator
if( t->end == 0 ) {
t->start = ubrk_first(t->boundary);
}
// Find next word
again:
t->end = ubrk_next(t->boundary);
if( t->end == UBRK_DONE ) {
return -1;
}
// Null terminate
savedEndChar = t->str[t->end];
t->str[t->end] = 0;
// Skip unct
if( t->end - t->start == 1 && u_ispunct( t->str[t->start] ) ) {
t->str[t->end] = savedEndChar;
t->start = t->end;
goto again;
}
// Skip whitespace
for( k=t->start; k<t->end; k++ ) {
if( u_isspace( t->str[k] ) == 1 ) {
t->str[t->end] = savedEndChar;
t->start = t->end;
goto again;
}
}
// Copy to C bffer
u_austrncpy(word, t->str+t->start, size-1);
word[size-1] = 0;
printf("string[%2d..%2d] \"%s\" %d\n", t->start, t->end-1, word, u_isspace( t->str[t->start]));
t->str[t->end] = savedEndChar;
t->start = t->end;
return 0;
}
ERL_NIF_TERM len(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
{
ErlNifBinary in;
cloner* ptr;
UBreakIterator* iter;
int count = 0, pos;
UErrorCode status = U_ZERO_ERROR;
if (argc != 2)
return enif_make_badarg(env);
/* Last argument must be a binary */
if (!(enif_inspect_binary(env, argv[1], &in)
&& enif_get_resource(env, argv[0], iterator_type, (void**) &ptr))) {
return enif_make_badarg(env);
}
iter = (UBreakIterator*) cloner_get(ptr);
CHECK_RES(env, iter);
if (iter == NULL) {
return enif_make_badarg(env);
}
/* Do count */
ubrk_setText(iter,
(UChar *) in.data,
TO_ULEN(in.size),
&status);
CHECK(env, status);
pos = ubrk_first(iter);
if (pos != UBRK_DONE)
while (1) {
pos = ubrk_next(iter);
if (pos == UBRK_DONE)
break;
if (is_valid_elem(ptr, iter))
count++;
}
return enif_make_int(env, count);
}
int32_t grapheme_count(UBreakIterator *ubrk, const UString *ustr)
{
int32_t i, count;
UErrorCode status;
count = 0;
status = U_ZERO_ERROR;
ubrk_setText(ubrk, ustr->ptr, ustr->len, &status);
if (U_FAILURE(status)) {
return -1;
}
if (UBRK_DONE != (i = ubrk_first(ubrk))) {
while (UBRK_DONE != (i = ubrk_next(ubrk))) {
++count;
}
}
ubrk_unbindText(ubrk);
return count;
}
/*
* static void TestBreakIteratorUText(void);
*
* Test that ubrk_setUText() is present and works for a simple case.
*/
static void TestBreakIteratorUText(void) {
const char *UTF8Str = "\x41\xc3\x85\x5A\x20\x41\x52\x69\x6E\x67"; /* c3 85 is utf-8 for A with a ring on top */
/* 0 1 2 34567890 */
UErrorCode status = U_ZERO_ERROR;
UBreakIterator *bi = NULL;
int32_t pos = 0;
UText *ut = utext_openUTF8(NULL, UTF8Str, -1, &status);
TEST_ASSERT_SUCCESS(status);
bi = ubrk_open(UBRK_WORD, "en_US", NULL, 0, &status);
if (U_FAILURE(status)) {
log_err_status(status, "Failure at file %s, line %d, error = %s\n", __FILE__, __LINE__, u_errorName(status));
return;
}
ubrk_setUText(bi, ut, &status);
if (U_FAILURE(status)) {
log_err("Failure at file %s, line %d, error = %s\n", __FILE__, __LINE__, u_errorName(status));
return;
}
pos = ubrk_first(bi);
TEST_ASSERT(pos == 0);
pos = ubrk_next(bi);
TEST_ASSERT(pos == 4);
pos = ubrk_next(bi);
TEST_ASSERT(pos == 5);
pos = ubrk_next(bi);
TEST_ASSERT(pos == 10);
pos = ubrk_next(bi);
TEST_ASSERT(pos == UBRK_DONE);
ubrk_close(bi);
utext_close(ut);
}
// BreakIterator.index {{{
static PyObject *
icu_BreakIterator_index(icu_BreakIterator *self, PyObject *args, PyObject *kwargs) {
#if PY_VERSION_HEX >= 0x03030000
#error Not implemented for python >= 3.3
#endif
UChar *buf = NULL;
int32_t prev = 0, p = 0, sz = 0, tsz = 0, ans = -1;
PyObject *token = NULL;
if (!PyArg_ParseTuple(args, "O", &token)) return NULL;
buf = python_to_icu(token, &sz, 1);
if (buf == NULL) return NULL;
if (sz < 1) goto end;
Py_BEGIN_ALLOW_THREADS;
p = ubrk_first(self->break_iterator);
while (p != UBRK_DONE) {
prev = p; p = ubrk_next(self->break_iterator);
if (self->type == UBRK_WORD && ubrk_getRuleStatus(self->break_iterator) == UBRK_WORD_NONE)
continue; // We are not at the start of a word
tsz = (p == UBRK_DONE) ? self->text_len - prev : p - prev;
if (sz == tsz && memcmp(self->text + prev, buf, sz * sizeof(UChar)) == 0) {
#ifdef PY_UNICODE_WIDE
ans = u_countChar32(self->text, prev);
#else
ans = prev;
#endif
break;
}
}
Py_END_ALLOW_THREADS;
end:
free(buf);
return Py_BuildValue("i", ans);
} // }}}
// BreakIterator.split2 {{{
static PyObject *
icu_BreakIterator_split2(icu_BreakIterator *self, PyObject *args, PyObject *kwargs) {
#if PY_VERSION_HEX >= 0x03030000
#error Not implemented for python >= 3.3
#endif
int32_t prev = 0, p = 0, sz = 0;
PyObject *ans = NULL, *temp = NULL;
ans = PyList_New(0);
if (ans == NULL) return PyErr_NoMemory();
p = ubrk_first(self->break_iterator);
while (p != UBRK_DONE) {
prev = p; p = ubrk_next(self->break_iterator);
if (self->type == UBRK_WORD && ubrk_getRuleStatus(self->break_iterator) == UBRK_WORD_NONE)
continue; // We are not at the start of a word
sz = (p == UBRK_DONE) ? self->text_len - prev : p - prev;
if (sz > 0) {
#ifdef Py_UNICODE_WIDE
sz = u_countChar32(self->text + prev, sz);
prev = u_countChar32(self->text, prev);
#endif
temp = Py_BuildValue("II", prev, sz);
if (temp == NULL) {
Py_DECREF(ans); ans = NULL; break;
}
if (PyList_Append(ans, temp) != 0) {
Py_DECREF(temp); Py_DECREF(ans); ans = NULL; break;
}
Py_DECREF(temp);
}
}
return ans;
} // }}}
static void TestBreakIteratorSuppressions(void) {
const TestBISuppressionsItem * itemPtr;
for (itemPtr = testBISuppressionsItems; itemPtr->locale != NULL; itemPtr++) {
UChar textU[kTextULenMax];
int32_t textULen = u_unescape(itemPtr->text, textU, kTextULenMax);
UErrorCode status = U_ZERO_ERROR;
UBreakIterator *bi = ubrk_open(UBRK_SENTENCE, itemPtr->locale, textU, textULen, &status);
log_verbose("#%d: %s\n", (itemPtr-testBISuppressionsItems), itemPtr->locale);
if (U_SUCCESS(status)) {
int32_t offset, start;
const int32_t * expOffsetPtr;
const int32_t * expOffsetStart;
expOffsetStart = expOffsetPtr = itemPtr->expFwdOffsets;
ubrk_first(bi);
for (; (offset = ubrk_next(bi)) != UBRK_DONE && *expOffsetPtr >= 0; expOffsetPtr++) {
if (offset != *expOffsetPtr) {
log_err("FAIL: ubrk_next loc \"%s\", expected %d, got %d\n", itemPtr->locale, *expOffsetPtr, offset);
}
}
if (offset != UBRK_DONE || *expOffsetPtr >= 0) {
log_err("FAIL: ubrk_next loc \"%s\", expected UBRK_DONE & expOffset -1, got %d and %d\n", itemPtr->locale, offset, *expOffsetPtr);
}
expOffsetStart = expOffsetPtr = itemPtr->expFwdOffsets;
start = ubrk_first(bi) + 1;
for (; (offset = ubrk_following(bi, start)) != UBRK_DONE && *expOffsetPtr >= 0; expOffsetPtr++) {
if (offset != *expOffsetPtr) {
log_err("FAIL: ubrk_following(%d) loc \"%s\", expected %d, got %d\n", start, itemPtr->locale, *expOffsetPtr, offset);
}
start = *expOffsetPtr + 1;
}
if (offset != UBRK_DONE || *expOffsetPtr >= 0) {
log_err("FAIL: ubrk_following(%d) loc \"%s\", expected UBRK_DONE & expOffset -1, got %d and %d\n", start, itemPtr->locale, offset, *expOffsetPtr);
}
expOffsetStart = expOffsetPtr = itemPtr->expRevOffsets;
offset = ubrk_last(bi);
log_verbose("___ @%d ubrk_last\n", offset);
if(offset == 0) {
log_err("FAIL: ubrk_last loc \"%s\" unexpected %d\n", itemPtr->locale, offset);
}
for (; (offset = ubrk_previous(bi)) != UBRK_DONE && *expOffsetPtr >= 0; expOffsetPtr++) {
if (offset != *expOffsetPtr) {
log_err("FAIL: ubrk_previous loc \"%s\", expected %d, got %d\n", itemPtr->locale, *expOffsetPtr, offset);
} else {
log_verbose("[%d] @%d ubrk_previous()\n", (expOffsetPtr - expOffsetStart), offset);
}
}
if (offset != UBRK_DONE || *expOffsetPtr >= 0) {
log_err("FAIL: ubrk_previous loc \"%s\", expected UBRK_DONE & expOffset[%d] -1, got %d and %d\n", itemPtr->locale,
expOffsetPtr - expOffsetStart,
offset, *expOffsetPtr);
}
expOffsetStart = expOffsetPtr = itemPtr->expRevOffsets;
start = ubrk_last(bi) - 1;
for (; (offset = ubrk_preceding(bi, start)) != UBRK_DONE && *expOffsetPtr >= 0; expOffsetPtr++) {
if (offset != *expOffsetPtr) {
log_err("FAIL: ubrk_preceding(%d) loc \"%s\", expected %d, got %d\n", start, itemPtr->locale, *expOffsetPtr, offset);
}
start = *expOffsetPtr - 1;
}
if (start >=0 && (offset != UBRK_DONE || *expOffsetPtr >= 0)) {
log_err("FAIL: ubrk_preceding loc(%d) \"%s\", expected UBRK_DONE & expOffset -1, got %d and %d\n", start, itemPtr->locale, offset, *expOffsetPtr);
}
ubrk_close(bi);
} else {
log_data_err("FAIL: ubrk_open(UBRK_SENTENCE, \"%s\", ...) status %s (Are you missing data?)\n", itemPtr->locale, u_errorName(status));
}
}
}
// BreakIterator.split2 {{{
static PyObject *
icu_BreakIterator_split2(icu_BreakIterator *self, PyObject *args) {
#if PY_VERSION_HEX >= 0x03030000
#error Not implemented for python >= 3.3
#endif
int32_t word_start = 0, p = 0, sz = 0, last_pos = 0, last_sz = 0;
int is_hyphen_sep = 0, leading_hyphen = 0, trailing_hyphen = 0;
UChar sep = 0;
PyObject *ans = NULL, *temp = NULL, *t = NULL;
ans = PyList_New(0);
if (ans == NULL) return PyErr_NoMemory();
p = ubrk_first(self->break_iterator);
while (p != UBRK_DONE) {
word_start = p; p = ubrk_next(self->break_iterator);
if (self->type == UBRK_WORD && ubrk_getRuleStatus(self->break_iterator) == UBRK_WORD_NONE)
continue; // We are not at the start of a word
sz = (p == UBRK_DONE) ? self->text_len - word_start : p - word_start;
if (sz > 0) {
// ICU breaks on words containing hyphens, we do not want that, so we recombine manually
is_hyphen_sep = 0; leading_hyphen = 0; trailing_hyphen = 0;
if (word_start > 0) { // Look for a leading hyphen
sep = *(self->text + word_start - 1);
if (IS_HYPHEN_CHAR(sep)) {
leading_hyphen = 1;
if (last_pos > 0 && word_start - last_pos == 1) is_hyphen_sep = 1;
}
}
if (word_start + sz < self->text_len) { // Look for a trailing hyphen
sep = *(self->text + word_start + sz);
if (IS_HYPHEN_CHAR(sep)) trailing_hyphen = 1;
}
last_pos = p;
#ifdef Py_UNICODE_WIDE
sz = u_countChar32(self->text + word_start, sz);
word_start = u_countChar32(self->text, word_start);
#endif
if (is_hyphen_sep && PyList_GET_SIZE(ans) > 0) {
sz = last_sz + sz + trailing_hyphen;
last_sz = sz;
t = PyInt_FromLong((long)sz);
if (t == NULL) { Py_DECREF(ans); ans = NULL; break; }
temp = PyList_GET_ITEM(ans, PyList_GET_SIZE(ans) - 1);
Py_DECREF(PyTuple_GET_ITEM(temp, 1));
PyTuple_SET_ITEM(temp, 1, t);
} else {
sz += leading_hyphen + trailing_hyphen;
last_sz = sz;
temp = Py_BuildValue("ll", (long)(word_start - leading_hyphen), (long)sz);
if (temp == NULL) {
Py_DECREF(ans); ans = NULL; break;
}
if (PyList_Append(ans, temp) != 0) {
Py_DECREF(temp); Py_DECREF(ans); ans = NULL; break;
}
Py_DECREF(temp);
}
}
}
return ans;
} // }}}
static engine_return_t engine_fixed_match_all(error_t **error, void *data, const UString *subject, interval_list_t *intervals)
{
int32_t matches;
UErrorCode status;
FETCH_DATA(data, p, fixed_pattern_t);
matches = 0;
status = U_ZERO_ERROR;
if (ustring_empty(p->pattern)) {
if (IS_WORD_BOUNDED(p->flags)) {
if (ustring_empty(subject)) {
return ENGINE_MATCH_FOUND;
} else {
int32_t l, u, lastState, state;
ubrk_setText(p->ubrk, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "ubrk_setText");
return ENGINE_FAILURE;
}
if (UBRK_DONE != (l = ubrk_first(p->ubrk))) {
lastState = ubrk_getRuleStatus(p->ubrk);
while (UBRK_DONE != (u = ubrk_next(p->ubrk))) {
state = ubrk_getRuleStatus(p->ubrk);
if (UBRK_WORD_NONE == lastState && lastState == state) {
return ENGINE_MATCH_FOUND;
}
lastState = state;
l = u;
}
}
return ENGINE_NO_MATCH;
}
} else {
return ENGINE_MATCH_FOUND;
}
} else if (NULL != p->usearch) {
int32_t l, u;
if (subject->len > 0) {
usearch_setText(p->usearch, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "usearch_setText");
return ENGINE_FAILURE;
}
for (l = usearch_first(p->usearch, &status); U_SUCCESS(status) && USEARCH_DONE != l; l = usearch_next(p->usearch, &status)) {
matches++;
u = l + usearch_getMatchedLength(p->usearch);
if (interval_list_add(intervals, subject->len, l, u)) {
return ENGINE_WHOLE_LINE_MATCH;
}
}
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "usearch_[first|next]");
return ENGINE_FAILURE;
}
usearch_unbindText(p->usearch);
return (matches ? ENGINE_MATCH_FOUND : ENGINE_NO_MATCH);
} else {
return ENGINE_NO_MATCH;
}
} else {
UChar *m;
int32_t pos;
pos = 0;
if (NULL != p->ubrk) {
ubrk_setText(p->ubrk, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "ubrk_setText");
return ENGINE_FAILURE;
}
}
while (NULL != (m = u_strFindFirst(subject->ptr + pos, subject->len - pos, p->pattern->ptr, p->pattern->len))) {
pos = m - subject->ptr;
if (NULL == p->ubrk || (ubrk_isBoundary(p->ubrk, pos) && ubrk_isBoundary(p->ubrk, pos + p->pattern->len))) {
matches++;
if (interval_list_add(intervals, subject->len, pos, pos + p->pattern->len)) {
return ENGINE_WHOLE_LINE_MATCH;
}
}
pos += p->pattern->len;
}
ubrk_unbindText(p->ubrk);
return (matches ? ENGINE_MATCH_FOUND : ENGINE_NO_MATCH);
}
}
/* Print first element */
void printFirst(UBreakIterator* boundary, UChar* str) {
int32_t end;
int32_t start = ubrk_first(boundary);
end = ubrk_next(boundary);
printTextRange( str, start, end );
}
static engine_return_t engine_fixed_match(error_t **error, void *data, const UString *subject)
{
int32_t ret;
UErrorCode status;
FETCH_DATA(data, p, fixed_pattern_t);
status = U_ZERO_ERROR;
if (ustring_empty(p->pattern)) {
if (IS_WORD_BOUNDED(p->flags)) {
if (ustring_empty(subject)) {
return ENGINE_MATCH_FOUND;
} else {
int32_t l, u, lastState, state;
ubrk_setText(p->ubrk, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "ubrk_setText");
return ENGINE_FAILURE;
}
if (UBRK_DONE != (l = ubrk_first(p->ubrk))) {
lastState = ubrk_getRuleStatus(p->ubrk);
while (UBRK_DONE != (u = ubrk_next(p->ubrk))) {
state = ubrk_getRuleStatus(p->ubrk);
if (UBRK_WORD_NONE == lastState && lastState == state) {
return ENGINE_MATCH_FOUND;
}
lastState = state;
l = u;
}
}
return ENGINE_NO_MATCH;
}
} else {
return ENGINE_MATCH_FOUND;
}
} else if (NULL != p->usearch) {
if (subject->len > 0) {
usearch_setText(p->usearch, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "usearch_setText");
return ENGINE_FAILURE;
}
ret = usearch_first(p->usearch, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "usearch_first");
return ENGINE_FAILURE;
}
usearch_unbindText(p->usearch);
return (ret != USEARCH_DONE ? ENGINE_MATCH_FOUND : ENGINE_NO_MATCH);
} else {
return ENGINE_NO_MATCH;
}
} else {
UChar *m;
int32_t pos;
pos = 0;
ret = ENGINE_NO_MATCH;
if (NULL != p->ubrk) {
ubrk_setText(p->ubrk, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "ubrk_setText");
return ENGINE_FAILURE;
}
}
while (NULL != (m = u_strFindFirst(subject->ptr + pos, subject->len - pos, p->pattern->ptr, p->pattern->len))) {
pos = m - subject->ptr;
if (NULL == p->ubrk || (ubrk_isBoundary(p->ubrk, pos) && ubrk_isBoundary(p->ubrk, pos + p->pattern->len))) {
ret = ENGINE_MATCH_FOUND;
}
pos += p->pattern->len;
}
ubrk_unbindText(p->ubrk);
return ret;
}
}
static void TestBreakIteratorSafeClone(void)
{
UChar text[51]; /* Keep this odd to test for 64-bit memory alignment */
/* NOTE: This doesn't reliably force mis-alignment of following items. */
uint8_t buffer [CLONETEST_ITERATOR_COUNT] [U_BRK_SAFECLONE_BUFFERSIZE];
int32_t bufferSize = U_BRK_SAFECLONE_BUFFERSIZE;
UBreakIterator * someIterators [CLONETEST_ITERATOR_COUNT];
UBreakIterator * someClonedIterators [CLONETEST_ITERATOR_COUNT];
UBreakIterator * brk;
UErrorCode status = U_ZERO_ERROR;
int32_t start,pos;
int32_t i;
/*Testing ubrk_safeClone */
/* Note: the adjacent "" are concatenating strings, not adding a \" to the
string, which is probably what whoever wrote this intended. Don't fix,
because it would throw off the hard coded break positions in the following
tests. */
u_uastrcpy(text, "He's from Africa. ""Mr. Livingston, I presume?"" Yeah");
/* US & Thai - rule-based & dictionary based */
someIterators[0] = ubrk_open(UBRK_WORD, "en_US", text, u_strlen(text), &status);
if(!someIterators[0] || U_FAILURE(status)) {
log_data_err("Couldn't open en_US word break iterator - %s\n", u_errorName(status));
return;
}
someIterators[1] = ubrk_open(UBRK_WORD, "th_TH", text, u_strlen(text), &status);
if(!someIterators[1] || U_FAILURE(status)) {
log_data_err("Couldn't open th_TH word break iterator - %s\n", u_errorName(status));
return;
}
/* test each type of iterator */
for (i = 0; i < CLONETEST_ITERATOR_COUNT; i++)
{
/* Check the various error & informational states */
/* Null status - just returns NULL */
if (0 != ubrk_safeClone(someIterators[i], buffer[i], &bufferSize, 0))
{
log_err("FAIL: Cloned Iterator failed to deal correctly with null status\n");
}
/* error status - should return 0 & keep error the same */
status = U_MEMORY_ALLOCATION_ERROR;
if (0 != ubrk_safeClone(someIterators[i], buffer[i], &bufferSize, &status) || status != U_MEMORY_ALLOCATION_ERROR)
{
log_err("FAIL: Cloned Iterator failed to deal correctly with incoming error status\n");
}
status = U_ZERO_ERROR;
/* Null buffer size pointer - just returns NULL & set error to U_ILLEGAL_ARGUMENT_ERROR*/
if (0 != ubrk_safeClone(someIterators[i], buffer[i], 0, &status) || status != U_ILLEGAL_ARGUMENT_ERROR)
{
log_err("FAIL: Cloned Iterator failed to deal correctly with null bufferSize pointer\n");
}
status = U_ZERO_ERROR;
/* buffer size pointer is 0 - fill in pbufferSize with a size */
bufferSize = 0;
if (0 != ubrk_safeClone(someIterators[i], buffer[i], &bufferSize, &status) || U_FAILURE(status) || bufferSize <= 0)
{
log_err("FAIL: Cloned Iterator failed a sizing request ('preflighting')\n");
}
/* Verify our define is large enough */
if (U_BRK_SAFECLONE_BUFFERSIZE < bufferSize)
{
log_err("FAIL: Pre-calculated buffer size is too small\n");
}
/* Verify we can use this run-time calculated size */
if (0 == (brk = ubrk_safeClone(someIterators[i], buffer[i], &bufferSize, &status)) || U_FAILURE(status))
{
log_err("FAIL: Iterator can't be cloned with run-time size\n");
}
if (brk)
ubrk_close(brk);
/* size one byte too small - should allocate & let us know */
--bufferSize;
if (0 == (brk = ubrk_safeClone(someIterators[i], 0, &bufferSize, &status)) || status != U_SAFECLONE_ALLOCATED_WARNING)
{
log_err("FAIL: Cloned Iterator failed to deal correctly with too-small buffer size\n");
}
if (brk)
ubrk_close(brk);
status = U_ZERO_ERROR;
bufferSize = U_BRK_SAFECLONE_BUFFERSIZE;
/* Null buffer pointer - return Iterator & set error to U_SAFECLONE_ALLOCATED_ERROR */
if (0 == (brk = ubrk_safeClone(someIterators[i], 0, &bufferSize, &status)) || status != U_SAFECLONE_ALLOCATED_WARNING)
{
log_err("FAIL: Cloned Iterator failed to deal correctly with null buffer pointer\n");
}
if (brk)
ubrk_close(brk);
status = U_ZERO_ERROR;
/* Mis-aligned buffer pointer. */
{
char stackBuf[U_BRK_SAFECLONE_BUFFERSIZE+sizeof(void *)];
void *p;
int32_t offset;
brk = ubrk_safeClone(someIterators[i], &stackBuf[1], &bufferSize, &status);
if (U_FAILURE(status) || brk == 0) {
log_err("FAIL: Cloned Iterator failed with misaligned buffer pointer\n");
}
if (status == U_SAFECLONE_ALLOCATED_WARNING) {
log_err("FAIL: Cloned Iterator allocated when using a mis-aligned buffer.\n");
}
offset = (int32_t)((char *)&p-(char*)brk);
if (offset < 0) {
offset = -offset;
}
if (offset % sizeof(void *) != 0) {
log_err("FAIL: Cloned Iterator failed to align correctly with misaligned buffer pointer\n");
}
if (brk)
ubrk_close(brk);
}
/* Null Iterator - return NULL & set U_ILLEGAL_ARGUMENT_ERROR */
if (0 != ubrk_safeClone(0, buffer[i], &bufferSize, &status) || status != U_ILLEGAL_ARGUMENT_ERROR)
{
log_err("FAIL: Cloned Iterator failed to deal correctly with null Iterator pointer\n");
}
status = U_ZERO_ERROR;
/* Do these cloned Iterators work at all - make a first & next call */
bufferSize = U_BRK_SAFECLONE_BUFFERSIZE;
someClonedIterators[i] = ubrk_safeClone(someIterators[i], buffer[i], &bufferSize, &status);
start = ubrk_first(someClonedIterators[i]);
if(start!=0)
log_err("error ubrk_start(clone) did not return 0\n");
pos=ubrk_next(someClonedIterators[i]);
if(pos!=4)
log_err("error ubrk_next(clone) did not return 4\n");
ubrk_close(someClonedIterators[i]);
ubrk_close(someIterators[i]);
}
}
int textBreakFirst(TextBreakIterator* iterator)
{
return ubrk_first(reinterpret_cast<UBreakIterator*>(iterator));
}
/*
* Internal titlecasing function.
*
* Must get titleIter!=NULL.
*/
static int32_t
_toTitle(const UCaseProps *csp,
UChar *dest, int32_t destCapacity,
const UChar *src, UCaseContext *csc,
int32_t srcLength,
UBreakIterator *titleIter,
const char *locale, int32_t *locCache,
UErrorCode *pErrorCode) {
const UChar *s;
UChar32 c;
int32_t prev, titleStart, titleLimit, index, destIndex, length;
UBool isFirstIndex;
/* set up local variables */
destIndex=0;
prev=0;
isFirstIndex=TRUE;
/* titlecasing loop */
while(prev<srcLength) {
/* find next index where to titlecase */
if(isFirstIndex) {
isFirstIndex=FALSE;
index=ubrk_first(titleIter);
} else {
index=ubrk_next(titleIter);
}
if(index==UBRK_DONE || index>srcLength) {
index=srcLength;
}
/*
* Unicode 4 & 5 section 3.13 Default Case Operations:
*
* R3 toTitlecase(X): Find the word boundaries based on Unicode Standard Annex
* #29, "Text Boundaries." Between each pair of word boundaries, find the first
* cased character F. If F exists, map F to default_title(F); then map each
* subsequent character C to default_lower(C).
*
* In this implementation, segment [prev..index[ into 3 parts:
* a) uncased characters (copy as-is) [prev..titleStart[
* b) first case letter (titlecase) [titleStart..titleLimit[
* c) subsequent characters (lowercase) [titleLimit..index[
*/
if(prev<index) {
/* find and copy uncased characters [prev..titleStart[ */
titleStart=titleLimit=prev;
for(;;) {
U16_NEXT(src, titleLimit, srcLength, c);
if(UCASE_NONE!=ucase_getType(csp, c)) {
break; /* cased letter at [titleStart..titleLimit[ */
}
titleStart=titleLimit;
if(titleLimit==index) {
/*
* only uncased characters in [prev..index[
* stop with titleStart==titleLimit==index
*/
break;
}
}
length=titleStart-prev;
if(length>0) {
if((destIndex+length)<=destCapacity) {
uprv_memcpy(dest+destIndex, src+prev, length*U_SIZEOF_UCHAR);
}
destIndex+=length;
}
if(titleStart<titleLimit) {
/* titlecase c which is from [titleStart..titleLimit[ */
csc->cpStart=titleStart;
csc->cpLimit=titleLimit;
c=ucase_toFullTitle(csp, c, utf16_caseContextIterator, csc, &s, locale, locCache);
destIndex=appendResult(dest, destIndex, destCapacity, c, s);
/* lowercase [titleLimit..index[ */
if(titleLimit<index) {
destIndex+=
_caseMap(
csp, ucase_toFullLower,
dest+destIndex, destCapacity-destIndex,
src, csc,
titleLimit, index,
locale, locCache,
pErrorCode);
}
}
}
prev=index;
}
if(destIndex>destCapacity) {
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
}
return destIndex;
}
/*
* Internal titlecasing function.
*/
static int32_t
_toTitle(UCaseMap *csm,
UChar *dest, int32_t destCapacity,
const UChar *src, UCaseContext *csc,
int32_t srcLength,
UErrorCode *pErrorCode) {
const UChar *s;
UChar32 c;
int32_t prev, titleStart, titleLimit, idx, destIndex, length;
UBool isFirstIndex;
if(csm->iter!=NULL) {
ubrk_setText(csm->iter, src, srcLength, pErrorCode);
} else {
csm->iter=ubrk_open(UBRK_WORD, csm->locale,
src, srcLength,
pErrorCode);
}
if(U_FAILURE(*pErrorCode)) {
return 0;
}
/* set up local variables */
destIndex=0;
prev=0;
isFirstIndex=TRUE;
/* titlecasing loop */
while(prev<srcLength) {
/* find next index where to titlecase */
if(isFirstIndex) {
isFirstIndex=FALSE;
idx=ubrk_first(csm->iter);
} else {
idx=ubrk_next(csm->iter);
}
if(idx==UBRK_DONE || idx>srcLength) {
idx=srcLength;
}
/*
* Unicode 4 & 5 section 3.13 Default Case Operations:
*
* R3 toTitlecase(X): Find the word boundaries based on Unicode Standard Annex
* #29, "Text Boundaries." Between each pair of word boundaries, find the first
* cased character F. If F exists, map F to default_title(F); then map each
* subsequent character C to default_lower(C).
*
* In this implementation, segment [prev..index[ into 3 parts:
* a) uncased characters (copy as-is) [prev..titleStart[
* b) first case letter (titlecase) [titleStart..titleLimit[
* c) subsequent characters (lowercase) [titleLimit..index[
*/
if(prev<idx) {
/* find and copy uncased characters [prev..titleStart[ */
titleStart=titleLimit=prev;
U16_NEXT(src, titleLimit, idx, c);
if((csm->options&U_TITLECASE_NO_BREAK_ADJUSTMENT)==0 && UCASE_NONE==ucase_getType(csm->csp, c)) {
/* Adjust the titlecasing index (titleStart) to the next cased character. */
for(;;) {
titleStart=titleLimit;
if(titleLimit==idx) {
/*
* only uncased characters in [prev..index[
* stop with titleStart==titleLimit==index
*/
break;
}
U16_NEXT(src, titleLimit, idx, c);
if(UCASE_NONE!=ucase_getType(csm->csp, c)) {
break; /* cased letter at [titleStart..titleLimit[ */
}
}
length=titleStart-prev;
if(length>0) {
if((destIndex+length)<=destCapacity) {
uprv_memcpy(dest+destIndex, src+prev, length*U_SIZEOF_UCHAR);
}
destIndex+=length;
}
}
if(titleStart<titleLimit) {
/* titlecase c which is from [titleStart..titleLimit[ */
csc->cpStart=titleStart;
csc->cpLimit=titleLimit;
c=ucase_toFullTitle(csm->csp, c, utf16_caseContextIterator, csc, &s, csm->locale, &csm->locCache);
destIndex=appendResult(dest, destIndex, destCapacity, c, s);
/* Special case Dutch IJ titlecasing */
if ( titleStart+1 < idx &&
ucase_getCaseLocale(csm->locale,&csm->locCache) == UCASE_LOC_DUTCH &&
( src[titleStart] == (UChar32) 0x0049 || src[titleStart] == (UChar32) 0x0069 ) &&
( src[titleStart+1] == (UChar32) 0x004A || src[titleStart+1] == (UChar32) 0x006A )) {
c=(UChar32) 0x004A;
destIndex=appendResult(dest, destIndex, destCapacity, c, s);
titleLimit++;
}
/* lowercase [titleLimit..index[ */
if(titleLimit<idx) {
if((csm->options&U_TITLECASE_NO_LOWERCASE)==0) {
/* Normal operation: Lowercase the rest of the word. */
destIndex+=
_caseMap(
csm, ucase_toFullLower,
dest+destIndex, destCapacity-destIndex,
src, csc,
titleLimit, idx,
pErrorCode);
} else {
/* Optionally just copy the rest of the word unchanged. */
length=idx-titleLimit;
if((destIndex+length)<=destCapacity) {
uprv_memcpy(dest+destIndex, src+titleLimit, length*U_SIZEOF_UCHAR);
}
destIndex+=length;
}
}
}
}
prev=idx;
}
if(destIndex>destCapacity) {
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
}
return destIndex;
}
/*
* imp: common/ubrk.cpp
* hdr: common/unicode/ubrk.h
* @stable ICU 2.0
* #if !UCONFIG_NO_BREAK_ITERATION
* (don't actually conditionalize this, if the underlying library is not
* built with break iteration, we want to fail at build time, not runtime)
*/
U_CAPI int32_t U_EXPORT2
ubrk_first_4_0(UBreakIterator *bi)
{
return ubrk_first(bi);
}
static jint firstImpl(JNIEnv*, jclass, jint address) {
return ubrk_first(breakIterator(address));
}
static void TestBreakIteratorCAPI()
{
UErrorCode status = U_ZERO_ERROR;
UBreakIterator *word, *sentence, *line, *character, *b, *bogus;
int32_t start,pos,end,to;
int32_t i;
int32_t count = 0;
UChar text[50];
/* Note: the adjacent "" are concatenating strings, not adding a \" to the
string, which is probably what whoever wrote this intended. Don't fix,
because it would throw off the hard coded break positions in the following
tests. */
u_uastrcpy(text, "He's from Africa. ""Mr. Livingston, I presume?"" Yeah");
/*test ubrk_open()*/
log_verbose("\nTesting BreakIterator open functions\n");
/* Use french for fun */
word = ubrk_open(UBRK_WORD, "en_US", text, u_strlen(text), &status);
if(status == U_FILE_ACCESS_ERROR) {
log_data_err("Check your data - it doesn't seem to be around\n");
return;
} else if(U_FAILURE(status)){
log_err_status(status, "FAIL: Error in ubrk_open() for word breakiterator: %s\n", myErrorName(status));
}
else{
log_verbose("PASS: Successfully opened word breakiterator\n");
}
sentence = ubrk_open(UBRK_SENTENCE, "en_US", text, u_strlen(text), &status);
if(U_FAILURE(status)){
log_err_status(status, "FAIL: Error in ubrk_open() for sentence breakiterator: %s\n", myErrorName(status));
return;
}
else{
log_verbose("PASS: Successfully opened sentence breakiterator\n");
}
line = ubrk_open(UBRK_LINE, "en_US", text, u_strlen(text), &status);
if(U_FAILURE(status)){
log_err("FAIL: Error in ubrk_open() for line breakiterator: %s\n", myErrorName(status));
return;
}
else{
log_verbose("PASS: Successfully opened line breakiterator\n");
}
character = ubrk_open(UBRK_CHARACTER, "en_US", text, u_strlen(text), &status);
if(U_FAILURE(status)){
log_err("FAIL: Error in ubrk_open() for character breakiterator: %s\n", myErrorName(status));
return;
}
else{
log_verbose("PASS: Successfully opened character breakiterator\n");
}
/*trying to open an illegal iterator*/
bogus = ubrk_open((UBreakIteratorType)5, "en_US", text, u_strlen(text), &status);
if(U_SUCCESS(status)){
log_err("FAIL: Error in ubrk_open() for BOGUS breakiterator. Expected U_ILLEGAL_ARGUMENT_ERROR\n");
}
if(U_FAILURE(status)){
if(status != U_ILLEGAL_ARGUMENT_ERROR){
log_err("FAIL: Error in ubrk_open() for BOGUS breakiterator. Expected U_ILLEGAL_ARGUMENT_ERROR\n Got %s\n", myErrorName(status));
}
}
status=U_ZERO_ERROR;
/* ======= Test ubrk_countAvialable() and ubrk_getAvialable() */
log_verbose("\nTesting ubrk_countAvailable() and ubrk_getAvailable()\n");
count=ubrk_countAvailable();
/* use something sensible w/o hardcoding the count */
if(count < 0){
log_err("FAIL: Error in ubrk_countAvialable() returned %d\n", count);
}
else{
log_verbose("PASS: ubrk_countAvialable() successful returned %d\n", count);
}
for(i=0;i<count;i++)
{
log_verbose("%s\n", ubrk_getAvailable(i));
if (ubrk_getAvailable(i) == 0)
log_err("No locale for which breakiterator is applicable\n");
else
log_verbose("A locale %s for which breakiterator is applicable\n",ubrk_getAvailable(i));
}
/*========Test ubrk_first(), ubrk_last()...... and other functions*/
log_verbose("\nTesting the functions for word\n");
start = ubrk_first(word);
if(start!=0)
log_err("error ubrk_start(word) did not return 0\n");
log_verbose("first (word = %d\n", (int32_t)start);
pos=ubrk_next(word);
if(pos!=4)
log_err("error ubrk_next(word) did not return 4\n");
log_verbose("next (word = %d\n", (int32_t)pos);
pos=ubrk_following(word, 4);
if(pos!=5)
log_err("error ubrl_following(word,4) did not return 6\n");
log_verbose("next (word = %d\n", (int32_t)pos);
end=ubrk_last(word);
if(end!=49)
log_err("error ubrk_last(word) did not return 49\n");
log_verbose("last (word = %d\n", (int32_t)end);
pos=ubrk_previous(word);
log_verbose("%d %d\n", end, pos);
pos=ubrk_previous(word);
log_verbose("%d \n", pos);
if (ubrk_isBoundary(word, 2) != FALSE) {
log_err("error ubrk_isBoundary(word, 2) did not return FALSE\n");
}
pos=ubrk_current(word);
if (pos != 4) {
log_err("error ubrk_current() != 4 after ubrk_isBoundary(word, 2)\n");
}
if (ubrk_isBoundary(word, 4) != TRUE) {
log_err("error ubrk_isBoundary(word, 4) did not return TRUE\n");
}
log_verbose("\nTesting the functions for character\n");
ubrk_first(character);
pos = ubrk_following(character, 5);
if(pos!=6)
log_err("error ubrk_following(character,5) did not return 6\n");
log_verbose("Following (character,5) = %d\n", (int32_t)pos);
pos=ubrk_following(character, 18);
if(pos!=19)
log_err("error ubrk_following(character,18) did not return 19\n");
log_verbose("Followingcharacter,18) = %d\n", (int32_t)pos);
pos=ubrk_preceding(character, 22);
if(pos!=21)
log_err("error ubrk_preceding(character,22) did not return 21\n");
log_verbose("preceding(character,22) = %d\n", (int32_t)pos);
log_verbose("\nTesting the functions for line\n");
pos=ubrk_first(line);
if(pos != 0)
log_err("error ubrk_first(line) returned %d, expected 0\n", (int32_t)pos);
pos = ubrk_next(line);
pos=ubrk_following(line, 18);
if(pos!=22)
log_err("error ubrk_following(line) did not return 22\n");
log_verbose("following (line) = %d\n", (int32_t)pos);
log_verbose("\nTesting the functions for sentence\n");
ubrk_first(sentence);
pos = ubrk_current(sentence);
log_verbose("Current(sentence) = %d\n", (int32_t)pos);
pos = ubrk_last(sentence);
if(pos!=49)
log_err("error ubrk_last for sentence did not return 49\n");
log_verbose("Last (sentence) = %d\n", (int32_t)pos);
ubrk_first(sentence);
to = ubrk_following( sentence, 0 );
if (to == 0) log_err("ubrk_following returned 0\n");
to = ubrk_preceding( sentence, to );
if (to != 0) log_err("ubrk_preceding didn't return 0\n");
if (ubrk_first(sentence)!=ubrk_current(sentence)) {
log_err("error in ubrk_first() or ubrk_current()\n");
}
/*---- */
/*Testing ubrk_open and ubrk_close()*/
log_verbose("\nTesting open and close for us locale\n");
b = ubrk_open(UBRK_WORD, "fr_FR", text, u_strlen(text), &status);
if (U_FAILURE(status)) {
log_err("ubrk_open for word returned NULL: %s\n", myErrorName(status));
}
ubrk_close(b);
/* Test setText and setUText */
{
UChar s1[] = {0x41, 0x42, 0x20, 0};
UChar s2[] = {0x41, 0x42, 0x43, 0x44, 0x45, 0};
UText *ut = NULL;
UBreakIterator *bb;
int j;
log_verbose("\nTesting ubrk_setText() and ubrk_setUText()\n");
status = U_ZERO_ERROR;
bb = ubrk_open(UBRK_WORD, "en_US", NULL, 0, &status);
TEST_ASSERT_SUCCESS(status);
ubrk_setText(bb, s1, -1, &status);
TEST_ASSERT_SUCCESS(status);
ubrk_first(bb);
j = ubrk_next(bb);
TEST_ASSERT(j == 2);
ut = utext_openUChars(ut, s2, -1, &status);
ubrk_setUText(bb, ut, &status);
TEST_ASSERT_SUCCESS(status);
j = ubrk_next(bb);
TEST_ASSERT(j == 5);
ubrk_close(bb);
utext_close(ut);
}
ubrk_close(word);
ubrk_close(sentence);
ubrk_close(line);
ubrk_close(character);
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is pInput[0..nBytes-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int icuOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *zInput, /* Input string */
int nInput, /* Length of zInput in bytes */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
IcuCursor *pCsr;
const int32_t opt = U_FOLD_CASE_DEFAULT;
UErrorCode status = U_ZERO_ERROR;
int nChar;
UChar32 c;
int iInput = 0;
int iOut = 0;
*ppCursor = 0;
if( nInput<0 ){
nInput = strlen(zInput);
}
nChar = nInput+1;
pCsr = (IcuCursor *)sqlite3_malloc(
sizeof(IcuCursor) + /* IcuCursor */
nChar * sizeof(UChar) + /* IcuCursor.aChar[] */
(nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */
);
if( !pCsr ){
return SQLITE_NOMEM;
}
memset(pCsr, 0, sizeof(IcuCursor));
pCsr->aChar = (UChar *)&pCsr[1];
pCsr->aOffset = (int *)&pCsr->aChar[nChar];
pCsr->aOffset[iOut] = iInput;
U8_NEXT(zInput, iInput, nInput, c);
while( c>0 ){
int isError = 0;
c = u_foldCase(c, opt);
U16_APPEND(pCsr->aChar, iOut, nChar, c, isError);
if( isError ){
sqlite3_free(pCsr);
return SQLITE_ERROR;
}
pCsr->aOffset[iOut] = iInput;
if( iInput<nInput ){
U8_NEXT(zInput, iInput, nInput, c);
}else{
c = 0;
}
}
pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status);
if( !U_SUCCESS(status) ){
sqlite3_free(pCsr);
return SQLITE_ERROR;
}
pCsr->nChar = iOut;
ubrk_first(pCsr->pIter);
*ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
return SQLITE_OK;
}
int32_t __hs_ubrk_first(UBreakIterator *bi)
{
return ubrk_first(bi);
}