ERL_NIF_TERM split(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
{
ErlNifBinary in;
cloner* ptr;
UBreakIterator* iter;
int len = -1, last, pos, is_valid;
UErrorCode status = U_ZERO_ERROR;
ERL_NIF_TERM head, tail;
UChar* bin;
UChar* text;
if (argc != 2)
return enif_make_badarg(env);
/* Last argument must be a binary */
if (!(enif_inspect_binary(env, argv[1], &in)
&& enif_get_resource(env, argv[0], iterator_type, (void**) &ptr))) {
return enif_make_badarg(env);
}
iter = (UBreakIterator*) cloner_get(ptr);
CHECK_RES(env, iter);
if (iter == NULL) {
return enif_make_badarg(env);
}
text = (UChar*) in.data;
ubrk_setText(iter, text, TO_ULEN(in.size), &status);
CHECK(env, status);
tail = enif_make_list(env, 0);
pos = (int) ubrk_last(iter);
while (pos) {
last = pos;
is_valid = is_valid_elem(ptr, iter);
/* get the next elem. */
pos = (int) ubrk_previous(iter);
if (pos == UBRK_DONE)
pos = 0;
if (is_valid) /* Is the old element valid? */
{
len = FROM_ULEN(last - pos);
bin = (UChar*) enif_make_new_binary(env, len, &head);
memcpy(bin,
(const char*) (text + pos),
len);
tail = enif_make_list_cell(env, head, tail);
}
};
return tail;
}
static void setTextImpl(JNIEnv* env, jclass, jint address, jstring text) {
UErrorCode status = U_ZERO_ERROR;
const UChar* chars = env->GetStringChars(text, NULL);
ubrk_setText(breakIterator(address), chars, env->GetStringLength(text), &status);
env->ReleaseStringChars(text, chars);
icu4jni_error(env, status);
}
/* {{{ grapheme_split_string: find and optionally return grapheme boundaries */
int grapheme_split_string(const UChar *text, int32_t text_length, int boundary_array[], int boundary_array_len TSRMLS_DC )
{
unsigned char u_break_iterator_buffer[U_BRK_SAFECLONE_BUFFERSIZE];
UErrorCode status = U_ZERO_ERROR;
int ret_len, pos;
UBreakIterator* bi;
bi = grapheme_get_break_iterator((void*)u_break_iterator_buffer, &status TSRMLS_CC );
if( U_FAILURE(status) ) {
return -1;
}
ubrk_setText(bi, text, text_length, &status);
pos = 0;
for ( ret_len = 0; pos != UBRK_DONE; ) {
pos = ubrk_next(bi);
if ( pos != UBRK_DONE ) {
if ( NULL != boundary_array && ret_len < boundary_array_len ) {
boundary_array[ret_len] = pos;
}
ret_len++;
}
}
ubrk_close(bi);
return ret_len;
}
//------------------------------------------------------------------------------
//
// ubrk_openRules open a break iterator from a set of break rules.
// Invokes the rule builder.
//
//------------------------------------------------------------------------------
U_CAPI UBreakIterator* U_EXPORT2
ubrk_openRules( const UChar *rules,
int32_t rulesLength,
const UChar *text,
int32_t textLength,
UParseError *parseErr,
UErrorCode *status) {
if (status == NULL || U_FAILURE(*status)){
return 0;
}
BreakIterator *result = 0;
UnicodeString ruleString(rules, rulesLength);
result = RBBIRuleBuilder::createRuleBasedBreakIterator(ruleString, parseErr, *status);
if(U_FAILURE(*status)) {
return 0;
}
UBreakIterator *uBI = (UBreakIterator *)result;
if (text != NULL) {
ubrk_setText(uBI, text, textLength, status);
}
return uBI;
}
static void NativeBreakIterator_setTextImpl(JNIEnv* env, jclass, jint address, jstring javaText) {
ScopedJavaUnicodeString text(env, javaText);
UnicodeString& s(text.unicodeString());
UErrorCode status = U_ZERO_ERROR;
ubrk_setText(breakIterator(address), s.getBuffer(), s.length(), &status);
icu4jni_error(env, status);
}
static TextBreakIterator* setTextForIterator(TextBreakIterator& iterator, StringView string)
{
if (string.is8Bit()) {
UTextWithBuffer textLocal;
textLocal.text = UTEXT_INITIALIZER;
textLocal.text.extraSize = sizeof(textLocal.buffer);
textLocal.text.pExtra = textLocal.buffer;
UErrorCode openStatus = U_ZERO_ERROR;
UText* text = openLatin1UTextProvider(&textLocal, string.characters8(), string.length(), &openStatus);
if (U_FAILURE(openStatus)) {
LOG_ERROR("uTextOpenLatin1 failed with status %d", openStatus);
return nullptr;
}
UErrorCode setTextStatus = U_ZERO_ERROR;
ubrk_setUText(reinterpret_cast<UBreakIterator*>(&iterator), text, &setTextStatus);
if (U_FAILURE(setTextStatus)) {
LOG_ERROR("ubrk_setUText failed with status %d", setTextStatus);
return nullptr;
}
utext_close(text);
} else {
UErrorCode setTextStatus = U_ZERO_ERROR;
ubrk_setText(reinterpret_cast<UBreakIterator*>(&iterator), string.characters16(), string.length(), &setTextStatus);
if (U_FAILURE(setTextStatus))
return nullptr;
}
return &iterator;
}
/*
* imp: common/ubrk.cpp
* hdr: common/unicode/ubrk.h
* @stable ICU 2.0
* #if !UCONFIG_NO_BREAK_ITERATION
* (don't actually conditionalize this, if the underlying library is not
* built with break iteration, we want to fail at build time, not runtime)
*/
U_CAPI void U_EXPORT2
ubrk_setText_4_0(UBreakIterator* bi,
const UChar* text,
int32_t textLength,
UErrorCode* status)
{
return ubrk_setText(bi, text, textLength, status);
}
static int32_t U_CALLCONV
unistr_case_internalToTitle(const UCaseMap *csm,
UChar *dest, int32_t destCapacity,
const UChar *src, int32_t srcLength,
UErrorCode *pErrorCode) {
ubrk_setText(csm->iter, src, srcLength, pErrorCode);
return ustrcase_internalToTitle(csm, dest, destCapacity, src, srcLength, pErrorCode);
}
//------------------------------------------------------------------------------
//
// ubrk_open Create a canned type of break iterator based on type (word, line, etc.)
// and locale.
//
//------------------------------------------------------------------------------
U_CAPI UBreakIterator* U_EXPORT2
ubrk_open(UBreakIteratorType type,
const char *locale,
const UChar *text,
int32_t textLength,
UErrorCode *status)
{
if(U_FAILURE(*status)) return 0;
BreakIterator *result = 0;
switch(type) {
case UBRK_CHARACTER:
result = BreakIterator::createCharacterInstance(Locale(locale), *status);
break;
case UBRK_WORD:
result = BreakIterator::createWordInstance(Locale(locale), *status);
break;
case UBRK_LINE:
result = BreakIterator::createLineInstance(Locale(locale), *status);
break;
case UBRK_SENTENCE:
result = BreakIterator::createSentenceInstance(Locale(locale), *status);
break;
case UBRK_TITLE:
result = BreakIterator::createTitleInstance(Locale(locale), *status);
break;
default:
*status = U_ILLEGAL_ARGUMENT_ERROR;
}
// check for allocation error
if (U_FAILURE(*status)) {
return 0;
}
if(result == 0) {
*status = U_MEMORY_ALLOCATION_ERROR;
return 0;
}
UBreakIterator *uBI = (UBreakIterator *)result;
if (text != NULL) {
ubrk_setText(uBI, text, textLength, status);
}
return uBI;
}
TextBreakIterator* acquireLineBreakIterator(const UChar* string, int length, const AtomicString& locale)
{
UBreakIterator* iterator = LineBreakIteratorPool::sharedPool().take(locale);
UErrorCode setTextStatus = U_ZERO_ERROR;
ubrk_setText(iterator, string, length, &setTextStatus);
if (U_FAILURE(setTextStatus)) {
LOG_ERROR("ubrk_setText failed with status %d", setTextStatus);
return 0;
}
return reinterpret_cast<TextBreakIterator*>(iterator);
}
/*
* TestBreakIteratorRules - Verify that a break iterator can be created from
* a set of source rules.
*/
static void TestBreakIteratorRules() {
/* Rules will keep together any run of letters not including 'a', OR
* keep together 'abc', but only when followed by 'def', OTHERWISE
* just return one char at a time.
*/
char rules[] = "abc{666}/def;\n [\\p{L} - [a]]* {2}; . {1};";
/* 0123456789012345678 */
char data[] = "abcdex abcdefgh-def"; /* the test data string */
char breaks[] = "** ** * ** *"; /* * the expected break positions */
char tags[] = "01 21 6 21 2"; /* expected tag values at break positions */
int32_t tagMap[] = {0, 1, 2, 3, 4, 5, 666};
UChar *uData;
void *freeHook = NULL;
UErrorCode status = U_ZERO_ERROR;
int32_t pos;
int i;
UBreakIterator *bi = testOpenRules(rules);
if (bi == NULL) {return;}
uData = toUChar(data, &freeHook);
ubrk_setText(bi, uData, -1, &status);
pos = ubrk_first(bi);
for (i=0; i<sizeof(breaks); i++) {
if (pos == i && breaks[i] != '*') {
log_err("FAIL: unexpected break at position %d found\n", pos);
break;
}
if (pos != i && breaks[i] == '*') {
log_err("FAIL: expected break at position %d not found.\n", i);
break;
}
if (pos == i) {
int32_t tag, expectedTag;
tag = ubrk_getRuleStatus(bi);
expectedTag = tagMap[tags[i]&0xf];
if (tag != expectedTag) {
log_err("FAIL: incorrect tag value. Position = %d; expected tag %d, got %d",
pos, expectedTag, tag);
break;
}
pos = ubrk_next(bi);
}
}
freeToUCharStrings(&freeHook);
ubrk_close(bi);
}
ERL_NIF_TERM split_index(ErlNifEnv* env, int argc,
const ERL_NIF_TERM argv[])
{
ErlNifBinary in;
cloner* ptr;
UBreakIterator* iter;
int pos;
UErrorCode status = U_ZERO_ERROR;
ERL_NIF_TERM head, tail;
if (argc != 2)
return enif_make_badarg(env);
/* Last argument must be a binary */
if (!(enif_inspect_binary(env, argv[1], &in)
&& enif_get_resource(env, argv[0], iterator_type, (void**) &ptr))) {
return enif_make_badarg(env);
}
iter = (UBreakIterator*) cloner_get(ptr);
CHECK_RES(env, iter);
if (iter == NULL) {
return enif_make_badarg(env);
}
ubrk_setText(iter,
(UChar*) in.data,
TO_ULEN(in.size),
&status);
CHECK(env, status);
tail = enif_make_list(env, 0);
pos = (int) ubrk_last(iter);
while ((pos != UBRK_DONE) && (pos != 0)) {
if (is_valid_elem(ptr, iter))
{
head = enif_make_int(env, pos);
tail = enif_make_list_cell(env, head, tail);
}
/* get the next elem. */
pos = (int) ubrk_previous(iter);
};
return tail;
}
U_CAPI int32_t U_EXPORT2
ucasemap_toTitle(UCaseMap *csm,
UChar *dest, int32_t destCapacity,
const UChar *src, int32_t srcLength,
UErrorCode *pErrorCode) {
if(csm->iter!=NULL) {
ubrk_setText(csm->iter, src, srcLength, pErrorCode);
} else {
csm->iter=ubrk_open(UBRK_WORD, csm->locale, src, srcLength, pErrorCode);
}
return ustrcase_map(
csm,
dest, destCapacity,
src, srcLength,
ustrcase_internalToTitle, pErrorCode);
}
MojErr MojDbTextTokenizer::tokenize(const MojString& text, MojDbTextCollator* collator, KeySet& keysOut) const
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(m_ubrk.get());
// convert to UChar from str
MojDbTextUtils::UnicodeVec unicodeStr;
MojErr err = MojDbTextUtils::strToUnicode(text, unicodeStr);
MojErrCheck(err);
// clone break iterator and set text
MojByte buf[U_BRK_SAFECLONE_BUFFERSIZE];
UErrorCode status = U_ZERO_ERROR;
MojInt32 size = sizeof(buf);
IterPtr ubrk(ubrk_safeClone(m_ubrk.get(), buf, &size, &status));
MojUnicodeErrCheck(status);
MojAssert(ubrk.get());
ubrk_setText(ubrk.get(), unicodeStr.begin(), (MojInt32) unicodeStr.size(), &status);
MojUnicodeErrCheck(status);
MojInt32 tokBegin = -1;
MojInt32 pos = ubrk_first(ubrk.get());
while (pos != UBRK_DONE) {
UWordBreak status = (UWordBreak) ubrk_getRuleStatus(ubrk.get());
if (status != UBRK_WORD_NONE) {
MojAssert(tokBegin != -1);
MojDbKey key;
const UChar* tokChars = unicodeStr.begin() + tokBegin;
MojSize tokSize = (MojSize) (pos - tokBegin);
if (collator) {
err = collator->sortKey(tokChars, tokSize, key);
MojErrCheck(err);
} else {
MojString tok;
err = MojDbTextUtils::unicodeToStr(tokChars, tokSize, tok);
MojErrCheck(err);
err = key.assign(tok);
MojErrCheck(err);
}
err = keysOut.put(key);
MojErrCheck(err);
}
tokBegin = pos;
pos = ubrk_next(ubrk.get());
}
return MojErrNone;
}
// BreakIterator.set_text {{{
static PyObject *
icu_BreakIterator_set_text(icu_BreakIterator *self, PyObject *input) {
int32_t sz = 0;
UChar *buf = NULL;
UErrorCode status = U_ZERO_ERROR;
buf = python_to_icu(input, &sz);
if (buf == NULL) return NULL;
ubrk_setText(self->break_iterator, buf, sz, &status);
if (U_FAILURE(status)) {
free(buf); buf = NULL;
PyErr_SetString(PyExc_ValueError, u_errorName(status));
} else { self->text = buf; self->text_len = sz; }
Py_RETURN_NONE;
} // }}}
/* {{{ grapheme_get_haystack_offset - bump the haystack pointer based on the grapheme count offset */
UChar *
grapheme_get_haystack_offset(UBreakIterator* bi, UChar *uhaystack, int32_t uhaystack_len, int32_t offset)
{
UErrorCode status;
int32_t pos;
int32_t (*iter_op)(UBreakIterator* bi);
int iter_incr;
if ( NULL != bi ) {
status = U_ZERO_ERROR;
ubrk_setText (bi, uhaystack, uhaystack_len, &status);
}
if ( 0 == offset ) {
return uhaystack;
}
if ( offset < 0 ) {
iter_op = ubrk_previous;
ubrk_last(bi); /* one past the end */
iter_incr = 1;
}
else {
iter_op = ubrk_next;
iter_incr = -1;
}
pos = 0;
while ( pos != UBRK_DONE && offset != 0 ) {
pos = iter_op(bi);
if ( UBRK_DONE != pos ) {
offset += iter_incr;
}
}
if ( offset != 0 ) {
return NULL;
}
return uhaystack + pos;
}
ERL_NIF_TERM len(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
{
ErlNifBinary in;
cloner* ptr;
UBreakIterator* iter;
int count = 0, pos;
UErrorCode status = U_ZERO_ERROR;
if (argc != 2)
return enif_make_badarg(env);
/* Last argument must be a binary */
if (!(enif_inspect_binary(env, argv[1], &in)
&& enif_get_resource(env, argv[0], iterator_type, (void**) &ptr))) {
return enif_make_badarg(env);
}
iter = (UBreakIterator*) cloner_get(ptr);
CHECK_RES(env, iter);
if (iter == NULL) {
return enif_make_badarg(env);
}
/* Do count */
ubrk_setText(iter,
(UChar *) in.data,
TO_ULEN(in.size),
&status);
CHECK(env, status);
pos = ubrk_first(iter);
if (pos != UBRK_DONE)
while (1) {
pos = ubrk_next(iter);
if (pos == UBRK_DONE)
break;
if (is_valid_elem(ptr, iter))
count++;
}
return enif_make_int(env, count);
}
int32_t grapheme_count(UBreakIterator *ubrk, const UString *ustr)
{
int32_t i, count;
UErrorCode status;
count = 0;
status = U_ZERO_ERROR;
ubrk_setText(ubrk, ustr->ptr, ustr->len, &status);
if (U_FAILURE(status)) {
return -1;
}
if (UBRK_DONE != (i = ubrk_first(ubrk))) {
while (UBRK_DONE != (i = ubrk_next(ubrk))) {
++count;
}
}
ubrk_unbindText(ubrk);
return count;
}
/* {{{ grapheme_count_graphemes */
int32_t grapheme_count_graphemes(UBreakIterator *bi, UChar *string, int32_t string_len)
{
int ret_len = 0;
int pos = 0;
UErrorCode status = U_ZERO_ERROR;
ubrk_setText(bi, string, string_len, &status);
do {
pos = ubrk_next(bi);
if ( UBRK_DONE != pos ) {
ret_len++;
}
} while ( UBRK_DONE != pos );
return ret_len;
}
/* {{{ grapheme_memnstr_grapheme: find needle in haystack using grapheme boundaries */
inline int32_t
grapheme_memnstr_grapheme(UBreakIterator *bi, UChar *haystack, UChar *needle, int32_t needle_len, UChar *end)
{
UChar *p = haystack;
UChar ne = needle[needle_len-1];
UErrorCode status;
int32_t grapheme_offset;
end -= needle_len;
while (p <= end) {
if ((p = u_memchr(p, *needle, (end-p+1))) && ne == p[needle_len-1]) {
if (!u_memcmp(needle, p, needle_len - 1)) { /* needle_len - 1 works because if needle_len is 1, we've already tested the char */
/* does the grapheme end here? */
status = U_ZERO_ERROR;
ubrk_setText (bi, haystack, (end - haystack) + needle_len, &status);
if ( ubrk_isBoundary (bi, (p - haystack) + needle_len) ) {
/* found it, get grapheme count offset */
grapheme_offset = grapheme_count_graphemes(bi, haystack, (p - haystack));
return grapheme_offset;
}
}
}
if (p == NULL) {
return -1;
}
p++;
}
return -1;
}
static TextBreakIterator* setUpIterator(bool& createdIterator, TextBreakIterator*& iterator,
UBreakIteratorType type, const UChar* string, int length)
{
if (!string)
return 0;
if (!createdIterator) {
UErrorCode openStatus = U_ZERO_ERROR;
iterator = reinterpret_cast<TextBreakIterator*>(ubrk_open(type, currentTextBreakLocaleID(), 0, 0, &openStatus));
createdIterator = true;
ASSERT_WITH_MESSAGE(U_SUCCESS(openStatus), "ICU could not open a break iterator: %s (%d)", u_errorName(openStatus), openStatus);
}
if (!iterator)
return 0;
UErrorCode setTextStatus = U_ZERO_ERROR;
ubrk_setText(reinterpret_cast<UBreakIterator*>(iterator), string, length, &setTextStatus);
if (U_FAILURE(setTextStatus))
return 0;
return iterator;
}
U_CAPI int32_t U_EXPORT2
u_strToTitle(UChar *dest, int32_t destCapacity,
const UChar *src, int32_t srcLength,
UBreakIterator *titleIter,
const char *locale,
UErrorCode *pErrorCode) {
UCaseMap csm=UCASEMAP_INITIALIZER;
setTempCaseMap(&csm, locale);
if(titleIter!=NULL) {
ubrk_setText(csm.iter=titleIter, src, srcLength, pErrorCode);
} else {
csm.iter=ubrk_open(UBRK_WORD, csm.locale, src, srcLength, pErrorCode);
}
int32_t length=ustrcase_map(
&csm,
dest, destCapacity,
src, srcLength,
ustrcase_internalToTitle, pErrorCode);
if(titleIter==NULL && csm.iter!=NULL) {
ubrk_close(csm.iter);
}
return length;
}
static TextBreakIterator* setUpIteratorWithRules(bool& createdIterator, TextBreakIterator*& iterator,
const char* breakRules, const UChar* string, int length)
{
if (!string)
return 0;
if (!createdIterator) {
UParseError parseStatus;
UErrorCode openStatus = U_ZERO_ERROR;
String rules(breakRules);
iterator = reinterpret_cast<TextBreakIterator*>(ubrk_openRules(rules.characters(), rules.length(), 0, 0, &parseStatus, &openStatus));
createdIterator = true;
ASSERT_WITH_MESSAGE(U_SUCCESS(openStatus), "ICU could not open a break iterator: %s (%d)", u_errorName(openStatus), openStatus);
}
if (!iterator)
return 0;
UErrorCode setTextStatus = U_ZERO_ERROR;
ubrk_setText(reinterpret_cast<UBreakIterator*>(iterator), string, length, &setTextStatus);
if (U_FAILURE(setTextStatus))
return 0;
return iterator;
}
/**
* Don't modify this function, it reproduces a part of fixed engine (engine_fixed_split)
**/
static UBool engine_bin_split(error_t **error, void *data, const UString *subject, DArray *array, interval_list_t *intervals)
{
UErrorCode status;
int32_t l, lastU;
dlist_element_t *el;
FETCH_DATA(data, p, bin_pattern_t);
lastU = l = 0;
status = U_ZERO_ERROR;
CASE_FOLD_FORBIDDEN(error, p, FALSE); /* the only engine specific thing */
if (NULL != p->ubrk) {
ubrk_setText(p->ubrk, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "ubrk_setText");
return FALSE;
}
}
for (el = intervals->head; NULL != el; el = el->next) {
FETCH_DATA(el->data, i, interval_t);
if (i->lower_limit > 0) {
if (!binary_fwd_n(p->ubrk, p->pattern, subject, NULL, i->lower_limit - lastU, &l)) {
break;
}
}
if (!binary_fwd_n(p->ubrk, p->pattern, subject, array, i->upper_limit - i->lower_limit, &l)) {
break;
}
lastU = i->upper_limit;
}
ubrk_unbindText(p->ubrk);
return TRUE;
}
/*
* Internal titlecasing function.
*/
static int32_t
_toTitle(UCaseMap *csm,
UChar *dest, int32_t destCapacity,
const UChar *src, UCaseContext *csc,
int32_t srcLength,
UErrorCode *pErrorCode) {
const UChar *s;
UChar32 c;
int32_t prev, titleStart, titleLimit, idx, destIndex, length;
UBool isFirstIndex;
if(csm->iter!=NULL) {
ubrk_setText(csm->iter, src, srcLength, pErrorCode);
} else {
csm->iter=ubrk_open(UBRK_WORD, csm->locale,
src, srcLength,
pErrorCode);
}
if(U_FAILURE(*pErrorCode)) {
return 0;
}
/* set up local variables */
destIndex=0;
prev=0;
isFirstIndex=TRUE;
/* titlecasing loop */
while(prev<srcLength) {
/* find next index where to titlecase */
if(isFirstIndex) {
isFirstIndex=FALSE;
idx=ubrk_first(csm->iter);
} else {
idx=ubrk_next(csm->iter);
}
if(idx==UBRK_DONE || idx>srcLength) {
idx=srcLength;
}
/*
* Unicode 4 & 5 section 3.13 Default Case Operations:
*
* R3 toTitlecase(X): Find the word boundaries based on Unicode Standard Annex
* #29, "Text Boundaries." Between each pair of word boundaries, find the first
* cased character F. If F exists, map F to default_title(F); then map each
* subsequent character C to default_lower(C).
*
* In this implementation, segment [prev..index[ into 3 parts:
* a) uncased characters (copy as-is) [prev..titleStart[
* b) first case letter (titlecase) [titleStart..titleLimit[
* c) subsequent characters (lowercase) [titleLimit..index[
*/
if(prev<idx) {
/* find and copy uncased characters [prev..titleStart[ */
titleStart=titleLimit=prev;
U16_NEXT(src, titleLimit, idx, c);
if((csm->options&U_TITLECASE_NO_BREAK_ADJUSTMENT)==0 && UCASE_NONE==ucase_getType(csm->csp, c)) {
/* Adjust the titlecasing index (titleStart) to the next cased character. */
for(;;) {
titleStart=titleLimit;
if(titleLimit==idx) {
/*
* only uncased characters in [prev..index[
* stop with titleStart==titleLimit==index
*/
break;
}
U16_NEXT(src, titleLimit, idx, c);
if(UCASE_NONE!=ucase_getType(csm->csp, c)) {
break; /* cased letter at [titleStart..titleLimit[ */
}
}
length=titleStart-prev;
if(length>0) {
if((destIndex+length)<=destCapacity) {
uprv_memcpy(dest+destIndex, src+prev, length*U_SIZEOF_UCHAR);
}
destIndex+=length;
}
}
if(titleStart<titleLimit) {
/* titlecase c which is from [titleStart..titleLimit[ */
csc->cpStart=titleStart;
csc->cpLimit=titleLimit;
c=ucase_toFullTitle(csm->csp, c, utf16_caseContextIterator, csc, &s, csm->locale, &csm->locCache);
destIndex=appendResult(dest, destIndex, destCapacity, c, s);
/* Special case Dutch IJ titlecasing */
if ( titleStart+1 < idx &&
ucase_getCaseLocale(csm->locale,&csm->locCache) == UCASE_LOC_DUTCH &&
( src[titleStart] == (UChar32) 0x0049 || src[titleStart] == (UChar32) 0x0069 ) &&
( src[titleStart+1] == (UChar32) 0x004A || src[titleStart+1] == (UChar32) 0x006A )) {
c=(UChar32) 0x004A;
destIndex=appendResult(dest, destIndex, destCapacity, c, s);
titleLimit++;
}
/* lowercase [titleLimit..index[ */
if(titleLimit<idx) {
if((csm->options&U_TITLECASE_NO_LOWERCASE)==0) {
/* Normal operation: Lowercase the rest of the word. */
destIndex+=
_caseMap(
csm, ucase_toFullLower,
dest+destIndex, destCapacity-destIndex,
src, csc,
titleLimit, idx,
pErrorCode);
} else {
/* Optionally just copy the rest of the word unchanged. */
length=idx-titleLimit;
if((destIndex+length)<=destCapacity) {
uprv_memcpy(dest+destIndex, src+titleLimit, length*U_SIZEOF_UCHAR);
}
destIndex+=length;
}
}
}
}
prev=idx;
}
if(destIndex>destCapacity) {
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
}
return destIndex;
}
static UBool engine_fixed_split(error_t **error, void *data, const UString *subject, DArray *array, interval_list_t *intervals)
{
UErrorCode status;
int32_t l, lastU;
dlist_element_t *el;
FETCH_DATA(data, p, fixed_pattern_t);
lastU = l = 0;
status = U_ZERO_ERROR;
if (NULL != p->usearch) {
usearch_setText(p->usearch, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "usearch_setText");
return FALSE;
}
/* <X> */
if (NULL == intervals) {
int32_t u;
while (U_SUCCESS(status) && USEARCH_DONE != (u = usearch_next(p->usearch, &status))) {
add_match(array, subject, l, u);
l = u += usearch_getMatchedLength(p->usearch);
}
add_match(array, subject, l, subject->len);
} else {
/* </X> */
for (el = intervals->head; NULL != el; el = el->next) {
FETCH_DATA(el->data, i, interval_t);
if (i->lower_limit > 0) {
if (!usearch_fwd_n(p->usearch, subject, NULL, i->lower_limit - lastU, &l, &status)) {
break;
}
}
if (!usearch_fwd_n(p->usearch, subject, array, i->upper_limit - i->lower_limit, &l, &status)) {
break;
}
lastU = i->upper_limit;
}
/* <X> */
}
/* </X> */
usearch_unbindText(p->usearch);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "usearch_next");
return FALSE;
}
} else {
if (NULL != p->ubrk) {
ubrk_setText(p->ubrk, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "ubrk_setText");
return FALSE;
}
}
/* <X> */
if (NULL == intervals) {
UChar *m;
int32_t u;
u = 0;
while (NULL != (m = u_strFindFirst(subject->ptr + u, subject->len - u, p->pattern->ptr, p->pattern->len))) {
u = m - subject->ptr;
if (NULL == p->ubrk || (ubrk_isBoundary(p->ubrk, u) && ubrk_isBoundary(p->ubrk, u + p->pattern->len))) {
add_match(array, subject, l, u);
}
l = u = u + p->pattern->len;
}
add_match(array, subject, l, subject->len);
} else {
/* </X> */
for (el = intervals->head; NULL != el; el = el->next) {
FETCH_DATA(el->data, i, interval_t);
if (i->lower_limit > 0) {
if (!binary_fwd_n(p->ubrk, p->pattern, subject, NULL, i->lower_limit - lastU, &l)) {
break;
}
}
if (!binary_fwd_n(p->ubrk, p->pattern, subject, array, i->upper_limit - i->lower_limit, &l)) {
break;
}
lastU = i->upper_limit;
}
/* <X> */
}
/* </X> */
ubrk_unbindText(p->ubrk);
}
return TRUE;
}
static engine_return_t engine_fixed_match(error_t **error, void *data, const UString *subject)
{
int32_t ret;
UErrorCode status;
FETCH_DATA(data, p, fixed_pattern_t);
status = U_ZERO_ERROR;
if (ustring_empty(p->pattern)) {
if (IS_WORD_BOUNDED(p->flags)) {
if (ustring_empty(subject)) {
return ENGINE_MATCH_FOUND;
} else {
int32_t l, u, lastState, state;
ubrk_setText(p->ubrk, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "ubrk_setText");
return ENGINE_FAILURE;
}
if (UBRK_DONE != (l = ubrk_first(p->ubrk))) {
lastState = ubrk_getRuleStatus(p->ubrk);
while (UBRK_DONE != (u = ubrk_next(p->ubrk))) {
state = ubrk_getRuleStatus(p->ubrk);
if (UBRK_WORD_NONE == lastState && lastState == state) {
return ENGINE_MATCH_FOUND;
}
lastState = state;
l = u;
}
}
return ENGINE_NO_MATCH;
}
} else {
return ENGINE_MATCH_FOUND;
}
} else if (NULL != p->usearch) {
if (subject->len > 0) {
usearch_setText(p->usearch, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "usearch_setText");
return ENGINE_FAILURE;
}
ret = usearch_first(p->usearch, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "usearch_first");
return ENGINE_FAILURE;
}
usearch_unbindText(p->usearch);
return (ret != USEARCH_DONE ? ENGINE_MATCH_FOUND : ENGINE_NO_MATCH);
} else {
return ENGINE_NO_MATCH;
}
} else {
UChar *m;
int32_t pos;
pos = 0;
ret = ENGINE_NO_MATCH;
if (NULL != p->ubrk) {
ubrk_setText(p->ubrk, subject->ptr, subject->len, &status);
if (U_FAILURE(status)) {
icu_error_set(error, FATAL, status, "ubrk_setText");
return ENGINE_FAILURE;
}
}
while (NULL != (m = u_strFindFirst(subject->ptr + pos, subject->len - pos, p->pattern->ptr, p->pattern->len))) {
pos = m - subject->ptr;
if (NULL == p->ubrk || (ubrk_isBoundary(p->ubrk, pos) && ubrk_isBoundary(p->ubrk, pos + p->pattern->len))) {
ret = ENGINE_MATCH_FOUND;
}
pos += p->pattern->len;
}
ubrk_unbindText(p->ubrk);
return ret;
}
}
void swift::__swift_stdlib_ubrk_setText(
swift::__swift_stdlib_UBreakIterator *bi, const __swift_stdlib_UChar *text,
__swift_int32_t textLength, __swift_stdlib_UErrorCode *status) {
return ubrk_setText(ptr_cast<UBreakIterator>(bi), ptr_cast<UChar>(text),
textLength, ptr_cast<UErrorCode>(status));
}
static void TestBreakIteratorCAPI()
{
UErrorCode status = U_ZERO_ERROR;
UBreakIterator *word, *sentence, *line, *character, *b, *bogus;
int32_t start,pos,end,to;
int32_t i;
int32_t count = 0;
UChar text[50];
/* Note: the adjacent "" are concatenating strings, not adding a \" to the
string, which is probably what whoever wrote this intended. Don't fix,
because it would throw off the hard coded break positions in the following
tests. */
u_uastrcpy(text, "He's from Africa. ""Mr. Livingston, I presume?"" Yeah");
/*test ubrk_open()*/
log_verbose("\nTesting BreakIterator open functions\n");
/* Use french for fun */
word = ubrk_open(UBRK_WORD, "en_US", text, u_strlen(text), &status);
if(status == U_FILE_ACCESS_ERROR) {
log_data_err("Check your data - it doesn't seem to be around\n");
return;
} else if(U_FAILURE(status)){
log_err_status(status, "FAIL: Error in ubrk_open() for word breakiterator: %s\n", myErrorName(status));
}
else{
log_verbose("PASS: Successfully opened word breakiterator\n");
}
sentence = ubrk_open(UBRK_SENTENCE, "en_US", text, u_strlen(text), &status);
if(U_FAILURE(status)){
log_err_status(status, "FAIL: Error in ubrk_open() for sentence breakiterator: %s\n", myErrorName(status));
return;
}
else{
log_verbose("PASS: Successfully opened sentence breakiterator\n");
}
line = ubrk_open(UBRK_LINE, "en_US", text, u_strlen(text), &status);
if(U_FAILURE(status)){
log_err("FAIL: Error in ubrk_open() for line breakiterator: %s\n", myErrorName(status));
return;
}
else{
log_verbose("PASS: Successfully opened line breakiterator\n");
}
character = ubrk_open(UBRK_CHARACTER, "en_US", text, u_strlen(text), &status);
if(U_FAILURE(status)){
log_err("FAIL: Error in ubrk_open() for character breakiterator: %s\n", myErrorName(status));
return;
}
else{
log_verbose("PASS: Successfully opened character breakiterator\n");
}
/*trying to open an illegal iterator*/
bogus = ubrk_open((UBreakIteratorType)5, "en_US", text, u_strlen(text), &status);
if(U_SUCCESS(status)){
log_err("FAIL: Error in ubrk_open() for BOGUS breakiterator. Expected U_ILLEGAL_ARGUMENT_ERROR\n");
}
if(U_FAILURE(status)){
if(status != U_ILLEGAL_ARGUMENT_ERROR){
log_err("FAIL: Error in ubrk_open() for BOGUS breakiterator. Expected U_ILLEGAL_ARGUMENT_ERROR\n Got %s\n", myErrorName(status));
}
}
status=U_ZERO_ERROR;
/* ======= Test ubrk_countAvialable() and ubrk_getAvialable() */
log_verbose("\nTesting ubrk_countAvailable() and ubrk_getAvailable()\n");
count=ubrk_countAvailable();
/* use something sensible w/o hardcoding the count */
if(count < 0){
log_err("FAIL: Error in ubrk_countAvialable() returned %d\n", count);
}
else{
log_verbose("PASS: ubrk_countAvialable() successful returned %d\n", count);
}
for(i=0;i<count;i++)
{
log_verbose("%s\n", ubrk_getAvailable(i));
if (ubrk_getAvailable(i) == 0)
log_err("No locale for which breakiterator is applicable\n");
else
log_verbose("A locale %s for which breakiterator is applicable\n",ubrk_getAvailable(i));
}
/*========Test ubrk_first(), ubrk_last()...... and other functions*/
log_verbose("\nTesting the functions for word\n");
start = ubrk_first(word);
if(start!=0)
log_err("error ubrk_start(word) did not return 0\n");
log_verbose("first (word = %d\n", (int32_t)start);
pos=ubrk_next(word);
if(pos!=4)
log_err("error ubrk_next(word) did not return 4\n");
log_verbose("next (word = %d\n", (int32_t)pos);
pos=ubrk_following(word, 4);
if(pos!=5)
log_err("error ubrl_following(word,4) did not return 6\n");
log_verbose("next (word = %d\n", (int32_t)pos);
end=ubrk_last(word);
if(end!=49)
log_err("error ubrk_last(word) did not return 49\n");
log_verbose("last (word = %d\n", (int32_t)end);
pos=ubrk_previous(word);
log_verbose("%d %d\n", end, pos);
pos=ubrk_previous(word);
log_verbose("%d \n", pos);
if (ubrk_isBoundary(word, 2) != FALSE) {
log_err("error ubrk_isBoundary(word, 2) did not return FALSE\n");
}
pos=ubrk_current(word);
if (pos != 4) {
log_err("error ubrk_current() != 4 after ubrk_isBoundary(word, 2)\n");
}
if (ubrk_isBoundary(word, 4) != TRUE) {
log_err("error ubrk_isBoundary(word, 4) did not return TRUE\n");
}
log_verbose("\nTesting the functions for character\n");
ubrk_first(character);
pos = ubrk_following(character, 5);
if(pos!=6)
log_err("error ubrk_following(character,5) did not return 6\n");
log_verbose("Following (character,5) = %d\n", (int32_t)pos);
pos=ubrk_following(character, 18);
if(pos!=19)
log_err("error ubrk_following(character,18) did not return 19\n");
log_verbose("Followingcharacter,18) = %d\n", (int32_t)pos);
pos=ubrk_preceding(character, 22);
if(pos!=21)
log_err("error ubrk_preceding(character,22) did not return 21\n");
log_verbose("preceding(character,22) = %d\n", (int32_t)pos);
log_verbose("\nTesting the functions for line\n");
pos=ubrk_first(line);
if(pos != 0)
log_err("error ubrk_first(line) returned %d, expected 0\n", (int32_t)pos);
pos = ubrk_next(line);
pos=ubrk_following(line, 18);
if(pos!=22)
log_err("error ubrk_following(line) did not return 22\n");
log_verbose("following (line) = %d\n", (int32_t)pos);
log_verbose("\nTesting the functions for sentence\n");
ubrk_first(sentence);
pos = ubrk_current(sentence);
log_verbose("Current(sentence) = %d\n", (int32_t)pos);
pos = ubrk_last(sentence);
if(pos!=49)
log_err("error ubrk_last for sentence did not return 49\n");
log_verbose("Last (sentence) = %d\n", (int32_t)pos);
ubrk_first(sentence);
to = ubrk_following( sentence, 0 );
if (to == 0) log_err("ubrk_following returned 0\n");
to = ubrk_preceding( sentence, to );
if (to != 0) log_err("ubrk_preceding didn't return 0\n");
if (ubrk_first(sentence)!=ubrk_current(sentence)) {
log_err("error in ubrk_first() or ubrk_current()\n");
}
/*---- */
/*Testing ubrk_open and ubrk_close()*/
log_verbose("\nTesting open and close for us locale\n");
b = ubrk_open(UBRK_WORD, "fr_FR", text, u_strlen(text), &status);
if (U_FAILURE(status)) {
log_err("ubrk_open for word returned NULL: %s\n", myErrorName(status));
}
ubrk_close(b);
/* Test setText and setUText */
{
UChar s1[] = {0x41, 0x42, 0x20, 0};
UChar s2[] = {0x41, 0x42, 0x43, 0x44, 0x45, 0};
UText *ut = NULL;
UBreakIterator *bb;
int j;
log_verbose("\nTesting ubrk_setText() and ubrk_setUText()\n");
status = U_ZERO_ERROR;
bb = ubrk_open(UBRK_WORD, "en_US", NULL, 0, &status);
TEST_ASSERT_SUCCESS(status);
ubrk_setText(bb, s1, -1, &status);
TEST_ASSERT_SUCCESS(status);
ubrk_first(bb);
j = ubrk_next(bb);
TEST_ASSERT(j == 2);
ut = utext_openUChars(ut, s2, -1, &status);
ubrk_setUText(bb, ut, &status);
TEST_ASSERT_SUCCESS(status);
j = ubrk_next(bb);
TEST_ASSERT(j == 5);
ubrk_close(bb);
utext_close(ut);
}
ubrk_close(word);
ubrk_close(sentence);
ubrk_close(line);
ubrk_close(character);
}
/* {{{ grapheme_strpos_utf16 - strrpos using utf16*/
int32_t grapheme_strpos_utf16(char *haystack, size_t haystack_len, char *needle, size_t needle_len, int32_t offset, int32_t *puchar_pos, int f_ignore_case, int last)
{
UChar *uhaystack = NULL, *uneedle = NULL;
int32_t uhaystack_len = 0, uneedle_len = 0, char_pos, ret_pos, offset_pos = 0;
unsigned char u_break_iterator_buffer[U_BRK_SAFECLONE_BUFFERSIZE];
UBreakIterator* bi = NULL;
UErrorCode status;
UStringSearch* src = NULL;
UCollator *coll;
if(puchar_pos) {
*puchar_pos = -1;
}
/* convert the strings to UTF-16. */
status = U_ZERO_ERROR;
intl_convert_utf8_to_utf16(&uhaystack, &uhaystack_len, haystack, haystack_len, &status );
STRPOS_CHECK_STATUS(status, "Error converting input string to UTF-16");
status = U_ZERO_ERROR;
intl_convert_utf8_to_utf16(&uneedle, &uneedle_len, needle, needle_len, &status );
STRPOS_CHECK_STATUS(status, "Error converting needle string to UTF-16");
/* get a pointer to the haystack taking into account the offset */
status = U_ZERO_ERROR;
bi = grapheme_get_break_iterator(u_break_iterator_buffer, &status );
STRPOS_CHECK_STATUS(status, "Failed to get iterator");
status = U_ZERO_ERROR;
ubrk_setText(bi, uhaystack, uhaystack_len, &status);
STRPOS_CHECK_STATUS(status, "Failed to set up iterator");
status = U_ZERO_ERROR;
src = usearch_open(uneedle, uneedle_len, uhaystack, uhaystack_len, "", bi, &status);
STRPOS_CHECK_STATUS(status, "Error creating search object");
if(f_ignore_case) {
coll = usearch_getCollator(src);
status = U_ZERO_ERROR;
ucol_setAttribute(coll, UCOL_STRENGTH, UCOL_SECONDARY, &status);
STRPOS_CHECK_STATUS(status, "Error setting collation strength");
usearch_reset(src);
}
if(offset != 0) {
offset_pos = grapheme_get_haystack_offset(bi, offset);
if(offset_pos == -1) {
status = U_ILLEGAL_ARGUMENT_ERROR;
STRPOS_CHECK_STATUS(status, "Invalid search offset");
}
status = U_ZERO_ERROR;
usearch_setOffset(src, offset_pos, &status);
STRPOS_CHECK_STATUS(status, "Invalid search offset");
}
if(last) {
char_pos = usearch_last(src, &status);
if(char_pos < offset_pos) {
/* last one is beyound our start offset */
char_pos = USEARCH_DONE;
}
} else {
char_pos = usearch_next(src, &status);
}
STRPOS_CHECK_STATUS(status, "Error looking up string");
if(char_pos != USEARCH_DONE && ubrk_isBoundary(bi, char_pos)) {
ret_pos = grapheme_count_graphemes(bi, uhaystack,char_pos);
if(puchar_pos) {
*puchar_pos = char_pos;
}
} else {
ret_pos = -1;
}
if (uhaystack) {
efree( uhaystack );
}
if (uneedle) {
efree( uneedle );
}
ubrk_close (bi);
usearch_close (src);
return ret_pos;
}
