unsigned numCodeUnitsInGraphemeClusters(StringView string, unsigned numGraphemeClusters)
{
unsigned stringLength = string.length();
if (stringLength <= numGraphemeClusters)
return stringLength;
// The only Latin-1 Extended Grapheme Cluster is CRLF.
if (string.is8Bit()) {
auto* characters = string.characters8();
unsigned i, j;
for (i = 0, j = 0; i < numGraphemeClusters && j + 1 < stringLength; ++i, ++j)
j += characters[j] == '\r' && characters[j + 1] == '\n';
return j + (i < numGraphemeClusters);
}
NonSharedCharacterBreakIterator iterator { string };
if (!iterator) {
ASSERT_NOT_REACHED();
return stringLength;
}
for (unsigned i = 0; i < numGraphemeClusters; ++i) {
if (ubrk_next(iterator) == UBRK_DONE)
return stringLength;
}
return ubrk_current(iterator);
}
static jint nextImpl(JNIEnv* env, jclass, jint address, jint n) {
UBreakIterator* bi = breakIterator(address);
if (n < 0) {
while (n++ < -1) {
ubrk_previous(bi);
}
return ubrk_previous(bi);
} else if (n == 0) {
return ubrk_current(bi);
} else {
while (n-- > 1) {
ubrk_next(bi);
}
return ubrk_next(bi);
}
return -1;
}
/*
** Extract the next token from a tokenization cursor.
*/
static int icuNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
const char **ppToken, /* OUT: *ppToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
IcuCursor *pCsr = (IcuCursor *)pCursor;
int iStart = 0;
int iEnd = 0;
int nByte = 0;
while( iStart==iEnd ){
UChar32 c;
iStart = ubrk_current(pCsr->pIter);
iEnd = ubrk_next(pCsr->pIter);
if( iEnd==UBRK_DONE ){
return SQLITE_DONE;
}
while( iStart<iEnd ){
int iWhite = iStart;
U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
if( u_isspace(c) ){
iStart = iWhite;
}else{
break;
}
}
assert(iStart<=iEnd);
}
do {
UErrorCode status = U_ZERO_ERROR;
if( nByte ){
char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
if( !zNew ){
return SQLITE_NOMEM;
}
pCsr->zBuffer = zNew;
pCsr->nBuffer = nByte;
}
u_strToUTF8(
pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */
&pCsr->aChar[iStart], iEnd-iStart, /* Input vars */
&status /* Output success/failure */
);
} while( nByte>pCsr->nBuffer );
*ppToken = pCsr->zBuffer;
*pnBytes = nByte;
*piStartOffset = pCsr->aOffset[iStart];
*piEndOffset = pCsr->aOffset[iEnd];
*piPosition = pCsr->iToken++;
return SQLITE_OK;
}
static jint currentImpl(JNIEnv*, jclass, jint address) {
return ubrk_current(breakIterator(address));
}
/* {{{ grapheme_strpos_utf16 - strrpos using utf16*/
int
grapheme_strpos_utf16(unsigned char *haystack, int32_t haystack_len, unsigned char*needle, int32_t needle_len, int32_t offset, int32_t *puchar_pos, int f_ignore_case TSRMLS_DC)
{
UChar *uhaystack, *puhaystack, *uneedle;
int32_t uhaystack_len, uneedle_len;
int ret_pos;
unsigned char u_break_iterator_buffer[U_BRK_SAFECLONE_BUFFERSIZE];
UBreakIterator* bi;
UErrorCode status;
*puchar_pos = -1;
/* convert the strings to UTF-16. */
uhaystack = NULL;
uhaystack_len = 0;
status = U_ZERO_ERROR;
intl_convert_utf8_to_utf16(&uhaystack, &uhaystack_len, (char *) haystack, haystack_len, &status );
if ( U_FAILURE( status ) ) {
/* Set global error code. */
intl_error_set_code( NULL, status TSRMLS_CC );
/* Set error messages. */
intl_error_set_custom_msg( NULL, "Error converting input string to UTF-16", 0 TSRMLS_CC );
efree( uhaystack );
return -1;
}
/* get a pointer to the haystack taking into account the offset */
bi = NULL;
status = U_ZERO_ERROR;
bi = grapheme_get_break_iterator(u_break_iterator_buffer, &status TSRMLS_CC );
puhaystack = grapheme_get_haystack_offset(bi, uhaystack, uhaystack_len, offset);
uhaystack_len = (uhaystack_len - ( puhaystack - uhaystack));
if ( NULL == puhaystack ) {
intl_error_set( NULL, U_ILLEGAL_ARGUMENT_ERROR, "grapheme_strpos: Offset not contained in string", 1 TSRMLS_CC );
efree( uhaystack );
ubrk_close (bi);
return -1;
}
if ( f_ignore_case ) {
grapheme_intl_case_fold(&uhaystack, &puhaystack, &uhaystack_len, &status );
}
uneedle = NULL;
uneedle_len = 0;
status = U_ZERO_ERROR;
intl_convert_utf8_to_utf16(&uneedle, &uneedle_len, (char *) needle, needle_len, &status );
if ( U_FAILURE( status ) ) {
/* Set global error code. */
intl_error_set_code( NULL, status TSRMLS_CC );
/* Set error messages. */
intl_error_set_custom_msg( NULL, "Error converting input string to UTF-16", 0 TSRMLS_CC );
efree( uhaystack );
efree( uneedle );
ubrk_close (bi);
return -1;
}
if ( f_ignore_case ) {
grapheme_intl_case_fold(&uneedle, &uneedle, &uneedle_len, &status );
}
ret_pos = grapheme_memnstr_grapheme(bi, puhaystack, uneedle, uneedle_len, puhaystack + uhaystack_len );
*puchar_pos = ubrk_current(bi);
efree( uhaystack );
efree( uneedle );
ubrk_close (bi);
return ret_pos;
}
int textBreakCurrent(TextBreakIterator* iterator)
{
return ubrk_current(reinterpret_cast<UBreakIterator*>(iterator));
}
static void TestBreakIteratorCAPI()
{
UErrorCode status = U_ZERO_ERROR;
UBreakIterator *word, *sentence, *line, *character, *b, *bogus;
int32_t start,pos,end,to;
int32_t i;
int32_t count = 0;
UChar text[50];
/* Note: the adjacent "" are concatenating strings, not adding a \" to the
string, which is probably what whoever wrote this intended. Don't fix,
because it would throw off the hard coded break positions in the following
tests. */
u_uastrcpy(text, "He's from Africa. ""Mr. Livingston, I presume?"" Yeah");
/*test ubrk_open()*/
log_verbose("\nTesting BreakIterator open functions\n");
/* Use french for fun */
word = ubrk_open(UBRK_WORD, "en_US", text, u_strlen(text), &status);
if(status == U_FILE_ACCESS_ERROR) {
log_data_err("Check your data - it doesn't seem to be around\n");
return;
} else if(U_FAILURE(status)){
log_err_status(status, "FAIL: Error in ubrk_open() for word breakiterator: %s\n", myErrorName(status));
}
else{
log_verbose("PASS: Successfully opened word breakiterator\n");
}
sentence = ubrk_open(UBRK_SENTENCE, "en_US", text, u_strlen(text), &status);
if(U_FAILURE(status)){
log_err_status(status, "FAIL: Error in ubrk_open() for sentence breakiterator: %s\n", myErrorName(status));
return;
}
else{
log_verbose("PASS: Successfully opened sentence breakiterator\n");
}
line = ubrk_open(UBRK_LINE, "en_US", text, u_strlen(text), &status);
if(U_FAILURE(status)){
log_err("FAIL: Error in ubrk_open() for line breakiterator: %s\n", myErrorName(status));
return;
}
else{
log_verbose("PASS: Successfully opened line breakiterator\n");
}
character = ubrk_open(UBRK_CHARACTER, "en_US", text, u_strlen(text), &status);
if(U_FAILURE(status)){
log_err("FAIL: Error in ubrk_open() for character breakiterator: %s\n", myErrorName(status));
return;
}
else{
log_verbose("PASS: Successfully opened character breakiterator\n");
}
/*trying to open an illegal iterator*/
bogus = ubrk_open((UBreakIteratorType)5, "en_US", text, u_strlen(text), &status);
if(U_SUCCESS(status)){
log_err("FAIL: Error in ubrk_open() for BOGUS breakiterator. Expected U_ILLEGAL_ARGUMENT_ERROR\n");
}
if(U_FAILURE(status)){
if(status != U_ILLEGAL_ARGUMENT_ERROR){
log_err("FAIL: Error in ubrk_open() for BOGUS breakiterator. Expected U_ILLEGAL_ARGUMENT_ERROR\n Got %s\n", myErrorName(status));
}
}
status=U_ZERO_ERROR;
/* ======= Test ubrk_countAvialable() and ubrk_getAvialable() */
log_verbose("\nTesting ubrk_countAvailable() and ubrk_getAvailable()\n");
count=ubrk_countAvailable();
/* use something sensible w/o hardcoding the count */
if(count < 0){
log_err("FAIL: Error in ubrk_countAvialable() returned %d\n", count);
}
else{
log_verbose("PASS: ubrk_countAvialable() successful returned %d\n", count);
}
for(i=0;i<count;i++)
{
log_verbose("%s\n", ubrk_getAvailable(i));
if (ubrk_getAvailable(i) == 0)
log_err("No locale for which breakiterator is applicable\n");
else
log_verbose("A locale %s for which breakiterator is applicable\n",ubrk_getAvailable(i));
}
/*========Test ubrk_first(), ubrk_last()...... and other functions*/
log_verbose("\nTesting the functions for word\n");
start = ubrk_first(word);
if(start!=0)
log_err("error ubrk_start(word) did not return 0\n");
log_verbose("first (word = %d\n", (int32_t)start);
pos=ubrk_next(word);
if(pos!=4)
log_err("error ubrk_next(word) did not return 4\n");
log_verbose("next (word = %d\n", (int32_t)pos);
pos=ubrk_following(word, 4);
if(pos!=5)
log_err("error ubrl_following(word,4) did not return 6\n");
log_verbose("next (word = %d\n", (int32_t)pos);
end=ubrk_last(word);
if(end!=49)
log_err("error ubrk_last(word) did not return 49\n");
log_verbose("last (word = %d\n", (int32_t)end);
pos=ubrk_previous(word);
log_verbose("%d %d\n", end, pos);
pos=ubrk_previous(word);
log_verbose("%d \n", pos);
if (ubrk_isBoundary(word, 2) != FALSE) {
log_err("error ubrk_isBoundary(word, 2) did not return FALSE\n");
}
pos=ubrk_current(word);
if (pos != 4) {
log_err("error ubrk_current() != 4 after ubrk_isBoundary(word, 2)\n");
}
if (ubrk_isBoundary(word, 4) != TRUE) {
log_err("error ubrk_isBoundary(word, 4) did not return TRUE\n");
}
log_verbose("\nTesting the functions for character\n");
ubrk_first(character);
pos = ubrk_following(character, 5);
if(pos!=6)
log_err("error ubrk_following(character,5) did not return 6\n");
log_verbose("Following (character,5) = %d\n", (int32_t)pos);
pos=ubrk_following(character, 18);
if(pos!=19)
log_err("error ubrk_following(character,18) did not return 19\n");
log_verbose("Followingcharacter,18) = %d\n", (int32_t)pos);
pos=ubrk_preceding(character, 22);
if(pos!=21)
log_err("error ubrk_preceding(character,22) did not return 21\n");
log_verbose("preceding(character,22) = %d\n", (int32_t)pos);
log_verbose("\nTesting the functions for line\n");
pos=ubrk_first(line);
if(pos != 0)
log_err("error ubrk_first(line) returned %d, expected 0\n", (int32_t)pos);
pos = ubrk_next(line);
pos=ubrk_following(line, 18);
if(pos!=22)
log_err("error ubrk_following(line) did not return 22\n");
log_verbose("following (line) = %d\n", (int32_t)pos);
log_verbose("\nTesting the functions for sentence\n");
ubrk_first(sentence);
pos = ubrk_current(sentence);
log_verbose("Current(sentence) = %d\n", (int32_t)pos);
pos = ubrk_last(sentence);
if(pos!=49)
log_err("error ubrk_last for sentence did not return 49\n");
log_verbose("Last (sentence) = %d\n", (int32_t)pos);
ubrk_first(sentence);
to = ubrk_following( sentence, 0 );
if (to == 0) log_err("ubrk_following returned 0\n");
to = ubrk_preceding( sentence, to );
if (to != 0) log_err("ubrk_preceding didn't return 0\n");
if (ubrk_first(sentence)!=ubrk_current(sentence)) {
log_err("error in ubrk_first() or ubrk_current()\n");
}
/*---- */
/*Testing ubrk_open and ubrk_close()*/
log_verbose("\nTesting open and close for us locale\n");
b = ubrk_open(UBRK_WORD, "fr_FR", text, u_strlen(text), &status);
if (U_FAILURE(status)) {
log_err("ubrk_open for word returned NULL: %s\n", myErrorName(status));
}
ubrk_close(b);
/* Test setText and setUText */
{
UChar s1[] = {0x41, 0x42, 0x20, 0};
UChar s2[] = {0x41, 0x42, 0x43, 0x44, 0x45, 0};
UText *ut = NULL;
UBreakIterator *bb;
int j;
log_verbose("\nTesting ubrk_setText() and ubrk_setUText()\n");
status = U_ZERO_ERROR;
bb = ubrk_open(UBRK_WORD, "en_US", NULL, 0, &status);
TEST_ASSERT_SUCCESS(status);
ubrk_setText(bb, s1, -1, &status);
TEST_ASSERT_SUCCESS(status);
ubrk_first(bb);
j = ubrk_next(bb);
TEST_ASSERT(j == 2);
ut = utext_openUChars(ut, s2, -1, &status);
ubrk_setUText(bb, ut, &status);
TEST_ASSERT_SUCCESS(status);
j = ubrk_next(bb);
TEST_ASSERT(j == 5);
ubrk_close(bb);
utext_close(ut);
}
ubrk_close(word);
ubrk_close(sentence);
ubrk_close(line);
ubrk_close(character);
}
static int icuNext(
sqlite3_tokenizer_cursor *pCursor,
const char **ppToken,
int *pnBytes,
int *piStartOffset,
int *piEndOffset,
int *piPosition
){
IcuCursor *pCsr = (IcuCursor *)pCursor;
int iStart = 0;
int iEnd = 0;
int nByte = 0;
while( iStart==iEnd ){
UChar32 c;
iStart = ubrk_current(pCsr->pIter);
iEnd = ubrk_next(pCsr->pIter);
if( iEnd==UBRK_DONE ){
return SQLITE_DONE;
}
while( iStart<iEnd ){
int iWhite = iStart;
U16_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
if( u_isspace(c) ){
iStart = iWhite;
}else{
break;
}
}
assert(iStart<=iEnd);
}
do {
UErrorCode status = U_ZERO_ERROR;
if( nByte ){
char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
if( !zNew ){
return SQLITE_NOMEM;
}
pCsr->zBuffer = zNew;
pCsr->nBuffer = nByte;
}
u_strToUTF8(
pCsr->zBuffer, pCsr->nBuffer, &nByte,
&pCsr->aChar[iStart], iEnd-iStart,
&status
);
} while( nByte>pCsr->nBuffer );
*ppToken = pCsr->zBuffer;
*pnBytes = nByte;
*piStartOffset = pCsr->aOffset[iStart];
*piEndOffset = pCsr->aOffset[iEnd];
*piPosition = pCsr->iToken++;
return SQLITE_OK;
}
int32_t __hs_ubrk_current(const UBreakIterator *bi)
{
return ubrk_current(bi);
}