//---------------------------------------------------------------------------------------- // // ubrk_openRules open a break iterator from a set of break rules. // Invokes the rule builder. // //---------------------------------------------------------------------------------------- U_CAPI UBreakIterator* U_EXPORT2 ubrk_openRules( const UChar *rules, int32_t rulesLength, const UChar *text, int32_t textLength, UParseError *parseErr, UErrorCode *status) { if (status == NULL || U_FAILURE(*status)){ return 0; } BreakIterator *result = 0; UnicodeString ruleString(rules, rulesLength); result = RBBIRuleBuilder::createRuleBasedBreakIterator(ruleString, *parseErr, *status); if(U_FAILURE(*status)) { return 0; } if (text != NULL) { UCharCharacterIterator *iter = 0; iter = new UCharCharacterIterator(text, textLength); if(iter == 0) { *status = U_MEMORY_ALLOCATION_ERROR; delete result; return 0; } result->adoptText(iter); } return (UBreakIterator *)result; }
void ubrk_refreshUText(UBreakIterator *bi, UText *text, UErrorCode *status) { BreakIterator *bii = reinterpret_cast<BreakIterator *>(bi); bii->refreshInputText(text, *status); }
/* * Return the next break, counting words and spaces. */ int32_t SpaceBreakIterator::next() { if (fDone) { return BreakIterator::DONE; } int32_t nextBreak; do { nextBreak = fBreakIter->next(); if (nextBreak == BreakIterator::DONE) { fDone = TRUE; return BreakIterator::DONE; } } while(nextBreak > 0 && fComplexContext.contains(fText[nextBreak-1]) && fComplexContext.contains(fText[nextBreak])); int32_t result = nextBreak - fSpaceCount; if (nextBreak < fTextCount) { if (fText[nextBreak] == 0x0020 /*Unicode::isSpaceChar(fText[nextBreak])*/) { fSpaceCount += fBreakIter->next() - nextBreak; } } fWordCount += 1; return result; }
U_NAMESPACE_USE /* functions available in the common library (for unistr_case.cpp) */ /* public API functions */ U_CAPI int32_t U_EXPORT2 u_strToTitle(UChar *dest, int32_t destCapacity, const UChar *src, int32_t srcLength, UBreakIterator *titleIter, const char *locale, UErrorCode *pErrorCode) { LocalPointer<BreakIterator> ownedIter; BreakIterator *iter; if(titleIter!=NULL) { iter=reinterpret_cast<BreakIterator *>(titleIter); } else { iter=BreakIterator::createWordInstance(Locale(locale), *pErrorCode); ownedIter.adoptInstead(iter); } if(U_FAILURE(*pErrorCode)) { return 0; } UnicodeString s(srcLength<0, src, srcLength); iter->setText(s); return ustrcase_mapWithOverlap( ustrcase_getCaseLocale(locale), 0, iter, dest, destCapacity, src, srcLength, ustrcase_internalToTitle, *pErrorCode); }
U_DRAFT void U_EXPORT2 ubrk_setUText(UBreakIterator *bi, UText *text, UErrorCode *status) { BreakIterator *brit = (BreakIterator *)bi; brit->setText(text, *status); }
static void _breakiterator_rewind(zend_object_iterator *iter) { BreakIterator *biter = _breakiter_prolog(iter); zoi_with_current *zoi_iter = (zoi_with_current*)iter; int32_t pos = biter->first(); ZVAL_LONG(&zoi_iter->current, (zend_long)pos); }
//---------------------------------------------------------------------------------------- // // ubrk_open Create a canned type of break iterator based on type (word, line, etc.) // and locale. // //---------------------------------------------------------------------------------------- U_CAPI UBreakIterator* U_EXPORT2 ubrk_open(UBreakIteratorType type, const char *locale, const UChar *text, int32_t textLength, UErrorCode *status) { if(U_FAILURE(*status)) return 0; BreakIterator *result = 0; switch(type) { case UBRK_CHARACTER: result = BreakIterator::createCharacterInstance(Locale(locale), *status); break; case UBRK_WORD: result = BreakIterator::createWordInstance(Locale(locale), *status); break; case UBRK_LINE: result = BreakIterator::createLineInstance(Locale(locale), *status); break; case UBRK_SENTENCE: result = BreakIterator::createSentenceInstance(Locale(locale), *status); break; case UBRK_TITLE: result = BreakIterator::createTitleInstance(Locale(locale), *status); break; } // check for allocation error if (U_FAILURE(*status)) { return 0; } if(result == 0) { *status = U_MEMORY_ALLOCATION_ERROR; return 0; } if (text != NULL) { UCharCharacterIterator *iter = 0; iter = new UCharCharacterIterator(text, textLength); if(iter == 0) { *status = U_MEMORY_ALLOCATION_ERROR; delete result; return 0; } result->adoptText(iter); } return (UBreakIterator*)result; }
/* * This method does the acutal break comparison and reports the results. * It uses a SpaceBreakIterator to iterate over the text with spaces, * and a word instance of a Thai BreakIterator to iterate over the text * without spaces. */ UBool ThaiWordbreakTest::compareWordBreaks(const UChar *spaces, int32_t spaceCount, const UChar *noSpaces, int32_t noSpaceCount) { UBool result = TRUE; Locale thai("th"); UCharCharacterIterator *noSpaceIter = new UCharCharacterIterator(noSpaces, noSpaceCount); UErrorCode status = U_ZERO_ERROR; BreakIterator *breakIter = BreakIterator::createWordInstance(thai, status); breakIter->adoptText(noSpaceIter); SpaceBreakIterator spaceIter(spaces, spaceCount); int32_t nextBreak = 0; int32_t nextSpaceBreak = 0; int32_t iterCount = 0; while (TRUE) { nextSpaceBreak = spaceIter.next(); nextBreak = breakIter->next(); if (nextSpaceBreak == BreakIterator::DONE || nextBreak == BreakIterator::DONE) { if (nextBreak != BreakIterator::DONE) { fprintf(stderr, "break iterator didn't end.\n"); } else if (nextSpaceBreak != BreakIterator::DONE) { fprintf(stderr, "premature break iterator end.\n"); } break; } while (nextSpaceBreak != nextBreak && nextSpaceBreak != BreakIterator::DONE && nextBreak != BreakIterator::DONE) { if (nextSpaceBreak < nextBreak) { breakNotFound(nextSpaceBreak); result = FALSE; nextSpaceBreak = spaceIter.next(); } else if (nextSpaceBreak > nextBreak) { foundInvalidBreak(nextBreak); result = FALSE; nextBreak = breakIter->next(); } } if (fVerbose) { printf("%d %d\n", nextSpaceBreak, nextBreak); } } fWordCount = spaceIter.getWordCount(); delete breakIter; return result; }
/* Print each element in reverse order: */ void printEachBackward( BreakIterator& boundary) { int32_t end = boundary.last(); for (int32_t start = boundary.previous(); start != BreakIterator::DONE; end = start, start = boundary.previous()) { printTextRange( boundary, start, end ); } }
/* Print each element in order: */ void printEachForward( BreakIterator& boundary) { int32_t start = boundary.first(); for (int32_t end = boundary.next(); end != BreakIterator::DONE; start = end, end = boundary.next()) { printTextRange( boundary, start, end ); } }
/* Creating and using text boundaries */ int main( void ) { puts("ICU Break Iterator Sample Program\n"); puts("C++ Break Iteration\n"); BreakIterator* boundary; UnicodeString stringToExamine("Aaa bbb ccc. Ddd eee fff."); printf("Examining: "); printUnicodeString(stringToExamine); puts(""); //print each sentence in forward and reverse order UErrorCode status = U_ZERO_ERROR; boundary = BreakIterator::createSentenceInstance( Locale::getUS(), status ); if (U_FAILURE(status)) { printf("failed to create sentence break iterator. status = %s", u_errorName(status)); exit(1); } boundary->setText(stringToExamine); puts("\n Sentence Boundaries... "); puts("----- forward: -----------"); printEachForward(*boundary); puts("----- backward: ----------"); printEachBackward(*boundary); delete boundary; //print each word in order printf("\n Word Boundaries... \n"); boundary = BreakIterator::createWordInstance( Locale::getUS(), status); boundary->setText(stringToExamine); puts("----- forward: -----------"); printEachForward(*boundary); //print first element puts("----- first: -------------"); printFirst(*boundary); //print last element puts("----- last: --------------"); printLast(*boundary); //print word at charpos 10 puts("----- at pos 10: ---------"); printAt(*boundary, 10 ); delete boundary; puts("\nEnd C++ Break Iteration"); // Call the C version return c_main(); }
U_CAPI void U_EXPORT2 ubrk_close(UBreakIterator *bi) { BreakIterator *ubi = (BreakIterator*) bi; if (ubi) { if (ubi->isBufferClone()) { ubi->~BreakIterator(); *(uint32_t *)ubi = 0xdeadbeef; } else { delete ubi; } } }
U_CAPI void U_EXPORT2 ubrk_setText(UBreakIterator* bi, const UChar* text, int32_t textLength, UErrorCode* status) { BreakIterator *brit = (BreakIterator *)bi; UText ut = UTEXT_INITIALIZER; utext_openUChars(&ut, text, textLength, status); brit->setText(&ut, *status); // A stack allocated UText wrapping a UCHar * string // can be dumped without explicitly closing it. }
static void _breakiterator_move_forward(zend_object_iterator *iter) { BreakIterator *biter = _breakiter_prolog(iter); zoi_with_current *zoi_iter = (zoi_with_current*)iter; iter->funcs->invalidate_current(iter); if (biter == NULL) { return; } int32_t pos = biter->next(); if (pos != BreakIterator::DONE) { ZVAL_LONG(&zoi_iter->current, (zend_long)pos); } //else we've reached the end of the enum, nothing more is required }
/* * Generate a text file with spaces in it from a file without. */ int generateFile(const UChar *chars, int32_t length) { Locale root(""); UCharCharacterIterator *noSpaceIter = new UCharCharacterIterator(chars, length); UErrorCode status = U_ZERO_ERROR; UnicodeSet complexContext(UNICODE_STRING_SIMPLE("[:LineBreak=SA:]"), status); BreakIterator *breakIter = BreakIterator::createWordInstance(root, status); breakIter->adoptText(noSpaceIter); char outbuf[1024]; int32_t strlength; UChar bom = 0xFEFF; printf("%s", u_strToUTF8(outbuf, sizeof(outbuf), &strlength, &bom, 1, &status)); int32_t prevbreak = 0; while (U_SUCCESS(status)) { int32_t nextbreak = breakIter->next(); if (nextbreak == BreakIterator::DONE) { break; } printf("%s", u_strToUTF8(outbuf, sizeof(outbuf), &strlength, &chars[prevbreak], nextbreak-prevbreak, &status)); if (nextbreak > 0 && complexContext.contains(chars[nextbreak-1]) && complexContext.contains(chars[nextbreak])) { printf(" "); } prevbreak = nextbreak; } if (U_FAILURE(status)) { fprintf(stderr, "generate failed: %s\n", u_errorName(status)); return status; } else { return 0; } }
void printTextRange( BreakIterator& iterator, int32_t start, int32_t end ) { CharacterIterator *strIter = iterator.getText().clone(); UnicodeString s; strIter->getText(s); printf(" %ld %ld\t", (long)start, (long)end); printUnicodeString(UnicodeString(s, 0, start)); printf("|"); printUnicodeString(UnicodeString(s, start, end-start)); printf("|"); printUnicodeString(UnicodeString(s, end)); puts(""); delete strIter; }
void RBBIAPITest::doBoundaryTest(BreakIterator& bi, UnicodeString& text, int32_t *boundaries){ logln((UnicodeString)"testIsBoundary():"); int32_t p = 0; UBool isB; for (int32_t i = 0; i < text.length(); i++) { isB = bi.isBoundary(i); logln((UnicodeString)"bi.isBoundary(" + i + ") -> " + isB); if (i == boundaries[p]) { if (!isB) errln((UnicodeString)"Wrong result from isBoundary() for " + i + (UnicodeString)": expected true, got false"); p++; } else { if (isB) errln((UnicodeString)"Wrong result from isBoundary() for " + i + (UnicodeString)": expected false, got true"); } } }
void GetWordBoundaryPositions(const FunctionCallbackInfo<Value>& args) { Isolate* isolate = Isolate::GetCurrent(); HandleScope scope(isolate); if (args.Length() != 2) { isolate->ThrowException(Exception::TypeError(String::NewFromUtf8(isolate, "must supply locale and text"))); return; } if (!args[0]->IsString()) { isolate->ThrowException(Exception::TypeError(String::NewFromUtf8(isolate, "text is not specified"))); return; } if (!args[1]->IsString()) { isolate->ThrowException(Exception::TypeError(String::NewFromUtf8(isolate, "locale is not specified"))); return; } // convert v8 locale to ICU String::Utf8Value locale(args[1]->ToString()); const char* country = strtok(*locale, "_"), *language = strtok(NULL, "_"); Locale icuLocale(language, country); // create the BreakIterator instance UErrorCode err = U_ZERO_ERROR; BreakIterator *iterator = BreakIterator::createWordInstance(icuLocale, err); if (U_FAILURE(err)) { ErrorCode errCode; errCode.set(err); isolate->ThrowException(Exception::TypeError(String::NewFromUtf8(isolate, errCode.errorName()))); return; } // Convert v8 text to ICU Unicode value Local<String> textStr = args[0]->ToString(); String::Utf8Value textValue(textStr); UnicodeString uTextValue(*textValue, "UTF-8"); if (uTextValue.isBogus()) { isolate->ThrowException(Exception::TypeError(String::NewFromUtf8(isolate, "unable to create unicode string"))); return; } iterator->setText(uTextValue); // populate boundaries Local<Array> results = Array::New(isolate); int32_t arrayPosition = 0; int32_t currentBoundary = iterator->first(); int32_t previousBoundary = 0; while (currentBoundary != BreakIterator::DONE) { if (currentBoundary > 0) { Local<Object> boundaryResult = Object::New(isolate); boundaryResult->Set(String::NewFromUtf8(isolate, "start"), Number::New(isolate, previousBoundary)); boundaryResult->Set(String::NewFromUtf8(isolate, "end"), Number::New(isolate, currentBoundary)); results->Set(arrayPosition++, boundaryResult); } previousBoundary = currentBoundary; currentBoundary = iterator->next(); } // cleanup delete iterator; args.GetReturnValue().Set(results); }
void Hyphenator::slotHyphenate(PageItem* it) { if (!(it->asTextFrame()) || (it->itemText.length() == 0)) return; m_doc->DoDrawing = false; QString text = ""; int startC = 0; if (it->itemText.selectionLength() > 0) { startC = it->itemText.startOfSelection(); text = it->itemText.text(startC, it->itemText.selectionLength()); } else { text = it->itemText.text(0, it->itemText.length()); } rememberedWords.clear(); qApp->setOverrideCursor(QCursor(Qt::WaitCursor)); BreakIterator* bi = StoryText::getWordIterator(); bi->setText((const UChar*) text.utf16()); int pos = bi->first(); while (pos != BreakIterator::DONE) { int firstC = pos; pos = bi->next(); int lastC = pos; int countC = lastC - firstC; const CharStyle& style = it->itemText.charStyle(firstC); if (countC > 0 && countC > style.hyphenWordMin() - 1) { QString word = text.mid(firstC, countC); QString wordLower = QLocale(style.language()).toLower(word); if (wordLower.contains(SpecialChars::SHYPHEN)) break; bool ok = loadDict(style.language()); if (!ok) continue; QByteArray te = m_codec->fromUnicode(wordLower); char *buffer = static_cast<char*>(malloc(te.length() + 5)); if (buffer == nullptr) break; char **rep = nullptr; int *pos = nullptr; int *cut = nullptr; // TODO: support non-standard hyphenation, see hnj_hyphen_hyphenate2 docs if (!hnj_hyphen_hyphenate2(m_hdict, te.data(), te.length(), buffer, nullptr, &rep, &pos, &cut)) { int i = 0; buffer[te.length()] = '\0'; bool hasHyphen = false; for (i = 1; i < wordLower.length()-1; ++i) { if(buffer[i] & 1) { hasHyphen = true; break; } } QString outs = ""; QString input = ""; outs += word[0]; for (i = 1; i < wordLower.length()-1; ++i) { outs += word[i]; if(buffer[i] & 1) outs += "-"; } outs += word.rightRef(1); input = outs; if (!ignoredWords.contains(word)) { if (!hasHyphen) it->itemText.hyphenateWord(startC + firstC, wordLower.length(), nullptr); else if (m_automatic) { if (specialWords.contains(word)) { outs = specialWords.value(word); uint ii = 1; for (i = 1; i < outs.length()-1; ++i) { QChar cht = outs[i]; if (cht == '-') buffer[ii-1] = 1; else { buffer[ii] = 0; ++ii; } } } it->itemText.hyphenateWord(startC + firstC, wordLower.length(), buffer); } else { if (specialWords.contains(word)) { outs = specialWords.value(word); uint ii = 1; for (i = 1; i < outs.length()-1; ++i) { QChar cht = outs[i]; if (cht == '-') buffer[ii-1] = 1; else { buffer[ii] = 0; ++ii; } } } if (rememberedWords.contains(input)) { outs = rememberedWords.value(input); uint ii = 1; for (i = 1; i < outs.length()-1; ++i) { QChar cht = outs[i]; if (cht == '-') buffer[ii-1] = 1; else { buffer[ii] = 0; ++ii; } } it->itemText.hyphenateWord(firstC, wordLower.length(), buffer); } else { qApp->changeOverrideCursor(QCursor(Qt::ArrowCursor)); PrefsContext* prefs = PrefsManager::instance()->prefsFile->getContext("hyhpen_options"); int xpos = prefs->getInt("Xposition", -9999); int ypos = prefs->getInt("Yposition", -9999); HyAsk *dia = new HyAsk((QWidget*)parent(), outs); if ((xpos != -9999) && (ypos != -9999)) dia->move(xpos, ypos); qApp->processEvents(); if (dia->exec()) { outs = dia->Wort->text(); uint ii = 1; for (i = 1; i < outs.length()-1; ++i) { QChar cht = outs[i]; if (cht == '-') buffer[ii-1] = 1; else { buffer[ii] = 0; ++ii; } } if (!rememberedWords.contains(input)) rememberedWords.insert(input, outs); if (dia->addToIgnoreList->isChecked()) { if (!ignoredWords.contains(word)) ignoredWords.insert(word); } if (dia->addToExceptionList->isChecked()) { if (!specialWords.contains(word)) specialWords.insert(word, outs); } it->itemText.hyphenateWord(firstC, wordLower.length(), buffer); } else { free(buffer); buffer = nullptr; prefs->set("Xposition", dia->xpos); prefs->set("Yposition", dia->ypos); delete dia; break; } prefs->set("Xposition", dia->xpos); prefs->set("Yposition", dia->ypos); delete dia; qApp->changeOverrideCursor(QCursor(Qt::WaitCursor)); } } } } free(buffer); if (rep) { for (int i = 0; i < te.length() - 1; ++i) free(rep[i]); } free(rep); free(pos); free(cut); } } qApp->restoreOverrideCursor(); m_doc->DoDrawing = true; rememberedWords.clear(); }
TRI_vector_string_t* Utf8Helper::getWords (const char* const text, const size_t textLength, const size_t minimalLength, const size_t maximalLength, bool lowerCase) { TRI_vector_string_t* words; UErrorCode status = U_ZERO_ERROR; UnicodeString word; if (textLength == 0) { // input text is empty return NULL; } if (textLength < minimalLength) { // input text is shorter than required minimum length return NULL; } size_t textUtf16Length = 0; UChar* textUtf16 = NULL; if (lowerCase) { // lower case string int32_t lowerLength = 0; char* lower = tolower(TRI_UNKNOWN_MEM_ZONE, text, (int32_t) textLength, lowerLength); if (lower == NULL) { // out of memory return NULL; } if (lowerLength == 0) { TRI_Free(TRI_UNKNOWN_MEM_ZONE, lower); return NULL; } textUtf16 = TRI_Utf8ToUChar(TRI_UNKNOWN_MEM_ZONE, lower, lowerLength, &textUtf16Length); TRI_Free(TRI_UNKNOWN_MEM_ZONE, lower); } else { textUtf16 = TRI_Utf8ToUChar(TRI_UNKNOWN_MEM_ZONE, text, (int32_t) textLength, &textUtf16Length); } if (textUtf16 == NULL) { return NULL; } ULocDataLocaleType type = ULOC_VALID_LOCALE; const Locale& locale = _coll->getLocale(type, status); if (U_FAILURE(status)) { TRI_Free(TRI_UNKNOWN_MEM_ZONE, textUtf16); LOG_ERROR("error in Collator::getLocale(...): %s", u_errorName(status)); return NULL; } UChar* tempUtf16 = (UChar *) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, (textUtf16Length + 1) * sizeof(UChar), false); if (tempUtf16 == NULL) { TRI_Free(TRI_UNKNOWN_MEM_ZONE, textUtf16); return NULL; } words = (TRI_vector_string_t*) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_vector_string_t), false); if (words == NULL) { TRI_Free(TRI_UNKNOWN_MEM_ZONE, textUtf16); TRI_Free(TRI_UNKNOWN_MEM_ZONE, tempUtf16); return NULL; } // estimate an initial vector size. this is not accurate, but setting the initial size to some // value in the correct order of magnitude will save a lot of vector reallocations later size_t initialWordCount = textLength / (2 * (minimalLength + 1)); if (initialWordCount < 32) { // alloc at least 32 pointers (= 256b) initialWordCount = 32; } else if (initialWordCount > 8192) { // alloc at most 8192 pointers (= 64kb) initialWordCount = 8192; } TRI_InitVectorString2(words, TRI_UNKNOWN_MEM_ZONE, initialWordCount); BreakIterator* wordIterator = BreakIterator::createWordInstance(locale, status); UnicodeString utext(textUtf16); wordIterator->setText(utext); int32_t start = wordIterator->first(); for(int32_t end = wordIterator->next(); end != BreakIterator::DONE; start = end, end = wordIterator->next()) { size_t tempUtf16Length = (size_t) (end - start); // end - start = word length if (tempUtf16Length >= minimalLength) { size_t chunkLength = tempUtf16Length; if (chunkLength > maximalLength) { chunkLength = maximalLength; } utext.extractBetween(start, (int32_t) (start + chunkLength), tempUtf16, 0); size_t utf8WordLength; char* utf8Word = TRI_UCharToUtf8(TRI_UNKNOWN_MEM_ZONE, tempUtf16, chunkLength, &utf8WordLength); if (utf8Word != 0) { TRI_PushBackVectorString(words, utf8Word); } } } delete wordIterator; TRI_Free(TRI_UNKNOWN_MEM_ZONE, textUtf16); TRI_Free(TRI_UNKNOWN_MEM_ZONE, tempUtf16); if (words->_length == 0) { // no words found TRI_FreeVectorString(TRI_UNKNOWN_MEM_ZONE, words); return NULL; } return words; }
/* Print the first element */ void printFirst(BreakIterator& boundary) { int32_t start = boundary.first(); int32_t end = boundary.next(); printTextRange( boundary, start, end ); }
// // TestRuleStatus // Test word break rule status constants. // void RBBIAPITest::TestRuleStatus() { UChar str[30]; //no longer test Han or hiragana breaking here: ruleStatusVec would return nothing // changed UBRK_WORD_KANA to UBRK_WORD_IDEO u_unescape("plain word 123.45 \\u30a1\\u30a2 ", // 012345678901234567 8 9 0 // Katakana str, 30); UnicodeString testString1(str); int32_t bounds1[] = {0, 5, 6, 10, 11, 17, 18, 20, 21}; int32_t tag_lo[] = {UBRK_WORD_NONE, UBRK_WORD_LETTER, UBRK_WORD_NONE, UBRK_WORD_LETTER, UBRK_WORD_NONE, UBRK_WORD_NUMBER, UBRK_WORD_NONE, UBRK_WORD_IDEO, UBRK_WORD_NONE}; int32_t tag_hi[] = {UBRK_WORD_NONE_LIMIT, UBRK_WORD_LETTER_LIMIT, UBRK_WORD_NONE_LIMIT, UBRK_WORD_LETTER_LIMIT, UBRK_WORD_NONE_LIMIT, UBRK_WORD_NUMBER_LIMIT, UBRK_WORD_NONE_LIMIT, UBRK_WORD_IDEO_LIMIT, UBRK_WORD_NONE_LIMIT}; UErrorCode status=U_ZERO_ERROR; BreakIterator *bi = BreakIterator::createWordInstance(Locale::getEnglish(), status); if(U_FAILURE(status)) { errcheckln(status, "Fail : in construction - %s", u_errorName(status)); } else { bi->setText(testString1); // First test that the breaks are in the right spots. doBoundaryTest(*bi, testString1, bounds1); // Then go back and check tag values int32_t i = 0; int32_t pos, tag; for (pos = bi->first(); pos != BreakIterator::DONE; pos = bi->next(), i++) { if (pos != bounds1[i]) { errln("FAIL: unexpected word break at postion %d", pos); break; } tag = bi->getRuleStatus(); if (tag < tag_lo[i] || tag >= tag_hi[i]) { errln("FAIL: incorrect tag value %d at position %d", tag, pos); break; } // Check that we get the same tag values from getRuleStatusVec() int32_t vec[10]; int t = bi->getRuleStatusVec(vec, 10, status); TEST_ASSERT_SUCCESS(status); TEST_ASSERT(t==1); TEST_ASSERT(vec[0] == tag); } } delete bi; // Now test line break status. This test mostly is to confirm that the status constants // are correctly declared in the header. testString1 = "test line. \n"; // break type s s h bi = BreakIterator::createLineInstance(Locale::getEnglish(), status); if(U_FAILURE(status)) { errcheckln(status, "failed to create word break iterator. - %s", u_errorName(status)); } else { int32_t i = 0; int32_t pos, tag; UBool success; bi->setText(testString1); pos = bi->current(); tag = bi->getRuleStatus(); for (i=0; i<3; i++) { switch (i) { case 0: success = pos==0 && tag==UBRK_LINE_SOFT; break; case 1: success = pos==5 && tag==UBRK_LINE_SOFT; break; case 2: success = pos==12 && tag==UBRK_LINE_HARD; break; default: success = FALSE; break; } if (success == FALSE) { errln("Fail: incorrect word break status or position. i=%d, pos=%d, tag=%d", i, pos, tag); break; } pos = bi->next(); tag = bi->getRuleStatus(); } if (UBRK_LINE_SOFT >= UBRK_LINE_SOFT_LIMIT || UBRK_LINE_HARD >= UBRK_LINE_HARD_LIMIT || (UBRK_LINE_HARD > UBRK_LINE_SOFT && UBRK_LINE_HARD < UBRK_LINE_SOFT_LIMIT)) { errln("UBRK_LINE_* constants from header are inconsistent."); } } delete bi; }
/* Print the last element */ void printLast(BreakIterator& boundary) { int32_t end = boundary.last(); int32_t start = boundary.previous(); printTextRange( boundary, start, end ); }
/* Print the element at a specified position */ void printAt(BreakIterator &boundary, int32_t pos ) { int32_t end = boundary.following(pos); int32_t start = boundary.previous(); printTextRange( boundary, start, end ); }
// In the Unicode string characters are always stored in logical order. // This makes line breaking easy. One word is added to the current line at a time. Once the line is too long // we either go back one step or inset the line break at the current position (depending on "wrap_before" setting). // At the end everything that is left over is added as the final line. void text_layout::break_line(text_line & line, double wrap_width, unsigned text_ratio, bool wrap_before) { shape_text(line); if (!wrap_width || line.width() < wrap_width) { add_line(line); return; } if (text_ratio) { double wrap_at; double string_width = line.width(); double string_height = line.line_height(); for (double i = 1.0; ((wrap_at = string_width/i)/(string_height*i)) > text_ratio && (string_width/i) > wrap_width; i += 1.0) ; wrap_width = wrap_at; } mapnik::value_unicode_string const& text = itemizer_.text(); Locale locale; // TODO: Is the default constructor correct? UErrorCode status = U_ZERO_ERROR; BreakIterator *breakitr = BreakIterator::createLineInstance(locale, status); // Not breaking the text if an error occurs is probably the best thing we can do. // https://github.com/mapnik/mapnik/issues/2072 if (!U_SUCCESS(status)) { add_line(line); MAPNIK_LOG_ERROR(text_layout) << " could not create BreakIterator: " << u_errorName(status); return; } breakitr->setText(text); double current_line_length = 0; int last_break_position = static_cast<int>(line.first_char()); for (unsigned i=line.first_char(); i < line.last_char(); ++i) { // TODO: character_spacing std::map<unsigned, double>::const_iterator width_itr = width_map_.find(i); if (width_itr != width_map_.end()) { current_line_length += width_itr->second; } if (current_line_length <= wrap_width) continue; int break_position = wrap_before ? breakitr->preceding(i) : breakitr->following(i); // following() returns a break position after the last word. So DONE should only be returned // when calling preceding. if (break_position <= last_break_position || break_position == static_cast<int>(BreakIterator::DONE)) { // A single word is longer than the maximum line width. // Violate line width requirement and choose next break position break_position = breakitr->following(i); if (break_position == static_cast<int>(BreakIterator::DONE)) { break_position = line.last_char(); MAPNIK_LOG_ERROR(text_layout) << "Unexpected result in break_line. Trying to recover...\n"; } } // Break iterator operates on the whole string, while we only look at one line. So we need to // clamp break values. if (break_position < static_cast<int>(line.first_char())) { break_position = line.first_char(); } if (break_position > static_cast<int>(line.last_char())) { break_position = line.last_char(); } text_line new_line(last_break_position, break_position); clear_cluster_widths(last_break_position, break_position); shape_text(new_line); add_line(new_line); last_break_position = break_position; i = break_position - 1; current_line_length = 0; } if (last_break_position == static_cast<int>(line.first_char())) { // No line breaks => no reshaping required add_line(line); } else if (last_break_position != static_cast<int>(line.last_char())) { text_line new_line(last_break_position, line.last_char()); clear_cluster_widths(last_break_position, line.last_char()); shape_text(new_line); add_line(new_line); } }
/** Word wrap text * * @param str character vector * @param width single integer * @param cost_exponent single double * @param indent single integer * @param exdent single integer * @param prefix single string * @param initial single string * @param locale locale identifier or NULL for default locale * @param use_length single logical value * * @return list * * @version 0.1-?? (Bartek Tartanus) * * @version 0.2-2 (Marek Gagolewski, 2014-04-27) * single function for wrap_greedy and wrap_dynamic * (dispatch inside); * use BreakIterator * * @version 0.3-1 (Marek Gagolewski, 2014-11-04) * Issue #112: str_prepare_arg* retvals were not PROTECTed from gc * * @version 0.4-1 (Marek Gagolewski, 2014-12-06) * new args: indent, exdent, prefix, initial * * @version 0.5-1 (Marek Gagolewski, 2014-12-19) * #133 allow width <= 0 * * @version 0.5-1 (Marek Gagolewski, 2015-02-28) * don't trim so many white spaces at the end of each word (normalize arg does that) * #139: allow a "whitespace" break iterator * * @version 0.5-1 (Marek Gagolewski, 2015-04-23) * `use_length` arg added * * * @version 0.5-1 (Marek Gagolewski, 2015-06-09) * BIGSKIP: no more CHARSXP on out on "" input */ SEXP stri_wrap(SEXP str, SEXP width, SEXP cost_exponent, SEXP indent, SEXP exdent, SEXP prefix, SEXP initial, SEXP whitespace_only, SEXP use_length, SEXP locale) { bool use_length_val = stri__prepare_arg_logical_1_notNA(use_length, "use_length"); double exponent_val = stri__prepare_arg_double_1_notNA(cost_exponent, "cost_exponent"); bool whitespace_only_val = stri__prepare_arg_logical_1_notNA(whitespace_only, "whitespace_only"); int width_val = stri__prepare_arg_integer_1_notNA(width, "width"); if (width_val <= 0) width_val = 0; int indent_val = stri__prepare_arg_integer_1_notNA(indent, "indent"); if (indent_val < 0) Rf_error(MSG__EXPECTED_POSITIVE, "indent"); int exdent_val = stri__prepare_arg_integer_1_notNA(exdent, "exdent"); if (exdent_val < 0) Rf_error(MSG__EXPECTED_POSITIVE, "exdent"); const char* qloc = stri__prepare_arg_locale(locale, "locale", true); /* this is R_alloc'ed */ Locale loc = Locale::createFromName(qloc); PROTECT(str = stri_prepare_arg_string(str, "str")); PROTECT(prefix = stri_prepare_arg_string_1(prefix, "prefix")); PROTECT(initial = stri_prepare_arg_string_1(initial, "initial")); BreakIterator* briter = NULL; UText* str_text = NULL; STRI__ERROR_HANDLER_BEGIN(3) UErrorCode status = U_ZERO_ERROR; briter = BreakIterator::createLineInstance(loc, status); STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */}) R_len_t str_length = LENGTH(str); StriContainerUTF8_indexable str_cont(str, str_length); StriContainerUTF8 prefix_cont(prefix, 1); StriContainerUTF8 initial_cont(initial, 1); // prepare indent/exdent/prefix/initial stuff: // 1st line, 1st para (i==0, u==0): initial+indent // nth line, 1st para (i==0, u> 0): prefix +exdent // 1st line, nth para (i> 0, u==0): prefix +indent // nth line, nth para (i> 0, u> 0): prefix +exdent StriWrapLineStart ii(initial_cont.get(0), indent_val); StriWrapLineStart pi(prefix_cont.get(0), indent_val); StriWrapLineStart pe(prefix_cont.get(0), exdent_val); status = U_ZERO_ERROR; //Unicode Newline Guidelines - Unicode Technical Report #13 UnicodeSet uset_linebreaks(UnicodeString::fromUTF8("[\\u000A-\\u000D\\u0085\\u2028\\u2029]"), status); STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */}) uset_linebreaks.freeze(); status = U_ZERO_ERROR; UnicodeSet uset_whitespaces(UnicodeString::fromUTF8("\\p{White_space}"), status); STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */}) uset_whitespaces.freeze(); SEXP ret; STRI__PROTECT(ret = Rf_allocVector(VECSXP, str_length)); for (R_len_t i = 0; i < str_length; ++i) { if (str_cont.isNA(i) || prefix_cont.isNA(0) || initial_cont.isNA(0)) { SET_VECTOR_ELT(ret, i, stri__vector_NA_strings(1)); continue; } status = U_ZERO_ERROR; const char* str_cur_s = str_cont.get(i).c_str(); R_len_t str_cur_n = str_cont.get(i).length(); str_text = utext_openUTF8(str_text, str_cur_s, str_cont.get(i).length(), &status); STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */}) status = U_ZERO_ERROR; briter->setText(str_text, status); STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */}) // all right, first let's generate a list of places at which we may do line breaks deque< R_len_t > occurrences_list; // this could be an R_len_t queue R_len_t match = briter->first(); while (match != BreakIterator::DONE) { if (!whitespace_only_val) occurrences_list.push_back(match); else { if (match > 0 && match < str_cur_n) { UChar32 c; U8_GET((const uint8_t*)str_cur_s, 0, match-1, str_cur_n, c); if (uset_whitespaces.contains(c)) occurrences_list.push_back(match); } else occurrences_list.push_back(match); } match = briter->next(); } R_len_t noccurrences = (R_len_t)occurrences_list.size(); // number of boundaries if (noccurrences <= 1) { // no match (1 boundary == 0) SET_VECTOR_ELT(ret, i, Rf_ScalarString(str_cont.toR(i))); continue; } // the number of "words" is: R_len_t nwords = noccurrences - 1; // convert occurrences_list to a vector // in order to obtain end positions (in a string) of each "words", // noting that occurrences_list.at(0) == 0 #ifndef NDEBUG if (occurrences_list.at(0) != 0) throw StriException("NDEBUG: stri_wrap: (occurrences_list.at(0) != 0)"); #endif std::vector<R_len_t> end_pos_orig(nwords); deque<R_len_t>::iterator iter = ++(occurrences_list.begin()); for (R_len_t j = 0; iter != occurrences_list.end(); ++iter, ++j) { end_pos_orig[j] = (*iter); // this is a UTF-8 index } // now: // we'll get the total widths/number of code points in each "word" std::vector<R_len_t> widths_orig(nwords); // we'll get the total widths/number of code points without trailing whitespaces std::vector<R_len_t> widths_trim(nwords); // we'll get the end positions without trailing whitespaces std::vector<R_len_t> end_pos_trim(nwords); // detect line endings (fail on a match) UChar32 c = 0; R_len_t j = 0; R_len_t cur_block = 0; R_len_t cur_width_orig = 0; R_len_t cur_width_trim = 0; R_len_t cur_count_orig = 0; R_len_t cur_count_trim = 0; R_len_t cur_end_pos_trim = 0; while (j < str_cur_n) { R_len_t jlast = j; U8_NEXT(str_cur_s, j, str_cur_n, c); if (c < 0) // invalid utf-8 sequence throw StriException(MSG__INVALID_UTF8); if (uset_linebreaks.contains(c)) throw StriException(MSG__NEWLINE_FOUND); cur_width_orig += stri__width_char(c); ++cur_count_orig; if (uset_whitespaces.contains(c)) { // OLD: trim all white spaces from the end: // ++cur_count_trim; // [we have the normalize arg for that] // NEW: trim just one white space at the end: cur_width_trim = stri__width_char(c); cur_count_trim = 1; cur_end_pos_trim = jlast; } else { cur_width_trim = 0; cur_count_trim = 0; cur_end_pos_trim = j; } if (j >= str_cur_n || end_pos_orig[cur_block] <= j) { // we'll start a new block in a moment if (use_length_val) { widths_orig[cur_block] = cur_count_orig; widths_trim[cur_block] = cur_count_orig-cur_count_trim; } else { widths_orig[cur_block] = cur_width_orig; widths_trim[cur_block] = cur_width_orig-cur_width_trim; } end_pos_trim[cur_block] = cur_end_pos_trim; cur_block++; cur_width_orig = 0; cur_width_trim = 0; cur_count_orig = 0; cur_count_trim = 0; cur_end_pos_trim = j; } } // do wrap std::deque<R_len_t> wrap_after; // wrap line after which word in {0..nwords-1}? if (exponent_val <= 0.0) { stri__wrap_greedy(wrap_after, nwords, width_val, widths_orig, widths_trim, (use_length_val)?((i==0)?ii.count:pi.count):((i==0)?ii.width:pi.width), (use_length_val)?pe.count:pe.width); } else { stri__wrap_dynamic(wrap_after, nwords, width_val, exponent_val, widths_orig, widths_trim, (use_length_val)?((i==0)?ii.count:pi.count):((i==0)?ii.width:pi.width), (use_length_val)?pe.count:pe.width); } // wrap_after.size() line breaks => wrap_after.size()+1 lines R_len_t nlines = (R_len_t)wrap_after.size()+1; R_len_t last_pos = 0; SEXP ans; STRI__PROTECT(ans = Rf_allocVector(STRSXP, nlines)); deque<R_len_t>::iterator iter_wrap = wrap_after.begin(); for (R_len_t u = 0; iter_wrap != wrap_after.end(); ++iter_wrap, ++u) { R_len_t wrap_after_cur = *iter_wrap; R_len_t cur_pos = end_pos_trim[wrap_after_cur]; std::string cs; if (i == 0 && u == 0) cs = ii.str; else if (i > 0 && u == 0) cs = pi.str; else cs = pe.str; cs.append(str_cur_s+last_pos, cur_pos-last_pos); SET_STRING_ELT(ans, u, Rf_mkCharLenCE(cs.c_str(), cs.size(), CE_UTF8)); last_pos = end_pos_orig[wrap_after_cur]; } // last line goes here: std::string cs; if (i == 0 && nlines-1 == 0) cs = ii.str; else if (i > 0 && nlines-1 == 0) cs = pi.str; else cs = pe.str; cs.append(str_cur_s+last_pos, end_pos_trim[nwords-1]-last_pos); SET_STRING_ELT(ans, nlines-1, Rf_mkCharLenCE(cs.c_str(), cs.size(), CE_UTF8)); SET_VECTOR_ELT(ret, i, ans); STRI__UNPROTECT(1); } if (briter) { delete briter; briter = NULL; } if (str_text) { utext_close(str_text); str_text = NULL; } STRI__UNPROTECT_ALL return ret; STRI__ERROR_HANDLER_END({ if (briter) { delete briter; briter = NULL; } if (str_text) { utext_close(str_text); str_text = NULL; } }) }