//---------------------------------------------------------------------
//
// pattern
//
//---------------------------------------------------------------------
UnicodeString RegexPattern::pattern() const {
if (fPatternString != NULL) {
return *fPatternString;
} else if (fPattern == NULL) {
return UnicodeString();
} else {
UErrorCode status = U_ZERO_ERROR;
int64_t nativeLen = utext_nativeLength(fPattern);
int32_t len16 = utext_extract(fPattern, 0, nativeLen, NULL, 0, &status); // buffer overflow error
UnicodeString result;
status = U_ZERO_ERROR;
UChar *resultChars = result.getBuffer(len16);
utext_extract(fPattern, 0, nativeLen, resultChars, len16, &status); // unterminated warning
result.releaseBuffer(len16);
return result;
}
}
/*
* @param text A UText representing the text
* @param rangeStart The start of the range of dictionary characters
* @param rangeEnd The end of the range of dictionary characters
* @param foundBreaks Output of C array of int32_t break positions, or 0
* @return The number of breaks found
*/
int32_t
CjkBreakEngine::divideUpDictionaryRange( UText *text,
int32_t rangeStart,
int32_t rangeEnd,
UStack &foundBreaks ) const {
if (rangeStart >= rangeEnd) {
return 0;
}
const size_t defaultInputLength = 80;
size_t inputLength = rangeEnd - rangeStart;
// TODO: Replace by UnicodeString.
AutoBuffer<UChar, defaultInputLength> charString(inputLength);
// Normalize the input string and put it in normalizedText.
// The map from the indices of the normalized input to the raw
// input is kept in charPositions.
UErrorCode status = U_ZERO_ERROR;
utext_extract(text, rangeStart, rangeEnd, charString.elems(), inputLength, &status);
if (U_FAILURE(status)) {
return 0;
}
UnicodeString inputString(charString.elems(), inputLength);
// TODO: Use Normalizer2.
UNormalizationMode norm_mode = UNORM_NFKC;
UBool isNormalized =
Normalizer::quickCheck(inputString, norm_mode, status) == UNORM_YES ||
Normalizer::isNormalized(inputString, norm_mode, status);
// TODO: Replace by UVector32.
AutoBuffer<int32_t, defaultInputLength> charPositions(inputLength + 1);
int numChars = 0;
UText normalizedText = UTEXT_INITIALIZER;
// Needs to be declared here because normalizedText holds onto its buffer.
UnicodeString normalizedString;
if (isNormalized) {
int32_t index = 0;
charPositions[0] = 0;
while(index < inputString.length()) {
index = inputString.moveIndex32(index, 1);
charPositions[++numChars] = index;
}
utext_openUnicodeString(&normalizedText, &inputString, &status);
}
else {
Normalizer::normalize(inputString, norm_mode, 0, normalizedString, status);
if (U_FAILURE(status)) {
return 0;
}
charPositions.resize(normalizedString.length() + 1);
Normalizer normalizer(charString.elems(), inputLength, norm_mode);
int32_t index = 0;
charPositions[0] = 0;
while(index < normalizer.endIndex()){
/* UChar32 uc = */ normalizer.next();
charPositions[++numChars] = index = normalizer.getIndex();
}
utext_openUnicodeString(&normalizedText, &normalizedString, &status);
}
if (U_FAILURE(status)) {
return 0;
}
// From this point on, all the indices refer to the indices of
// the normalized input string.
// bestSnlp[i] is the snlp of the best segmentation of the first i
// characters in the range to be matched.
// TODO: Replace by UVector32.
AutoBuffer<uint32_t, defaultInputLength> bestSnlp(numChars + 1);
bestSnlp[0] = 0;
for(int i = 1; i <= numChars; i++) {
bestSnlp[i] = kuint32max;
}
// prev[i] is the index of the last CJK character in the previous word in
// the best segmentation of the first i characters.
// TODO: Replace by UVector32.
AutoBuffer<int, defaultInputLength> prev(numChars + 1);
for(int i = 0; i <= numChars; i++){
prev[i] = -1;
}
const size_t maxWordSize = 20;
// TODO: Replace both with UVector32.
AutoBuffer<int32_t, maxWordSize> values(numChars);
AutoBuffer<int32_t, maxWordSize> lengths(numChars);
// Dynamic programming to find the best segmentation.
bool is_prev_katakana = false;
for (int32_t i = 0; i < numChars; ++i) {
//utext_setNativeIndex(text, rangeStart + i);
utext_setNativeIndex(&normalizedText, i);
if (bestSnlp[i] == kuint32max)
continue;
int32_t count;
// limit maximum word length matched to size of current substring
int32_t maxSearchLength = (i + maxWordSize < (size_t) numChars)? maxWordSize : (numChars - i);
fDictionary->matches(&normalizedText, maxSearchLength, lengths.elems(), count, maxSearchLength, values.elems());
// if there are no single character matches found in the dictionary
// starting with this charcter, treat character as a 1-character word
// with the highest value possible, i.e. the least likely to occur.
// Exclude Korean characters from this treatment, as they should be left
// together by default.
if((count == 0 || lengths[0] != 1) &&
!fHangulWordSet.contains(utext_current32(&normalizedText))) {
values[count] = maxSnlp;
lengths[count++] = 1;
}
for (int j = 0; j < count; j++) {
uint32_t newSnlp = bestSnlp[i] + values[j];
if (newSnlp < bestSnlp[lengths[j] + i]) {
bestSnlp[lengths[j] + i] = newSnlp;
prev[lengths[j] + i] = i;
}
}
// In Japanese,
// Katakana word in single character is pretty rare. So we apply
// the following heuristic to Katakana: any continuous run of Katakana
// characters is considered a candidate word with a default cost
// specified in the katakanaCost table according to its length.
//utext_setNativeIndex(text, rangeStart + i);
utext_setNativeIndex(&normalizedText, i);
bool is_katakana = isKatakana(utext_current32(&normalizedText));
if (!is_prev_katakana && is_katakana) {
int j = i + 1;
utext_next32(&normalizedText);
// Find the end of the continuous run of Katakana characters
while (j < numChars && (j - i) < kMaxKatakanaGroupLength &&
isKatakana(utext_current32(&normalizedText))) {
utext_next32(&normalizedText);
++j;
}
if ((j - i) < kMaxKatakanaGroupLength) {
uint32_t newSnlp = bestSnlp[i] + getKatakanaCost(j - i);
if (newSnlp < bestSnlp[j]) {
bestSnlp[j] = newSnlp;
prev[j] = i;
}
}
}
is_prev_katakana = is_katakana;
}
// Start pushing the optimal offset index into t_boundary (t for tentative).
// prev[numChars] is guaranteed to be meaningful.
// We'll first push in the reverse order, i.e.,
// t_boundary[0] = numChars, and afterwards do a swap.
// TODO: Replace by UVector32.
AutoBuffer<int, maxWordSize> t_boundary(numChars + 1);
int numBreaks = 0;
// No segmentation found, set boundary to end of range
if (bestSnlp[numChars] == kuint32max) {
t_boundary[numBreaks++] = numChars;
} else {
for (int i = numChars; i > 0; i = prev[i]) {
t_boundary[numBreaks++] = i;
}
U_ASSERT(prev[t_boundary[numBreaks - 1]] == 0);
}
// Reverse offset index in t_boundary.
// Don't add a break for the start of the dictionary range if there is one
// there already.
if (foundBreaks.size() == 0 || foundBreaks.peeki() < rangeStart) {
t_boundary[numBreaks++] = 0;
}
// Now that we're done, convert positions in t_bdry[] (indices in
// the normalized input string) back to indices in the raw input string
// while reversing t_bdry and pushing values to foundBreaks.
for (int i = numBreaks-1; i >= 0; i--) {
foundBreaks.push(charPositions[t_boundary[i]] + rangeStart, status);
}
utext_close(&normalizedText);
return numBreaks;
}
static void TestAPI(void) {
UErrorCode status = U_ZERO_ERROR;
UBool gFailed = FALSE;
(void)gFailed; /* Suppress set but not used warning. */
/* Open */
{
UText utLoc = UTEXT_INITIALIZER;
const char * cString = "\x61\x62\x63\x64";
UChar uString[] = {0x41, 0x42, 0x43, 0};
UText *uta;
UText *utb;
UChar c;
uta = utext_openUChars(NULL, uString, -1, &status);
TEST_SUCCESS(status);
c = utext_next32(uta);
TEST_ASSERT(c == 0x41);
utb = utext_close(uta);
TEST_ASSERT(utb == NULL);
uta = utext_openUTF8(&utLoc, cString, -1, &status);
TEST_SUCCESS(status);
TEST_ASSERT(uta == &utLoc);
uta = utext_close(&utLoc);
TEST_ASSERT(uta == &utLoc);
}
/* utext_clone() */
{
UChar uString[] = {0x41, 0x42, 0x43, 0};
int64_t len;
UText *uta;
UText *utb;
status = U_ZERO_ERROR;
uta = utext_openUChars(NULL, uString, -1, &status);
TEST_SUCCESS(status);
utb = utext_clone(NULL, uta, FALSE, FALSE, &status);
TEST_SUCCESS(status);
TEST_ASSERT(utb != NULL);
TEST_ASSERT(utb != uta);
len = utext_nativeLength(uta);
TEST_ASSERT(len == u_strlen(uString));
utext_close(uta);
utext_close(utb);
}
/* basic access functions */
{
UChar uString[] = {0x41, 0x42, 0x43, 0};
UText *uta;
UChar32 c;
int64_t len;
UBool b;
int64_t i;
status = U_ZERO_ERROR;
uta = utext_openUChars(NULL, uString, -1, &status);
TEST_ASSERT(uta!=NULL);
TEST_SUCCESS(status);
b = utext_isLengthExpensive(uta);
TEST_ASSERT(b==TRUE);
len = utext_nativeLength(uta);
TEST_ASSERT(len == u_strlen(uString));
b = utext_isLengthExpensive(uta);
TEST_ASSERT(b==FALSE);
c = utext_char32At(uta, 0);
TEST_ASSERT(c==uString[0]);
c = utext_current32(uta);
TEST_ASSERT(c==uString[0]);
c = utext_next32(uta);
TEST_ASSERT(c==uString[0]);
c = utext_current32(uta);
TEST_ASSERT(c==uString[1]);
c = utext_previous32(uta);
TEST_ASSERT(c==uString[0]);
c = utext_current32(uta);
TEST_ASSERT(c==uString[0]);
c = utext_next32From(uta, 1);
TEST_ASSERT(c==uString[1]);
c = utext_next32From(uta, u_strlen(uString));
TEST_ASSERT(c==U_SENTINEL);
c = utext_previous32From(uta, 2);
TEST_ASSERT(c==uString[1]);
i = utext_getNativeIndex(uta);
TEST_ASSERT(i == 1);
utext_setNativeIndex(uta, 0);
b = utext_moveIndex32(uta, 1);
TEST_ASSERT(b==TRUE);
i = utext_getNativeIndex(uta);
TEST_ASSERT(i==1);
b = utext_moveIndex32(uta, u_strlen(uString)-1);
TEST_ASSERT(b==TRUE);
i = utext_getNativeIndex(uta);
TEST_ASSERT(i==u_strlen(uString));
b = utext_moveIndex32(uta, 1);
TEST_ASSERT(b==FALSE);
i = utext_getNativeIndex(uta);
TEST_ASSERT(i==u_strlen(uString));
utext_setNativeIndex(uta, 0);
c = UTEXT_NEXT32(uta);
TEST_ASSERT(c==uString[0]);
c = utext_current32(uta);
TEST_ASSERT(c==uString[1]);
c = UTEXT_PREVIOUS32(uta);
TEST_ASSERT(c==uString[0]);
c = UTEXT_PREVIOUS32(uta);
TEST_ASSERT(c==U_SENTINEL);
utext_close(uta);
}
{
/*
* UText opened on a NULL string with zero length
*/
UText *uta;
UChar32 c;
status = U_ZERO_ERROR;
uta = utext_openUChars(NULL, NULL, 0, &status);
TEST_SUCCESS(status);
c = UTEXT_NEXT32(uta);
TEST_ASSERT(c == U_SENTINEL);
utext_close(uta);
uta = utext_openUTF8(NULL, NULL, 0, &status);
TEST_SUCCESS(status);
c = UTEXT_NEXT32(uta);
TEST_ASSERT(c == U_SENTINEL);
utext_close(uta);
}
{
/*
* extract
*/
UText *uta;
UChar uString[] = {0x41, 0x42, 0x43, 0};
UChar buf[100];
int32_t i;
/* Test pinning of input bounds */
UChar uString2[] = {0x41, 0x42, 0x43, 0x44, 0x45,
0x46, 0x47, 0x48, 0x49, 0x4A, 0};
UChar * uString2Ptr = uString2 + 5;
status = U_ZERO_ERROR;
uta = utext_openUChars(NULL, uString, -1, &status);
TEST_SUCCESS(status);
status = U_ZERO_ERROR;
i = utext_extract(uta, 0, 100, NULL, 0, &status);
TEST_ASSERT(status==U_BUFFER_OVERFLOW_ERROR);
TEST_ASSERT(i == u_strlen(uString));
status = U_ZERO_ERROR;
memset(buf, 0, sizeof(buf));
i = utext_extract(uta, 0, 100, buf, 100, &status);
TEST_SUCCESS(status);
TEST_ASSERT(i == u_strlen(uString));
i = u_strcmp(uString, buf);
TEST_ASSERT(i == 0);
utext_close(uta);
/* Test pinning of input bounds */
status = U_ZERO_ERROR;
uta = utext_openUChars(NULL, uString2Ptr, -1, &status);
TEST_SUCCESS(status);
status = U_ZERO_ERROR;
memset(buf, 0, sizeof(buf));
i = utext_extract(uta, -3, 20, buf, 100, &status);
TEST_SUCCESS(status);
TEST_ASSERT(i == u_strlen(uString2Ptr));
i = u_strcmp(uString2Ptr, buf);
TEST_ASSERT(i == 0);
utext_close(uta);
}
{
/*
* Copy, Replace, isWritable
* Can't create an editable UText from plain C, so all we
* can easily do is check that errors returned.
*/
UText *uta;
UChar uString[] = {0x41, 0x42, 0x43, 0};
UBool b;
status = U_ZERO_ERROR;
uta = utext_openUChars(NULL, uString, -1, &status);
TEST_SUCCESS(status);
b = utext_isWritable(uta);
TEST_ASSERT(b == FALSE);
b = utext_hasMetaData(uta);
TEST_ASSERT(b == FALSE);
utext_replace(uta,
0, 1, /* start, limit */
uString, -1, /* replacement, replacement length */
&status);
TEST_ASSERT(status == U_NO_WRITE_PERMISSION);
utext_copy(uta,
0, 1, /* start, limit */
2, /* destination index */
FALSE, /* move flag */
&status);
TEST_ASSERT(status == U_NO_WRITE_PERMISSION);
utext_close(uta);
}
}
