RuleBasedCollator::~RuleBasedCollator()
{
if (dataIsOwned)
{
ucol_close(ucollator);
delete urulestring;
}
ucollator = 0;
urulestring = 0;
}
void Collator::releaseCollator()
{
{
Locker<Mutex> lock(cachedCollatorMutex());
if (cachedCollator)
ucol_close(cachedCollator);
cachedCollator = m_collator;
m_collator = 0;
}
}
void c_Collator::t___construct(CStrRef locale) {
INSTANCE_METHOD_INJECTION_BUILTIN(Collator, Collator::__construct);
if (m_ucoll) {
ucol_close(m_ucoll);
m_ucoll = NULL;
}
m_errcode.clear();
if (!locale.empty()) {
m_locale = locale;
m_ucoll = ucol_open(locale.data(), &(m_errcode.code));
if (!U_FAILURE(m_errcode.code)) {
// If the specified locale opened successfully, return
s_intl_error->m_error.clear();
s_intl_error->m_error.code = m_errcode.code;
return;
}
}
// If the empty string was given or if the specified locale did
// not open successfully, so fall back to using the default locale
m_errcode.code = U_USING_FALLBACK_WARNING;
s_intl_error->m_error.clear();
s_intl_error->m_error.code = m_errcode.code;
if (m_ucoll) {
ucol_close(m_ucoll);
m_ucoll = NULL;
}
UErrorCode errcode = U_ZERO_ERROR;
m_locale = String(uloc_getDefault(), CopyString);
m_ucoll = ucol_open(m_locale.data(), &errcode);
if (U_FAILURE(errcode)) {
m_errcode.code = errcode;
m_errcode.custom_error_message =
"collator_create: unable to open ICU collator";
s_intl_error->m_error.clear();
s_intl_error->m_error.code = m_errcode.code;
s_intl_error->m_error.custom_error_message = m_errcode.custom_error_message;
if (m_ucoll) {
ucol_close(m_ucoll);
m_ucoll = NULL;
}
}
}
Collator::~Collator()
{
std::lock_guard<Lock> lock(cachedCollatorMutex);
if (cachedCollator) {
ucol_close(cachedCollator);
fastFree(cachedCollatorLocale);
}
cachedCollator = m_collator;
cachedCollatorLocale = m_locale;
cachedCollatorShouldSortLowercaseFirst = m_shouldSortLowercaseFirst;
}
void CollationRegressionTest::TestT7189() {
UErrorCode status = U_ZERO_ERROR;
UCollator *coll;
uint32_t i;
static const UChar text1[][CollationRegressionTest::MAX_TOKEN_LEN] = {
// "Achter De Hoven"
{ 0x41, 0x63, 0x68, 0x74, 0x65, 0x72, 0x20, 0x44, 0x65, 0x20, 0x48, 0x6F, 0x76, 0x65, 0x6E, 0x00 },
// "ABC"
{ 0x41, 0x42, 0x43, 0x00 },
// "HELLO world!"
{ 0x48, 0x45, 0x4C, 0x4C, 0x4F, 0x20, 0x77, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x00 }
};
static const UChar text2[][CollationRegressionTest::MAX_TOKEN_LEN] = {
// "Achter de Hoven"
{ 0x41, 0x63, 0x68, 0x74, 0x65, 0x72, 0x20, 0x64, 0x65, 0x20, 0x48, 0x6F, 0x76, 0x65, 0x6E, 0x00 },
// "abc"
{ 0x61, 0x62, 0x63, 0x00 },
// "hello world!"
{ 0x68, 0x65, 0x6C, 0x6C, 0x6F, 0x20, 0x77, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x00 }
};
// Open the collator
coll = ucol_openFromShortString("EO_S1", FALSE, NULL, &status);
if (U_FAILURE(status)) {
errln("Failed to create a collator for short string EO_S1");
return;
}
for (i = 0; i < sizeof(text1) / (CollationRegressionTest::MAX_TOKEN_LEN * sizeof(UChar)); i++) {
uint8_t key1[100], key2[100];
int32_t len1, len2;
len1 = calcKeyIncremental(coll, text1[i], -1, key1, sizeof(key1), status);
if (U_FAILURE(status)) {
errln(UnicodeString("Failed to get a partial collation key for ") + text1[i]);
break;
}
len2 = calcKeyIncremental(coll, text2[i], -1, key2, sizeof(key2), status);
if (U_FAILURE(status)) {
errln(UnicodeString("Failed to get a partial collation key for ") + text2[i]);
break;
}
if (len1 == len2 && uprv_memcmp(key1, key2, len1) == 0) {
errln(UnicodeString("Failed: Identical key\n") + " text1: " + text1[i] + "\n" + " text2: " + text2[i] + "\n" + " key : " + TestUtility::hex(key1, len1));
} else {
logln(UnicodeString("Keys produced -\n") + " text1: " + text1[i] + "\n" + " key1 : " + TestUtility::hex(key1, len1) + "\n" + " text2: " + text2[i] + "\n" + " key2 : "
+ TestUtility::hex(key2, len2));
}
}
ucol_close(coll);
}
static void
TestJitterbug1098(){
UChar rule[1000];
UCollator* c1 = NULL;
UErrorCode status = U_ZERO_ERROR;
UParseError parseError;
char preContext[200]={0};
char postContext[200]={0};
int i=0;
const char* rules[] = {
"&''<\\\\",
"&\\'<\\\\",
"&\\\"<'\\'",
"&'\"'<\\'",
'\0'
};
const UCollationResult results1098[] = {
UCOL_LESS,
UCOL_LESS,
UCOL_LESS,
UCOL_LESS,
};
const UChar input[][2]= {
{0x0027,0x005c},
{0x0027,0x005c},
{0x0022,0x005c},
{0x0022,0x0027},
};
UChar X[2] ={0};
UChar Y[2] ={0};
u_memset(parseError.preContext,0x0000,U_PARSE_CONTEXT_LEN);
u_memset(parseError.postContext,0x0000,U_PARSE_CONTEXT_LEN);
for(;rules[i]!=0;i++){
u_uastrcpy(rule, rules[i]);
c1 = ucol_openRules(rule, u_strlen(rule), UCOL_OFF, UCOL_DEFAULT_STRENGTH, &parseError, &status);
if(U_FAILURE(status)){
log_err("Could not parse the rules syntax. Error: %s ", u_errorName(status));
if (status == U_PARSE_ERROR) {
u_UCharsToChars(parseError.preContext,preContext,20);
u_UCharsToChars(parseError.postContext,postContext,20);
log_verbose("\n\tPre-Context: %s \n\tPost-Context:%s \n",preContext,postContext);
}
return;
}
X[0] = input[i][0];
Y[0] = input[i][1];
doTest(c1,X,Y,results1098[i]);
ucol_close(c1);
}
}
void
RuleBasedCollator::setUCollator(const char *locale,
UErrorCode &status)
{
if (U_FAILURE(status))
return;
if (ucollator && dataIsOwned)
ucol_close(ucollator);
ucollator = ucol_open_internal(locale, &status);
dataIsOwned = TRUE;
isWriteThroughAlias = FALSE;
}
void Collator::releaseCollator()
{
#ifndef USE_TI_UCOL_REPLACEMENTS
{
Locker<Mutex> lock(cachedCollatorMutex());
if (cachedCollator)
ucol_close(cachedCollator);
cachedCollator = m_collator;
m_collator = 0;
}
#endif
}
void Collator::releaseCollator()
{
{
Locker<Mutex> lock(cachedCollatorMutex());
if (cachedCollator)
ucol_close(cachedCollator);
cachedCollator = m_collator;
strncpy(cachedEquivalentLocale, m_equivalentLocale, ulocFullnameCapacity);
m_collator = 0;
}
m_collator = 0;
}
static void TestJB581(void)
{
UChar dispName [100];
int32_t bufferLen = 0;
UChar source [100];
UChar target [100];
UCollationResult result = UCOL_EQUAL;
uint8_t sourceKeyArray [100];
uint8_t targetKeyArray [100];
int32_t sourceKeyOut = 0,
targetKeyOut = 0;
UCollator *myCollator = 0;
UErrorCode status = U_ZERO_ERROR;
/*u_uastrcpy(source, "This is a test.");*/
/*u_uastrcpy(target, "THISISATEST.");*/
u_uastrcpy(source, "THISISATEST.");
u_uastrcpy(target, "Thisisatest.");
myCollator = ucol_open("en_US", &status);
if (U_FAILURE(status)){
bufferLen = uloc_getDisplayName("en_US", 0, dispName, 100, &status);
/*Report the error with display name... */
log_err("ERROR: Failed to create the collator for : \"%s\"\n", dispName);
return;
}
result = ucol_strcoll(myCollator, source, -1, target, -1);
/* result is 1, secondary differences only for ignorable space characters*/
if (result != 1)
{
log_err("Comparing two strings with only secondary differences in C failed.\n");
}
/* To compare them with just primary differences */
ucol_setStrength(myCollator, UCOL_PRIMARY);
result = ucol_strcoll(myCollator, source, -1, target, -1);
/* result is 0 */
if (result != 0)
{
log_err("Comparing two strings with no differences in C failed.\n");
}
/* Now, do the same comparison with keys */
sourceKeyOut = ucol_getSortKey(myCollator, source, -1, sourceKeyArray, 100);
targetKeyOut = ucol_getSortKey(myCollator, target, -1, targetKeyArray, 100);
bufferLen = ((targetKeyOut > 100) ? 100 : targetKeyOut);
if (memcmp(sourceKeyArray, targetKeyArray, bufferLen) != 0)
{
log_err("Comparing two strings with sort keys in C failed.\n");
}
ucol_close(myCollator);
}
void
on_unload(ErlNifEnv* env, void* priv_data)
{
couch_ejson_global_ctx_t *globalCtx = (couch_ejson_global_ctx_t *) priv_data;
int i;
for (i = 0; i < globalCtx->numCollators; i++) {
ucol_close(globalCtx->collators[i]);
}
enif_free(globalCtx->collators);
enif_mutex_destroy(globalCtx->collMutex);
enif_free(globalCtx);
}
U_CDECL_BEGIN
static UBool U_CALLCONV
ucol_res_cleanup(void)
{
if (UCA_DATA_MEM) {
udata_close(UCA_DATA_MEM);
UCA_DATA_MEM = NULL;
}
if (_staticUCA) {
ucol_close(_staticUCA);
_staticUCA = NULL;
}
return TRUE;
}
void IntlCollator::createCollator(ExecState& state)
{
ASSERT(!m_collator);
if (!m_initializedCollator) {
initializeCollator(state, jsUndefined(), jsUndefined());
ASSERT(!state.hadException());
}
UErrorCode status = U_ZERO_ERROR;
UCollator* collator = ucol_open(m_locale.utf8().data(), &status);
if (U_FAILURE(status))
return;
UColAttributeValue strength = UCOL_PRIMARY;
UColAttributeValue caseLevel = UCOL_OFF;
switch (m_sensitivity) {
case Sensitivity::Base:
break;
case Sensitivity::Accent:
strength = UCOL_SECONDARY;
break;
case Sensitivity::Case:
caseLevel = UCOL_ON;
break;
case Sensitivity::Variant:
strength = UCOL_TERTIARY;
break;
default:
ASSERT_NOT_REACHED();
}
ucol_setAttribute(collator, UCOL_STRENGTH, strength, &status);
ucol_setAttribute(collator, UCOL_CASE_LEVEL, caseLevel, &status);
ucol_setAttribute(collator, UCOL_NUMERIC_COLLATION, m_numeric ? UCOL_ON : UCOL_OFF, &status);
// FIXME: Setting UCOL_ALTERNATE_HANDLING to UCOL_SHIFTED causes punctuation and whitespace to be
// ignored. There is currently no way to ignore only punctuation.
ucol_setAttribute(collator, UCOL_ALTERNATE_HANDLING, m_ignorePunctuation ? UCOL_SHIFTED : UCOL_DEFAULT, &status);
// "The method is required to return 0 when comparing Strings that are considered canonically
// equivalent by the Unicode standard."
ucol_setAttribute(collator, UCOL_NORMALIZATION_MODE, UCOL_ON, &status);
if (U_FAILURE(status)) {
ucol_close(collator);
return;
}
m_collator = collator;
}
int main() {
UErrorCode status = U_ZERO_ERROR;
UCollator *coll = ucol_open(0, &status);
ucol_setStrength(coll, UCOL_PRIMARY);
for (int i = 0; i < PASSES; ++i) {
UCollationResult coll_res = ucol_strcollUTF8(coll, INPUT1, -1, INPUT2, -1, &status);
(void)(coll_res);
}
ucol_close(coll);
return 0;
}
void icu_chain_destroy(struct icu_chain *chain)
{
if (chain)
{
if (chain->coll)
ucol_close(chain->coll);
if (chain->iter)
icu_iter_destroy(chain->iter);
icu_chain_step_destroy(chain->csteps);
xfree(chain->locale);
xfree(chain);
}
}
nsresult nsCollationMacUC::CleanUpCollator(void)
{
if (mHasCollator) {
if (mUseICU) {
ucol_close(mCollatorICU);
mHasCollator = false;
} else {
OSStatus err = ::UCDisposeCollator(&mCollator);
mHasCollator = false;
NS_ENSURE_TRUE((err == noErr), NS_ERROR_FAILURE);
}
}
return NS_OK;
}
void genericLocaleStarterWithResult(const char *locale, const char * const s[], uint32_t size, UCollationResult result) {
UErrorCode status = U_ZERO_ERROR;
UCollator *coll = ucol_open(locale, &status);
log_verbose("Locale starter for %s\n", locale);
if(U_SUCCESS(status)) {
genericOrderingTestWithResult(coll, s, size, result);
} else if(status == U_FILE_ACCESS_ERROR) {
log_data_err("Is your data around?\n");
return;
} else {
log_err("Unable to open collator for locale %s\n", locale);
}
ucol_close(coll);
}
static void TestPrimary()
{
int32_t i;
UErrorCode status = U_ZERO_ERROR;
myCollation = ucol_open("fi_FI@collation=standard", &status);
if(U_FAILURE(status)){
log_err_status(status, "ERROR: in creation of rule based collator: %s\n", myErrorName(status));
}
log_verbose("Testing Finnish Collation with Primary strength\n");
ucol_setStrength(myCollation, UCOL_PRIMARY);
for (i = 4; i < 5; i++)
{
doTest(myCollation, testSourceCases[i], testTargetCases[i], results[i]);
}
ucol_close(myCollation);
}
/*
* call-seq:
* string.unicode_sort_key -> string
*
* Returns a string that will sort according to the Unicode collation algorithm.
*
*/
static VALUE unicode_sort_key(VALUE string) {
char str[BUF_SIZE];
UChar ustr[BUF_SIZE];
int32_t len = 0;
int32_t ulen = 0;
UErrorCode status = U_ZERO_ERROR;
UCollator *col;
to_utf16(string, ustr, &ulen);
col = ucol_open("en_US", &status);
if (U_SUCCESS(status)) {
len = ucol_getSortKey(col, ustr, ulen, (uint8_t*)str, BUF_SIZE);
ucol_close(col);
}
return rb_str_new(str, len - 1);
}
// aliasing, not write-through
RuleBasedCollator& RuleBasedCollator::operator=(const RuleBasedCollator& that)
{
if (this != &that)
{
if (dataIsOwned)
{
ucol_close(ucollator);
ucollator = NULL;
delete urulestring;
}
dataIsOwned = FALSE;
isWriteThroughAlias = FALSE;
ucollator = that.ucollator;
urulestring = that.urulestring;
}
return *this;
}
static void TestTertiary( )
{
int32_t i;
UErrorCode status = U_ZERO_ERROR;
myCollation = ucol_open("es_ES", &status);
if(U_FAILURE(status)){
log_err_status(status, "ERROR: %s: in creation of rule based collator: %s\n", __FILE__, myErrorName(status));
return;
}
log_verbose("Testing Spanish Collation with Tertiary strength\n");
ucol_setStrength(myCollation, UCOL_TERTIARY);
for (i = 0; i < 5 ; i++)
{
doTest(myCollation, testSourceCases[i], testTargetCases[i], results[i]);
}
ucol_close(myCollation);
}
static void TestTertiary( )
{
int32_t i;
UErrorCode status = U_ZERO_ERROR;
myCollation = ucol_open("ja_JP", &status);
if(U_FAILURE(status)){
log_err_status(status, "ERROR: in creation of rule based collator: %s\n", myErrorName(status));
return;
}
log_verbose("Testing Kanna(Japan) Collation with Tertiary strength\n");
ucol_setStrength(myCollation, UCOL_TERTIARY);
ucol_setAttribute(myCollation, UCOL_CASE_LEVEL, UCOL_ON, &status);
for (i = 0; i < 6 ; i++)
{
doTest(myCollation, testSourceCases[i], testTargetCases[i], results[i]);
}
ucol_close(myCollation);
}
static void TestPrimary()
{
int32_t i;
UErrorCode status = U_ZERO_ERROR;
myCollation = ucol_open("en_US", &status);
if(U_FAILURE(status)){
log_err("ERROR: in creation of rule based collator: %s\n", myErrorName(status));
return;
}
ucol_setStrength(myCollation, UCOL_PRIMARY);
log_verbose("Testing English Collation with Primary strength\n");
for (i = 38; i < 43 ; i++)
{
doTest(myCollation, testSourceCases[i], testTargetCases[i], results[i]);
}
ucol_close(myCollation);
}
void genericRulesStarterWithResult(const char *rules, const char * const s[], uint32_t size, UCollationResult result) {
UErrorCode status = U_ZERO_ERROR;
UChar rlz[RULE_BUFFER_LEN] = { 0 };
uint32_t rlen = u_unescape(rules, rlz, RULE_BUFFER_LEN);
UCollator *coll = NULL;
coll = ucol_openRules(rlz, rlen, UCOL_DEFAULT, UCOL_DEFAULT,NULL, &status);
log_verbose("Rules starter for %s\n", rules);
if(U_SUCCESS(status)) {
genericOrderingTestWithResult(coll, s, size, result);
ucol_close(coll);
} else if(status == U_FILE_ACCESS_ERROR) {
log_data_err("Is your data around?\n");
} else {
log_err("Unable to open collator with rules %s\n", rules);
}
}
StringSearchPerformanceTest::~StringSearchPerformanceTest() {
CollData *data = bms->getData();
UCollator *coll = data->getCollator();
delete bms;
delete targetString;
CollData::close(data);
ucol_close(coll);
if (pttrn != NULL) {
free(pttrn);
}
#ifndef TEST_BOYER_MOORE_SEARCH
if (srch != NULL) {
usearch_close(srch);
}
#endif
}
extern "C" SortHandle* GetSortHandle(const char* lpLocaleName)
{
SortHandle* pSortHandle = new SortHandle();
UErrorCode err = U_ZERO_ERROR;
pSortHandle->regular = ucol_open(lpLocaleName, &err);
if (U_FAILURE(err))
{
if (pSortHandle->regular != nullptr)
ucol_close(pSortHandle->regular);
delete pSortHandle;
pSortHandle = nullptr;
}
return pSortHandle;
}
/* Testing base letters */
static void TestBase()
{
int32_t i;
UErrorCode status = U_ZERO_ERROR;
myCollation = ucol_open("ja_JP", &status);
if (U_FAILURE(status))
{
log_err_status(status, "ERROR: in creation of rule based collator: %s\n",
myErrorName(status));
return;
}
log_verbose("Testing Japanese Base Characters Collation\n");
ucol_setStrength(myCollation, UCOL_PRIMARY);
for (i = 0; i < 3 ; i++)
doTest(myCollation, testBaseCases[i], testBaseCases[i + 1], UCOL_LESS);
ucol_close(myCollation);
}
static void TestTertiary( )
{
int32_t testMoreSize;
UCollationResult expected=UCOL_EQUAL;
int32_t i,j;
UErrorCode status = U_ZERO_ERROR;
myCollation = ucol_open("en_US", &status);
if(U_FAILURE(status)){
log_err_status(status, "ERROR: in creation of rule based collator: %s\n", myErrorName(status));
return;
}
log_verbose("Testing English Collation with Tertiary strength\n");
ucol_setStrength(myCollation, UCOL_TERTIARY);
for (i = 0; i < 38 ; i++)
{
doTest(myCollation, testSourceCases[i], testTargetCases[i], results[i]);
}
j = 0;
for (i = 0; i < 10; i++)
{
for (j = i+1; j < 10; j++)
{
doTest(myCollation, testBugs[i], testBugs[j], UCOL_LESS);
}
}
/*test more interesting cases */
testMoreSize = UPRV_LENGTHOF(testMore);
for (i = 0; i < testMoreSize; i++)
{
for (j = 0; j < testMoreSize; j++)
{
if (i < j) expected = UCOL_LESS;
if (i == j) expected = UCOL_EQUAL;
if (i > j) expected = UCOL_GREATER;
doTest(myCollation, testMore[i], testMore[j], expected );
}
}
ucol_close(myCollation);
}
static void TestExtra()
{
int32_t i, j;
UErrorCode status = U_ZERO_ERROR;
myCollation = ucol_open("fr_CA", &status);
if(U_FAILURE(status)) {
log_err_status(status, "ERROR: in creation of rule based collator: %s\n", myErrorName(status));
return;
}
log_verbose("Testing fr_CA Collation extra with secondary strength\n");
ucol_setStrength(myCollation, UCOL_TERTIARY);
for (i = 0; i < 9 ; i++)
{
for (j = i + 1; j < 10; j += 1)
{
doTest(myCollation, testBugs[i], testBugs[j], UCOL_LESS);
}
}
ucol_close(myCollation);
}
extern "C" int32_t GetSortKey(const char* lpLocaleName,
const UChar* lpStr,
int32_t cwStrLength,
uint8_t* sortKey,
int32_t cbSortKeyLength,
int32_t options)
{
UErrorCode err = U_ZERO_ERROR;
UCollator* pColl = GetCollatorForLocaleAndOptions(lpLocaleName, options, &err);
int32_t result = 0;
if (U_SUCCESS(err))
{
result = ucol_getSortKey(pColl, lpStr, cwStrLength, sortKey, cbSortKeyLength);
ucol_close(pColl);
}
return result;
}
