static UBool U_CALLCONV
isAcceptable2(void *context,
const char *type, const char *name,
const UDataInfo *pInfo){
UVersionInfo unicodeVersion;
u_getUnicodeVersion(unicodeVersion);
if( pInfo->size>=20 &&
pInfo->isBigEndian==U_IS_BIG_ENDIAN &&
pInfo->charsetFamily==U_CHARSET_FAMILY &&
pInfo->dataFormat[0]==0x75 && /* dataFormat="unam" */
pInfo->dataFormat[1]==0x6e &&
pInfo->dataFormat[2]==0x61 &&
pInfo->dataFormat[3]==0x6d &&
pInfo->formatVersion[0]==1 &&
pInfo->dataVersion[0]==unicodeVersion[0] )
{
log_verbose("The data from \"%s.%s\" IS acceptable using the verifing function isAcceptable2()\n", name, type);
return TRUE;
} else {
log_verbose("The data from \"%s.%s\" IS NOT acceptable using the verifing function isAcceptable2()\n", name, type);
return FALSE;
}
}
static UBool U_CALLCONV
isAcceptableUCA(void * /*context*/,
const char * /*type*/, const char * /*name*/,
const UDataInfo *pInfo){
/* context, type & name are intentionally not used */
if( pInfo->size>=20 &&
pInfo->isBigEndian==U_IS_BIG_ENDIAN &&
pInfo->charsetFamily==U_CHARSET_FAMILY &&
pInfo->dataFormat[0]==UCA_DATA_FORMAT_0 && /* dataFormat="UCol" */
pInfo->dataFormat[1]==UCA_DATA_FORMAT_1 &&
pInfo->dataFormat[2]==UCA_DATA_FORMAT_2 &&
pInfo->dataFormat[3]==UCA_DATA_FORMAT_3 &&
pInfo->formatVersion[0]==UCA_FORMAT_VERSION_0 &&
pInfo->formatVersion[1]>=UCA_FORMAT_VERSION_1// &&
//pInfo->formatVersion[1]==UCA_FORMAT_VERSION_1 &&
//pInfo->formatVersion[2]==UCA_FORMAT_VERSION_2 && // Too harsh
//pInfo->formatVersion[3]==UCA_FORMAT_VERSION_3 && // Too harsh
) {
UVersionInfo UCDVersion;
u_getUnicodeVersion(UCDVersion);
return (UBool)(pInfo->dataVersion[0]==UCDVersion[0]
&& pInfo->dataVersion[1]==UCDVersion[1]);
//&& pInfo->dataVersion[2]==ucaDataInfo.dataVersion[2]
//&& pInfo->dataVersion[3]==ucaDataInfo.dataVersion[3]);
} else {
return FALSE;
}
}
Variant HHVM_STATIC_METHOD(IntlChar, getUnicodeVersion) {
UVersionInfo version;
u_getUnicodeVersion(version);
Array ret = Array::Create();
for(int i = 0; i < U_MAX_VERSION_LENGTH; ++i) {
ret.append(version[i]);
}
return ret;
}
UCAConformanceTest::UCAConformanceTest() :
rbUCA(NULL),
testFile(NULL),
status(U_ZERO_ERROR)
{
UCA = (RuleBasedCollator *)Collator::createInstance(Locale::getRoot(), status);
if(U_FAILURE(status)) {
dataerrln("Error - UCAConformanceTest: Unable to open UCA collator! - %s", u_errorName(status));
}
const char *srcDir = IntlTest::getSourceTestData(status);
if (U_FAILURE(status)) {
dataerrln("Could not open test data %s", u_errorName(status));
return;
}
uprv_strcpy(testDataPath, srcDir);
uprv_strcat(testDataPath, "CollationTest_");
UVersionInfo uniVersion;
static const UVersionInfo v62 = { 6, 2, 0, 0 };
u_getUnicodeVersion(uniVersion);
isAtLeastUCA62 = uprv_memcmp(uniVersion, v62, 4) >= 0;
}
PyMODINIT_FUNC
initicu(void)
{
PyObject* m;
UVersionInfo ver, uver;
UErrorCode status = U_ZERO_ERROR;
char version[U_MAX_VERSION_STRING_LENGTH+1] = {0}, uversion[U_MAX_VERSION_STRING_LENGTH+5] = {0};
if (sizeof(Py_UNICODE) != 2 && sizeof(Py_UNICODE) != 4) {
PyErr_SetString(PyExc_RuntimeError, "This module only works on python versions <= 3.2");
return;
}
u_init(&status);
if (U_FAILURE(status)) {
PyErr_SetString(PyExc_RuntimeError, u_errorName(status));
return;
}
u_getVersion(ver);
u_versionToString(ver, version);
u_getUnicodeVersion(uver);
u_versionToString(uver, uversion);
if (PyType_Ready(&icu_CollatorType) < 0)
return;
if (PyType_Ready(&icu_BreakIteratorType) < 0)
return;
m = Py_InitModule3("icu", icu_methods,
"Wrapper for the ICU internationalization library");
Py_INCREF(&icu_CollatorType); Py_INCREF(&icu_BreakIteratorType);
PyModule_AddObject(m, "Collator", (PyObject *)&icu_CollatorType);
PyModule_AddObject(m, "BreakIterator", (PyObject *)&icu_BreakIteratorType);
// uint8_t must be the same size as char
PyModule_AddIntConstant(m, "ok", (U_SUCCESS(status) && sizeof(uint8_t) == sizeof(char)) ? 1 : 0);
PyModule_AddStringConstant(m, "icu_version", version);
PyModule_AddStringConstant(m, "unicode_version", uversion);
ADDUCONST(USET_SPAN_NOT_CONTAINED);
ADDUCONST(USET_SPAN_CONTAINED);
ADDUCONST(USET_SPAN_SIMPLE);
ADDUCONST(UCOL_DEFAULT);
ADDUCONST(UCOL_PRIMARY);
ADDUCONST(UCOL_SECONDARY);
ADDUCONST(UCOL_TERTIARY);
ADDUCONST(UCOL_DEFAULT_STRENGTH);
ADDUCONST(UCOL_QUATERNARY);
ADDUCONST(UCOL_IDENTICAL);
ADDUCONST(UCOL_OFF);
ADDUCONST(UCOL_ON);
ADDUCONST(UCOL_SHIFTED);
ADDUCONST(UCOL_NON_IGNORABLE);
ADDUCONST(UCOL_LOWER_FIRST);
ADDUCONST(UCOL_UPPER_FIRST);
ADDUCONST(UNORM_NONE);
ADDUCONST(UNORM_NFD);
ADDUCONST(UNORM_NFKD);
ADDUCONST(UNORM_NFC);
ADDUCONST(UNORM_DEFAULT);
ADDUCONST(UNORM_NFKC);
ADDUCONST(UNORM_FCD);
ADDUCONST(UPPER_CASE);
ADDUCONST(LOWER_CASE);
ADDUCONST(TITLE_CASE);
ADDUCONST(UBRK_CHARACTER);
ADDUCONST(UBRK_WORD);
ADDUCONST(UBRK_LINE);
ADDUCONST(UBRK_SENTENCE);
}
static void
TestCaseCompare(void) {
static const UChar
mixed[]= { 0x61, 0x42, 0x131, 0x3a3, 0xdf, 0xfb03, 0xd93f, 0xdfff, 0 },
otherDefault[]= { 0x41, 0x62, 0x131, 0x3c3, 0x73, 0x53, 0x46, 0x66, 0x49, 0xd93f, 0xdfff, 0 },
otherExcludeSpecialI[]={ 0x41, 0x62, 0x131, 0x3c3, 0x53, 0x73, 0x66, 0x46, 0x69, 0xd93f, 0xdfff, 0 },
different[]= { 0x41, 0x62, 0x131, 0x3c3, 0x73, 0x53, 0x46, 0x66, 0x49, 0xd93f, 0xdffd, 0 };
UVersionInfo unicodeVersion={ 0, 0, 17, 89 }, unicode_3_1={ 3, 1, 0, 0 };
int32_t result, lenMixed, lenOtherDefault, lenOtherExcludeSpecialI, lenDifferent;
UErrorCode errorCode;
UBool isUnicode_3_1;
errorCode=U_ZERO_ERROR;
lenMixed=u_strlen(mixed);
lenOtherDefault=u_strlen(otherDefault);
(void)lenOtherDefault; /* Suppress set but not used warning. */
lenOtherExcludeSpecialI=u_strlen(otherExcludeSpecialI);
lenDifferent=u_strlen(different);
/* if unicodeVersion()>=3.1 then test exclude-special-i cases as well */
u_getUnicodeVersion(unicodeVersion);
isUnicode_3_1= uprv_memcmp(unicodeVersion, unicode_3_1, 4)>=0;
(void)isUnicode_3_1; /* Suppress set but not used warning. */
/* test u_strcasecmp() */
result=u_strcasecmp(mixed, otherDefault, U_FOLD_CASE_DEFAULT);
if(result!=0) {
log_err("error: u_strcasecmp(mixed, other, default)=%ld instead of 0\n", result);
}
result=u_strCaseCompare(mixed, -1, otherDefault, -1, U_FOLD_CASE_DEFAULT, &errorCode);
if(result!=0) {
log_err("error: u_strCaseCompare(mixed, other, default)=%ld instead of 0\n", result);
}
/* test u_strcasecmp() - exclude special i */
result=u_strcasecmp(mixed, otherExcludeSpecialI, U_FOLD_CASE_EXCLUDE_SPECIAL_I);
if(result!=0) {
log_err("error: u_strcasecmp(mixed, other, exclude special i)=%ld instead of 0\n", result);
}
result=u_strCaseCompare(mixed, lenMixed, otherExcludeSpecialI, lenOtherExcludeSpecialI, U_FOLD_CASE_EXCLUDE_SPECIAL_I, &errorCode);
if(result!=0) {
log_err("error: u_strCaseCompare(mixed, other, exclude special i)=%ld instead of 0\n", result);
}
/* test u_strcasecmp() */
result=u_strcasecmp(mixed, different, U_FOLD_CASE_DEFAULT);
if(result<=0) {
log_err("error: u_strcasecmp(mixed, different, default)=%ld instead of positive\n", result);
}
result=u_strCaseCompare(mixed, -1, different, lenDifferent, U_FOLD_CASE_DEFAULT, &errorCode);
if(result<=0) {
log_err("error: u_strCaseCompare(mixed, different, default)=%ld instead of positive\n", result);
}
/* test u_strncasecmp() - stop before the sharp s (U+00df) */
result=u_strncasecmp(mixed, different, 4, U_FOLD_CASE_DEFAULT);
if(result!=0) {
log_err("error: u_strncasecmp(mixed, different, 4, default)=%ld instead of 0\n", result);
}
result=u_strCaseCompare(mixed, 4, different, 4, U_FOLD_CASE_DEFAULT, &errorCode);
if(result!=0) {
log_err("error: u_strCaseCompare(mixed, 4, different, 4, default)=%ld instead of 0\n", result);
}
/* test u_strncasecmp() - stop in the middle of the sharp s (U+00df) */
result=u_strncasecmp(mixed, different, 5, U_FOLD_CASE_DEFAULT);
if(result<=0) {
log_err("error: u_strncasecmp(mixed, different, 5, default)=%ld instead of positive\n", result);
}
result=u_strCaseCompare(mixed, 5, different, 5, U_FOLD_CASE_DEFAULT, &errorCode);
if(result<=0) {
log_err("error: u_strCaseCompare(mixed, 5, different, 5, default)=%ld instead of positive\n", result);
}
/* test u_memcasecmp() - stop before the sharp s (U+00df) */
result=u_memcasecmp(mixed, different, 4, U_FOLD_CASE_DEFAULT);
if(result!=0) {
log_err("error: u_memcasecmp(mixed, different, 4, default)=%ld instead of 0\n", result);
}
/* test u_memcasecmp() - stop in the middle of the sharp s (U+00df) */
result=u_memcasecmp(mixed, different, 5, U_FOLD_CASE_DEFAULT);
if(result<=0) {
log_err("error: u_memcasecmp(mixed, different, 5, default)=%ld instead of positive\n", result);
}
}
static void
TestCaseFolding(void) {
/*
* CaseFolding.txt says about i and its cousins:
* 0049; C; 0069; # LATIN CAPITAL LETTER I
* 0049; T; 0131; # LATIN CAPITAL LETTER I
*
* 0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
* 0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
* That's all.
* See CaseFolding.txt and the Unicode Standard for how to apply the case foldings.
*/
static const UChar32
simple[]={
/* input, default, exclude special i */
0x61, 0x61, 0x61,
0x49, 0x69, 0x131,
0x130, 0x130, 0x69,
0x131, 0x131, 0x131,
0xdf, 0xdf, 0xdf,
0xfb03, 0xfb03, 0xfb03,
0x1040e,0x10436,0x10436,
0x5ffff,0x5ffff,0x5ffff
};
static const UChar
mixed[]= { 0x61, 0x42, 0x130, 0x49, 0x131, 0x3d0, 0xdf, 0xfb03, 0xd93f, 0xdfff },
foldedDefault[]= { 0x61, 0x62, 0x69, 0x307, 0x69, 0x131, 0x3b2, 0x73, 0x73, 0x66, 0x66, 0x69, 0xd93f, 0xdfff },
foldedExcludeSpecialI[]={ 0x61, 0x62, 0x69, 0x131, 0x131, 0x3b2, 0x73, 0x73, 0x66, 0x66, 0x69, 0xd93f, 0xdfff };
UVersionInfo unicodeVersion={ 0, 0, 17, 89 }, unicode_3_1={ 3, 1, 0, 0 };
const UChar32 *p;
int32_t i;
UChar buffer[32];
int32_t length;
UErrorCode errorCode;
UBool isUnicode_3_1;
/* if unicodeVersion()>=3.1 then test exclude-special-i cases as well */
u_getUnicodeVersion(unicodeVersion);
isUnicode_3_1= uprv_memcmp(unicodeVersion, unicode_3_1, 4)>=0;
/* test simple case folding */
p=simple;
for(i=0; i<sizeof(simple)/12; p+=3, ++i) {
if(u_foldCase(p[0], U_FOLD_CASE_DEFAULT)!=p[1]) {
log_err("error: u_foldCase(0x%04lx, default)=0x%04lx instead of 0x%04lx\n",
p[0], u_foldCase(p[0], U_FOLD_CASE_DEFAULT), p[1]);
return;
}
if(isUnicode_3_1 && u_foldCase(p[0], U_FOLD_CASE_EXCLUDE_SPECIAL_I)!=p[2]) {
log_err("error: u_foldCase(0x%04lx, exclude special i)=0x%04lx instead of 0x%04lx\n",
p[0], u_foldCase(p[0], U_FOLD_CASE_EXCLUDE_SPECIAL_I), p[2]);
return;
}
}
/* test full string case folding with default option and separate buffers */
buffer[0]=0xabcd;
errorCode=U_ZERO_ERROR;
length=u_strFoldCase(buffer, sizeof(buffer)/U_SIZEOF_UCHAR,
mixed, sizeof(mixed)/U_SIZEOF_UCHAR,
U_FOLD_CASE_DEFAULT,
&errorCode);
if( U_FAILURE(errorCode) ||
length!=(sizeof(foldedDefault)/U_SIZEOF_UCHAR) ||
uprv_memcmp(foldedDefault, buffer, length*U_SIZEOF_UCHAR)!=0 ||
buffer[length]!=0
) {
log_err("error in u_strFoldCase(default)=%ld error=%s string matches: %s\n",
length,
u_errorName(errorCode),
uprv_memcmp(foldedDefault, buffer, length*U_SIZEOF_UCHAR)==0 && buffer[length]==0 ? "yes" : "no");
}
/* exclude special i */
if(isUnicode_3_1) {
buffer[0]=0xabcd;
errorCode=U_ZERO_ERROR;
length=u_strFoldCase(buffer, sizeof(buffer)/U_SIZEOF_UCHAR,
mixed, sizeof(mixed)/U_SIZEOF_UCHAR,
U_FOLD_CASE_EXCLUDE_SPECIAL_I,
&errorCode);
if( U_FAILURE(errorCode) ||
length!=(sizeof(foldedExcludeSpecialI)/U_SIZEOF_UCHAR) ||
uprv_memcmp(foldedExcludeSpecialI, buffer, length*U_SIZEOF_UCHAR)!=0 ||
buffer[length]!=0
) {
log_err("error in u_strFoldCase(exclude special i)=%ld error=%s string matches: %s\n",
length,
u_errorName(errorCode),
uprv_memcmp(foldedExcludeSpecialI, buffer, length*U_SIZEOF_UCHAR)==0 && buffer[length]==0 ? "yes" : "no");
}
}
/* test full string case folding with default option and in the same buffer */
uprv_memcpy(buffer, mixed, sizeof(mixed));
buffer[sizeof(mixed)/U_SIZEOF_UCHAR]=0;
errorCode=U_ZERO_ERROR;
length=u_strFoldCase(buffer, sizeof(buffer)/U_SIZEOF_UCHAR,
buffer, -1, /* implicit srcLength */
U_FOLD_CASE_DEFAULT,
&errorCode);
if( U_FAILURE(errorCode) ||
length!=(sizeof(foldedDefault)/U_SIZEOF_UCHAR) ||
uprv_memcmp(foldedDefault, buffer, length*U_SIZEOF_UCHAR)!=0 ||
buffer[length]!=0
) {
log_err("error in u_strFoldCase(default same buffer)=%ld error=%s string matches: %s\n",
length,
u_errorName(errorCode),
uprv_memcmp(foldedDefault, buffer, length*U_SIZEOF_UCHAR)==0 && buffer[length]==0 ? "yes" : "no");
}
/* test full string case folding, exclude special i, in the same buffer */
if(isUnicode_3_1) {
uprv_memcpy(buffer, mixed, sizeof(mixed));
errorCode=U_ZERO_ERROR;
length=u_strFoldCase(buffer, sizeof(buffer)/U_SIZEOF_UCHAR,
buffer, sizeof(mixed)/U_SIZEOF_UCHAR,
U_FOLD_CASE_EXCLUDE_SPECIAL_I,
&errorCode);
if( U_FAILURE(errorCode) ||
length!=(sizeof(foldedExcludeSpecialI)/U_SIZEOF_UCHAR) ||
uprv_memcmp(foldedExcludeSpecialI, buffer, length*U_SIZEOF_UCHAR)!=0 ||
buffer[length]!=0
) {
log_err("error in u_strFoldCase(exclude special i same buffer)=%ld error=%s string matches: %s\n",
length,
u_errorName(errorCode),
uprv_memcmp(foldedExcludeSpecialI, buffer, length*U_SIZEOF_UCHAR)==0 && buffer[length]==0 ? "yes" : "no");
}
}
/* test preflighting */
buffer[0]=buffer[2]=0xabcd;
errorCode=U_ZERO_ERROR;
length=u_strFoldCase(buffer, 2, /* set destCapacity=2 */
mixed, sizeof(mixed)/U_SIZEOF_UCHAR,
U_FOLD_CASE_DEFAULT,
&errorCode);
if( errorCode!=U_BUFFER_OVERFLOW_ERROR ||
length!=(sizeof(foldedDefault)/U_SIZEOF_UCHAR) ||
uprv_memcmp(foldedDefault, buffer, 2*U_SIZEOF_UCHAR)!=0 ||
buffer[2]!=0xabcd
) {
log_err("error in u_strFoldCase(default preflighting)=%ld error=%s string matches: %s\n",
length,
u_errorName(errorCode),
uprv_memcmp(foldedDefault, buffer, 2*U_SIZEOF_UCHAR)==0 && buffer[2]==0xabcd ? "yes" : "no");
}
errorCode=U_ZERO_ERROR;
length=u_strFoldCase(NULL, 0,
mixed, sizeof(mixed)/U_SIZEOF_UCHAR,
U_FOLD_CASE_DEFAULT,
&errorCode);
if( errorCode!=U_BUFFER_OVERFLOW_ERROR ||
length!=(sizeof(foldedDefault)/U_SIZEOF_UCHAR)
) {
log_err("error in u_strFoldCase(default pure preflighting)=%ld error=%s\n",
length,
u_errorName(errorCode));
}
/* test error handling */
errorCode=U_ZERO_ERROR;
length=u_strFoldCase(NULL, sizeof(buffer)/U_SIZEOF_UCHAR,
mixed, sizeof(mixed)/U_SIZEOF_UCHAR,
U_FOLD_CASE_DEFAULT,
&errorCode);
if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR) {
log_err("error in u_strFoldCase(default dest=NULL)=%ld error=%s\n",
length,
u_errorName(errorCode));
}
buffer[0]=0xabcd;
errorCode=U_ZERO_ERROR;
length=u_strFoldCase(buffer, -1,
mixed, sizeof(mixed)/U_SIZEOF_UCHAR,
U_FOLD_CASE_DEFAULT,
&errorCode);
if( errorCode!=U_ILLEGAL_ARGUMENT_ERROR ||
buffer[0]!=0xabcd
) {
log_err("error in u_strFoldCase(default destCapacity=-1)=%ld error=%s buffer[0]==0x%lx\n",
length,
u_errorName(errorCode),
buffer[0]);
}
buffer[0]=0xabcd;
errorCode=U_ZERO_ERROR;
length=u_strFoldCase(buffer, sizeof(buffer)/U_SIZEOF_UCHAR,
NULL, sizeof(mixed)/U_SIZEOF_UCHAR,
U_FOLD_CASE_EXCLUDE_SPECIAL_I,
&errorCode);
if( errorCode!=U_ILLEGAL_ARGUMENT_ERROR ||
buffer[0]!=0xabcd
) {
log_err("error in u_strFoldCase(exclude special i src=NULL)=%ld error=%s buffer[0]==0x%lx\n",
length,
u_errorName(errorCode),
buffer[0]);
}
buffer[0]=0xabcd;
errorCode=U_ZERO_ERROR;
length=u_strFoldCase(buffer, sizeof(buffer)/U_SIZEOF_UCHAR,
mixed, -2,
U_FOLD_CASE_EXCLUDE_SPECIAL_I,
&errorCode);
if( errorCode!=U_ILLEGAL_ARGUMENT_ERROR ||
buffer[0]!=0xabcd
) {
log_err("error in u_strFoldCase(exclude special i srcLength=-2)=%ld error=%s buffer[0]==0x%lx\n",
length,
u_errorName(errorCode),
buffer[0]);
}
}
static jstring ICU_getUnicodeVersion(JNIEnv* env, jclass) {
UVersionInfo unicodeVersion;
u_getUnicodeVersion(unicodeVersion);
return versionString(env, unicodeVersion);
}
U_CAPI UCollator* U_EXPORT2
ucol_openRules( const UChar *rules,
int32_t rulesLength,
UColAttributeValue normalizationMode,
UCollationStrength strength,
UParseError *parseError,
UErrorCode *status)
{
UColTokenParser src;
UColAttributeValue norm;
UParseError tErr;
if(status == NULL || U_FAILURE(*status)){
return 0;
}
u_init(status);
if (U_FAILURE(*status)) {
return NULL;
}
if(rules == NULL || rulesLength < -1) {
*status = U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
if(rulesLength == -1) {
rulesLength = u_strlen(rules);
}
if(parseError == NULL){
parseError = &tErr;
}
switch(normalizationMode) {
case UCOL_OFF:
case UCOL_ON:
case UCOL_DEFAULT:
norm = normalizationMode;
break;
default:
*status = U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
UCollator *UCA = ucol_initUCA(status);
if(U_FAILURE(*status)){
return NULL;
}
ucol_tok_initTokenList(&src, rules, rulesLength, UCA, status);
ucol_tok_assembleTokenList(&src,parseError, status);
if(U_FAILURE(*status)) {
/* if status is U_ILLEGAL_ARGUMENT_ERROR, src->current points at the offending option */
/* if status is U_INVALID_FORMAT_ERROR, src->current points after the problematic part of the rules */
/* so something might be done here... or on lower level */
#ifdef UCOL_DEBUG
if(*status == U_ILLEGAL_ARGUMENT_ERROR) {
fprintf(stderr, "bad option starting at offset %i\n", src.current-src.source);
} else {
fprintf(stderr, "invalid rule just before offset %i\n", src.current-src.source);
}
#endif
ucol_tok_closeTokenList(&src);
return NULL;
}
UCollator *result = NULL;
UCATableHeader *table = NULL;
if(src.resultLen > 0 || src.removeSet != NULL) { /* we have a set of rules, let's make something of it */
/* also, if we wanted to remove some contractions, we should make a tailoring */
table = ucol_assembleTailoringTable(&src, status);
if(U_SUCCESS(*status)) {
// builder version
table->version[0] = UCOL_BUILDER_VERSION;
// no tailoring information on this level
table->version[1] = table->version[2] = table->version[3] = 0;
// set UCD version
u_getUnicodeVersion(table->UCDVersion);
// set UCA version
uprv_memcpy(table->UCAVersion, UCA->image->UCAVersion, sizeof(UVersionInfo));
result = ucol_initCollator(table, 0, UCA, status);
result->hasRealData = TRUE;
result->freeImageOnClose = TRUE;
}
} else { /* no rules, but no error either */
// must be only options
// We will init the collator from UCA
result = ucol_initCollator(UCA->image, 0, UCA, status);
// And set only the options
UColOptionSet *opts = (UColOptionSet *)uprv_malloc(sizeof(UColOptionSet));
/* test for NULL */
if (opts == NULL) {
*status = U_MEMORY_ALLOCATION_ERROR;
goto cleanup;
}
uprv_memcpy(opts, src.opts, sizeof(UColOptionSet));
ucol_setOptionsFromHeader(result, opts, status);
result->freeOptionsOnClose = TRUE;
result->hasRealData = FALSE;
result->freeImageOnClose = FALSE;
}
if(U_SUCCESS(*status)) {
UChar *newRules;
result->dataVersion[0] = UCOL_BUILDER_VERSION;
if(rulesLength > 0) {
newRules = (UChar *)uprv_malloc((rulesLength+1)*U_SIZEOF_UCHAR);
/* test for NULL */
if (newRules == NULL) {
*status = U_MEMORY_ALLOCATION_ERROR;
goto cleanup;
}
uprv_memcpy(newRules, rules, rulesLength*U_SIZEOF_UCHAR);
newRules[rulesLength]=0;
result->rules = newRules;
result->rulesLength = rulesLength;
result->freeRulesOnClose = TRUE;
}
result->rb = NULL;
result->elements = NULL;
result->validLocale = NULL;
result->requestedLocale = NULL;
ucol_setAttribute(result, UCOL_STRENGTH, strength, status);
ucol_setAttribute(result, UCOL_NORMALIZATION_MODE, norm, status);
} else {
cleanup:
if(result != NULL) {
ucol_close(result);
} else {
if(table != NULL) {
uprv_free(table);
}
}
result = NULL;
}
ucol_tok_closeTokenList(&src);
return result;
}
extern "C" jstring Java_libcore_icu_ICU_getUnicodeVersion(JNIEnv* env, jclass) {
UVersionInfo unicodeVersion;
u_getUnicodeVersion(unicodeVersion);
return versionString(env, unicodeVersion);
}
static UBool U_CALLCONV
loadData(UStringPrepProfile* profile,
const char* path,
const char* name,
const char* type,
UErrorCode* errorCode) {
/* load Unicode SPREP data from file */
UTrie _sprepTrie={ 0,0,0,0,0,0,0 };
UDataMemory *dataMemory;
const int32_t *p=NULL;
const uint8_t *pb;
UVersionInfo normUnicodeVersion;
int32_t normUniVer, sprepUniVer, normCorrVer;
if(errorCode==NULL || U_FAILURE(*errorCode)) {
return 0;
}
/* open the data outside the mutex block */
//TODO: change the path
dataMemory=udata_openChoice(path, type, name, isSPrepAcceptable, NULL, errorCode);
if(U_FAILURE(*errorCode)) {
return FALSE;
}
p=(const int32_t *)udata_getMemory(dataMemory);
pb=(const uint8_t *)(p+_SPREP_INDEX_TOP);
utrie_unserialize(&_sprepTrie, pb, p[_SPREP_INDEX_TRIE_SIZE], errorCode);
_sprepTrie.getFoldingOffset=getSPrepFoldingOffset;
if(U_FAILURE(*errorCode)) {
udata_close(dataMemory);
return FALSE;
}
/* in the mutex block, set the data for this process */
umtx_lock(usprepMutex());
if(profile->sprepData==NULL) {
profile->sprepData=dataMemory;
dataMemory=NULL;
uprv_memcpy(&profile->indexes, p, sizeof(profile->indexes));
uprv_memcpy(&profile->sprepTrie, &_sprepTrie, sizeof(UTrie));
} else {
p=(const int32_t *)udata_getMemory(profile->sprepData);
}
umtx_unlock(usprepMutex());
/* initialize some variables */
profile->mappingData=(uint16_t *)((uint8_t *)(p+_SPREP_INDEX_TOP)+profile->indexes[_SPREP_INDEX_TRIE_SIZE]);
u_getUnicodeVersion(normUnicodeVersion);
normUniVer = (normUnicodeVersion[0] << 24) + (normUnicodeVersion[1] << 16) +
(normUnicodeVersion[2] << 8 ) + (normUnicodeVersion[3]);
sprepUniVer = (dataVersion[0] << 24) + (dataVersion[1] << 16) +
(dataVersion[2] << 8 ) + (dataVersion[3]);
normCorrVer = profile->indexes[_SPREP_NORM_CORRECTNS_LAST_UNI_VERSION];
if(U_FAILURE(*errorCode)){
udata_close(dataMemory);
return FALSE;
}
if( normUniVer < sprepUniVer && /* the Unicode version of SPREP file must be less than the Unicode Vesion of the normalization data */
normUniVer < normCorrVer && /* the Unicode version of the NormalizationCorrections.txt file should be less than the Unicode Vesion of the normalization data */
((profile->indexes[_SPREP_OPTIONS] & _SPREP_NORMALIZATION_ON) > 0) /* normalization turned on*/
){
*errorCode = U_INVALID_FORMAT_ERROR;
udata_close(dataMemory);
return FALSE;
}
profile->isDataLoaded = TRUE;
/* if a different thread set it first, then close the extra data */
if(dataMemory!=NULL) {
udata_close(dataMemory); /* NULL if it was set correctly */
}
return profile->isDataLoaded;
}
static void TestHeapFunctions() {
UErrorCode status = U_ZERO_ERROR;
UResourceBundle *rb = NULL;
char *icuDataDir;
UVersionInfo unicodeVersion = {0,0,0,0};
icuDataDir = safeGetICUDataDirectory(); /* save icu data dir, so we can put it back
* after doing u_cleanup(). */
/* Verify that ICU can be cleaned up and reinitialized successfully.
* Failure here usually means that some ICU service didn't clean up successfully,
* probably because some earlier test accidently left something open. */
ctest_resetICU();
/* Can not set memory functions if ICU is already initialized */
u_setMemoryFunctions(&gContext, myMemAlloc, myMemRealloc, myMemFree, &status);
TEST_STATUS(status, U_INVALID_STATE_ERROR);
/* Un-initialize ICU */
u_cleanup();
/* Can not set memory functions with NULL values */
status = U_ZERO_ERROR;
u_setMemoryFunctions(&gContext, NULL, myMemRealloc, myMemFree, &status);
TEST_STATUS(status, U_ILLEGAL_ARGUMENT_ERROR);
status = U_ZERO_ERROR;
u_setMemoryFunctions(&gContext, myMemAlloc, NULL, myMemFree, &status);
TEST_STATUS(status, U_ILLEGAL_ARGUMENT_ERROR);
status = U_ZERO_ERROR;
u_setMemoryFunctions(&gContext, myMemAlloc, myMemRealloc, NULL, &status);
TEST_STATUS(status, U_ILLEGAL_ARGUMENT_ERROR);
/* u_setMemoryFunctions() should work with null or non-null context pointer */
status = U_ZERO_ERROR;
u_setMemoryFunctions(NULL, myMemAlloc, myMemRealloc, myMemFree, &status);
TEST_STATUS(status, U_ZERO_ERROR);
u_setMemoryFunctions(&gContext, myMemAlloc, myMemRealloc, myMemFree, &status);
TEST_STATUS(status, U_ZERO_ERROR);
/* After reinitializing ICU, we should not be able to set the memory funcs again. */
status = U_ZERO_ERROR;
u_setDataDirectory(icuDataDir);
u_init(&status);
TEST_STATUS(status, U_ZERO_ERROR);
u_setMemoryFunctions(NULL, myMemAlloc, myMemRealloc, myMemFree, &status);
TEST_STATUS(status, U_INVALID_STATE_ERROR);
/* Doing ICU operations should cause allocations to come through our test heap */
gBlockCount = 0;
status = U_ZERO_ERROR;
rb = ures_open(NULL, "es", &status);
TEST_STATUS(status, U_ZERO_ERROR);
if (gBlockCount == 0) {
log_err("Heap functions are not being called from ICU.\n");
}
ures_close(rb);
/* Cleanup should put the heap back to its default implementation. */
ctest_resetICU();
u_getUnicodeVersion(unicodeVersion);
if (unicodeVersion[0] <= 0) {
log_err("Properties doesn't reinitialize without u_init.\n");
}
status = U_ZERO_ERROR;
u_init(&status);
TEST_STATUS(status, U_ZERO_ERROR);
/* ICU operations should no longer cause allocations to come through our test heap */
gBlockCount = 0;
status = U_ZERO_ERROR;
rb = ures_open(NULL, "fr", &status);
TEST_STATUS(status, U_ZERO_ERROR);
if (gBlockCount != 0) {
log_err("Heap functions did not reset after u_cleanup.\n");
}
ures_close(rb);
free(icuDataDir);
ctest_resetICU();
}
void cmd_version(UBool noLoad)
{
UVersionInfo icu;
char str[200];
printf("<ICUINFO>\n");
printf("International Components for Unicode for C/C++\n");
printf("%s\n", U_COPYRIGHT_STRING);
printf("Compiled-Version: %s\n", U_ICU_VERSION);
u_getVersion(icu);
u_versionToString(icu, str);
printf("Runtime-Version: %s\n", str);
printf("Compiled-Unicode-Version: %s\n", U_UNICODE_VERSION);
u_getUnicodeVersion(icu);
u_versionToString(icu, str);
printf("Runtime-Unicode-Version: %s\n", U_UNICODE_VERSION);
printf("Platform: %s\n", getPlatform());
#if defined(U_BUILD)
printf("Build: %s\n", U_BUILD);
#if defined(U_HOST)
if(strcmp(U_BUILD,U_HOST)) {
printf("Host: %s\n", U_HOST);
}
#endif
#endif
#if defined(U_CC)
printf("C compiler: %s\n", U_CC);
#endif
#if defined(U_CXX)
printf("C++ compiler: %s\n", U_CXX);
#endif
#if defined(CYGWINMSVC)
printf("Cygwin: CYGWINMSVC\n");
#endif
printf("ICUDATA: %s\n", U_ICUDATA_NAME);
do_init();
printf("Data Directory: %s\n", u_getDataDirectory());
printf("ICU Initialization returned: %s\n", u_errorName(initStatus));
printf( "Default locale: %s\n", uloc_getDefault());
{
UErrorCode subStatus = U_ZERO_ERROR;
ulocdata_getCLDRVersion(icu, &subStatus);
if(U_SUCCESS(subStatus)) {
u_versionToString(icu, str);
printf("CLDR-Version: %s\n", str);
} else {
printf("CLDR-Version: %s\n", u_errorName(subStatus));
}
}
#if !UCONFIG_NO_CONVERSION
if(noLoad == FALSE)
{
printf("Default converter: %s\n", ucnv_getDefaultName());
}
#endif
#if !UCONFIG_NO_FORMATTING
{
UChar buf[100];
char buf2[100];
UErrorCode subsubStatus= U_ZERO_ERROR;
int32_t len;
len = ucal_getDefaultTimeZone(buf, 100, &subsubStatus);
if(U_SUCCESS(subsubStatus)&&len>0) {
u_UCharsToChars(buf, buf2, len+1);
printf("Default TZ: %s\n", buf2);
} else {
printf("Default TZ: %s\n", u_errorName(subsubStatus));
}
}
{
UErrorCode subStatus = U_ZERO_ERROR;
const char *tzVer = ucal_getTZDataVersion(&subStatus);
if(U_FAILURE(subStatus)) {
tzVer = u_errorName(subStatus);
}
printf("TZ data version: %s\n", tzVer);
}
#endif
#if U_ENABLE_DYLOAD
const char *pluginFile = uplug_getPluginFile();
printf("Plugin file is: %s\n", (pluginFile&&*pluginFile)?pluginFile:"(not set. try setting ICU_PLUGINS to a directory.)");
#else
fprintf(stderr, "Dynamic Loading: is disabled. No plugins will be loaded at start-up.\n");
#endif
printf("</ICUINFO>\n\n");
}
void MultithreadTest::TestCollators()
{
UErrorCode status = U_ZERO_ERROR;
FILE *testFile = NULL;
char testDataPath[1024];
strcpy(testDataPath, IntlTest::getSourceTestData(status));
if (U_FAILURE(status)) {
errln("ERROR: could not open test data %s", u_errorName(status));
return;
}
strcat(testDataPath, "CollationTest_");
const char* type = "NON_IGNORABLE";
const char *ext = ".txt";
if(testFile) {
fclose(testFile);
}
char buffer[1024];
strcpy(buffer, testDataPath);
strcat(buffer, type);
size_t bufLen = strlen(buffer);
// we try to open 3 files:
// path/CollationTest_type.txt
// path/CollationTest_type_SHORT.txt
// path/CollationTest_type_STUB.txt
// we are going to test with the first one that we manage to open.
strcpy(buffer+bufLen, ext);
testFile = fopen(buffer, "rb");
if(testFile == 0) {
strcpy(buffer+bufLen, "_SHORT");
strcat(buffer, ext);
testFile = fopen(buffer, "rb");
if(testFile == 0) {
strcpy(buffer+bufLen, "_STUB");
strcat(buffer, ext);
testFile = fopen(buffer, "rb");
if (testFile == 0) {
*(buffer+bufLen) = 0;
dataerrln("could not open any of the conformance test files, tried opening base %s", buffer);
return;
} else {
infoln(
"INFO: Working with the stub file.\n"
"If you need the full conformance test, please\n"
"download the appropriate data files from:\n"
"http://source.icu-project.org/repos/icu/tools/trunk/unicodetools/com/ibm/text/data/");
}
}
}
LocalArray<Line> lines(new Line[200000]);
memset(lines.getAlias(), 0, sizeof(Line)*200000);
int32_t lineNum = 0;
UChar bufferU[1024];
uint32_t first = 0;
while (fgets(buffer, 1024, testFile) != NULL) {
if(*buffer == 0 || buffer[0] == '#') {
// Store empty and comment lines so that errors are reported
// for the real test file lines.
lines[lineNum].buflen = 0;
lines[lineNum].buff[0] = 0;
} else {
int32_t buflen = u_parseString(buffer, bufferU, 1024, &first, &status);
lines[lineNum].buflen = buflen;
u_memcpy(lines[lineNum].buff, bufferU, buflen);
lines[lineNum].buff[buflen] = 0;
}
lineNum++;
}
fclose(testFile);
if(U_FAILURE(status)) {
dataerrln("Couldn't read the test file!");
return;
}
UVersionInfo uniVersion;
static const UVersionInfo v62 = { 6, 2, 0, 0 };
u_getUnicodeVersion(uniVersion);
UBool isAtLeastUCA62 = uprv_memcmp(uniVersion, v62, 4) >= 0;
LocalPointer<Collator> coll(Collator::createInstance(Locale::getRoot(), status));
if(U_FAILURE(status)) {
errcheckln(status, "Couldn't open UCA collator");
return;
}
coll->setAttribute(UCOL_NORMALIZATION_MODE, UCOL_ON, status);
coll->setAttribute(UCOL_CASE_FIRST, UCOL_OFF, status);
coll->setAttribute(UCOL_CASE_LEVEL, UCOL_OFF, status);
coll->setAttribute(UCOL_STRENGTH, isAtLeastUCA62 ? UCOL_IDENTICAL : UCOL_TERTIARY, status);
coll->setAttribute(UCOL_ALTERNATE_HANDLING, UCOL_NON_IGNORABLE, status);
int32_t spawnResult = 0;
LocalArray<CollatorThreadTest> tests(new CollatorThreadTest[kCollatorThreadThreads]);
logln(UnicodeString("Spawning: ") + kCollatorThreadThreads + " threads * " + kFormatThreadIterations + " iterations each.");
int32_t j = 0;
for(j = 0; j < kCollatorThreadThreads; j++) {
//logln("Setting collator %i", j);
tests[j].setCollator(coll.getAlias(), lines.getAlias(), lineNum, isAtLeastUCA62);
}
for(j = 0; j < kCollatorThreadThreads; j++) {
log("%i ", j);
spawnResult = tests[j].start();
if(spawnResult != 0) {
errln("%s:%d THREAD INFO: thread %d failed to start with status %d", __FILE__, __LINE__, j, spawnResult);
return;
}
}
logln("Spawned all");
for(int32_t i=0;i<kCollatorThreadThreads;i++) {
tests[i].join();
//logln(UnicodeString("Test #") + i + " is complete.. ");
}
}
