char *CollDataCache::getKey(UCollator *collator, char *keyBuffer, int32_t *keyBufferLength)
{
UErrorCode status = U_ZERO_ERROR;
int32_t len = ucol_getShortDefinitionString(collator, NULL, keyBuffer, *keyBufferLength, &status);
if (len >= *keyBufferLength) {
*keyBufferLength = (len + 2) & ~1; // round to even length, leaving room for terminating null
keyBuffer = NEW_ARRAY(char, *keyBufferLength);
status = U_ZERO_ERROR;
len = ucol_getShortDefinitionString(collator, NULL, keyBuffer, *keyBufferLength, &status);
}
extern "C" int register_localized_collators(sqlite3* handle, const char* systemLocale, int utf16Storage)
{
int err;
UErrorCode status = U_ZERO_ERROR;
void* icudata;
UCollator* collator = ucol_open(systemLocale, &status);
if (U_FAILURE(status)) {
return -1;
}
ucol_setAttribute(collator, UCOL_STRENGTH, UCOL_PRIMARY, &status);
if (U_FAILURE(status)) {
return -1;
}
status = U_ZERO_ERROR;
char buf[1024];
ucol_getShortDefinitionString(collator, NULL, buf, 1024, &status);
if (utf16Storage) {
err = sqlite3_create_collation_v2(handle, LOCALIZED_COLLATOR_NAME, SQLITE_UTF16, collator,
collate16, (void(*)(void*))localized_collator_dtor);
} else {
err = sqlite3_create_collation_v2(handle, LOCALIZED_COLLATOR_NAME, SQLITE_UTF8, collator,
collate8, (void(*)(void*))localized_collator_dtor);
}
if (err != SQLITE_OK) {
return err;
}
// Register the _TOKENIZE function
err = sqlite3_create_function(handle, "_TOKENIZE", 4, SQLITE_UTF16, collator, tokenize, NULL, NULL);
if (err != SQLITE_OK) {
return err;
}
err = sqlite3_create_function(handle, "_TOKENIZE", 5, SQLITE_UTF16, collator, tokenize, NULL, NULL);
if (err != SQLITE_OK) {
return err;
}
err = sqlite3_create_function(handle, "_TOKENIZE", 6, SQLITE_UTF16, collator, tokenize, NULL, NULL);
if (err != SQLITE_OK) {
return err;
}
//// PHONEBOOK_COLLATOR
status = U_ZERO_ERROR;
collator = ucol_open(systemLocale, &status);
if (U_FAILURE(status)) {
return -1;
}
status = U_ZERO_ERROR;
ucol_setAttribute(collator, UCOL_STRENGTH, UCOL_PRIMARY, &status);
if (U_FAILURE(status)) {
return -1;
}
status = U_ZERO_ERROR;
// ucol_getShortDefinitionString(collator, NULL, buf, 1024, &status);
if (utf16Storage) {
err = sqlite3_create_collation_v2(handle, PHONEBOOK_COLLATOR_NAME, SQLITE_UTF16, collator,
collate16, (void(*)(void*))localized_collator_dtor);
} else {
err = sqlite3_create_collation_v2(handle, PHONEBOOK_COLLATOR_NAME, SQLITE_UTF8, collator,
collate8, (void(*)(void*))localized_collator_dtor);
}
if (err != SQLITE_OK) {
return err;
}
//// PHONEBOOK_COLLATOR
return SQLITE_OK;
}
extern "C" int register_localized_collators(sqlite3* handle, const char* systemLocale, int utf16Storage)
{
int err;
UErrorCode status = U_ZERO_ERROR;
void* icudata;
UCollator* collator = ucol_open(systemLocale, &status);
if (U_FAILURE(status)) {
return -1;
}
ucol_setAttribute(collator, UCOL_STRENGTH, UCOL_PRIMARY, &status);
if (U_FAILURE(status)) {
return -1;
}
status = U_ZERO_ERROR;
char buf[1024];
ucol_getShortDefinitionString(collator, NULL, buf, 1024, &status);
if (utf16Storage) {
err = sqlite3_create_collation_v2(handle, LOCALIZED_COLLATOR_NAME, SQLITE_UTF16, collator,
collate16, (void(*)(void*))localized_collator_dtor);
} else {
err = sqlite3_create_collation_v2(handle, LOCALIZED_COLLATOR_NAME, SQLITE_UTF8, collator,
collate8, (void(*)(void*))localized_collator_dtor);
}
if (err != SQLITE_OK) {
return err;
}
// Register the _TOKENIZE function
err = sqlite3_create_function(handle, "_TOKENIZE", 4, SQLITE_UTF16, collator, tokenize, NULL, NULL);
if (err != SQLITE_OK) {
return err;
}
err = sqlite3_create_function(handle, "_TOKENIZE", 5, SQLITE_UTF16, collator, tokenize, NULL, NULL);
if (err != SQLITE_OK) {
return err;
}
err = sqlite3_create_function(handle, "_TOKENIZE", 6, SQLITE_UTF16, collator, tokenize, NULL, NULL);
if (err != SQLITE_OK) {
return err;
}
//// PHONEBOOK_COLLATOR
// The collator may be removed in the near future. Do not depend on it.
// TODO: it might be better to have another function for registering phonebook collator.
status = U_ZERO_ERROR;
if (strcmp(systemLocale, "ja") == 0 || strcmp(systemLocale, "ja_JP") == 0) {
collator = ucol_open("ja@collation=phonebook", &status);
} else {
collator = ucol_open(systemLocale, &status);
}
if (U_FAILURE(status)) {
return -1;
}
status = U_ZERO_ERROR;
ucol_setAttribute(collator, UCOL_STRENGTH, UCOL_PRIMARY, &status);
if (U_FAILURE(status)) {
return -1;
}
status = U_ZERO_ERROR;
// ucol_getShortDefinitionString(collator, NULL, buf, 1024, &status);
if (utf16Storage) {
err = sqlite3_create_collation_v2(handle, PHONEBOOK_COLLATOR_NAME, SQLITE_UTF16, collator,
collate16, (void(*)(void*))localized_collator_dtor);
} else {
err = sqlite3_create_collation_v2(handle, PHONEBOOK_COLLATOR_NAME, SQLITE_UTF8, collator,
collate8, (void(*)(void*))localized_collator_dtor);
}
if (err != SQLITE_OK) {
return err;
}
//// PHONEBOOK_COLLATOR
return SQLITE_OK;
}
int main(int /* argc*/ , const char * /*argv*/ []) {
UErrorCode status = U_ZERO_ERROR;
int diffs = 0;
int gbaddiffs =0;
setup(status);
if(U_FAILURE(status)) return 1;
int expected = PROVIDER_COUNT;
for(int l=0;l<LOCALE_COUNT;l++) {
printf("\n");
uint8_t oldBytes[200];
int32_t oldLen = -1;
for(int v=0;v<=expected;v++) {
// Construct the locale ID
char locID[200];
strcpy(locID, locale[l]);
if((v!=expected)) { // -1 = no version
strcat(locID, "@sp=icu");
strcat(locID, provider_version[v]);
}
printf("%-28s = ", locID);
UErrorCode subStatus = U_ZERO_ERROR;
uint8_t bytes[200];
uint8_t bytesb[200];
#define USE_CXX 0
#if USE_CXX
Collator *col = Collator::createInstance(Locale(locID),subStatus);
if(U_FAILURE(subStatus)) {
printf("ERR: %s\n", u_errorName(subStatus));
continue;
}
int32_t len = col->getSortKey(stuff, -1, bytes, 200);
#else
#if 1
char xbuf2[200];
strcpy(xbuf2,"X/");
strcat(xbuf2,locID);
strcat(xbuf2,"/");
//printf(" -> %s\n", xbuf2);
UCollator *col = ucol_openFromShortString(xbuf2, FALSE,NULL, &subStatus);
#else
UCollator *col = ucol_open(locID, &subStatus);
#endif
if(U_FAILURE(subStatus)) {
printf("ERR: %s\n", u_errorName(subStatus));
continue;
}
char xbuf3[200];
{
int32_t def = ucol_getShortDefinitionString(col,locID/*NULL*/,xbuf3,200,&subStatus);
if(U_FAILURE(subStatus)) {
printf("Err getting short string name: %s\n", u_errorName(subStatus));
} else {
printf(" --> %s\n", xbuf3);
}
}
int32_t len = ucol_getSortKey(col, stuff, -1, bytes, 200);
#endif
printf(" ");
int tdiffs=0;
for(int i=0;i<len;i++) {
if(i<oldLen&&bytes[i]!=oldBytes[i]) {
diffs++;
printf("*");
} else {
printf(" ");
}
printf("%02X", (0xFF&bytes[i]));
}
printf("\n");
char xbuf4[200];
UCollator *col2 = ucol_openFromShortString(xbuf3, FALSE, NULL, &subStatus);
if(U_FAILURE(subStatus)) {
printf("Err opening from new short string : %s\n", u_errorName(subStatus));
continue;
} else {
int32_t def4 = ucol_getShortDefinitionString(col,locID/*NULL*/,xbuf4,200,&subStatus);
if(strcmp(xbuf4,xbuf3)) {
printf(" --> reopened = %s (%s)\n", xbuf4, u_errorName(subStatus));
}
}
int32_t len2 = ucol_getSortKey(col2, stuff, -1, bytesb, 200);
int baddiffs=0;
for(int i=0;i<len;i++) {
if(i<len&&bytes[i]!=bytesb[i]) {
baddiffs++;
printf("!");
} else {
// printf(" ");
}
// printf("%02X", (0xFF&bytesb[i]));
}
if(baddiffs>0) {
printf(" - ERR! Diffs from %s in %d places\n", xbuf2,baddiffs);
gbaddiffs+=baddiffs;
} else {
//printf(" OK.\n");
}
// printf("\n");
#if USE_CXX
delete col;
#else
ucol_close(col);
#endif
oldLen = len;
memcpy(oldBytes, bytes, len);
}
}
if(diffs==0) {
#if (U_ICU_VERSION_MAJOR_NUM < 49)
printf("ERROR: 0 differences found between platforms. ICU " U_ICU_VERSION " does not support collator plugins properly (not until 49)\n");
#else
printf("ERROR: 0 differences found between platforms.. are the platforms installed? Try 'icuinfo -L'\n");
#endif
return 1;
} else {
printf("%d differences found among provider versions!\n", diffs);
}
if(gbaddiffs>0) {
printf("ERROR: %d diffs found between a collator and it's reopened (from shortstring) variant.\n", gbaddiffs);
return 2;
} else {
printf("Collator and reopened (shortstring) are OK.\n");
}
printf("Success!\n");
return 0;
}
