OSMAND_CORE_API QString OSMAND_CORE_CALL OsmAnd::ICU::transliterateToLatin(
const QString& input,
const bool keepAccentsAndDiacriticsInInput /*= true*/,
const bool keepAccentsAndDiacriticsInOutput /*= true*/)
{
QString output;
UErrorCode icuError = U_ZERO_ERROR;
bool ok = true;
const auto pAnyToLatinTransliterator = g_pIcuAnyToLatinTransliterator->clone();
if (pAnyToLatinTransliterator == nullptr || U_FAILURE(icuError))
{
LogPrintf(LogSeverityLevel::Error, "ICU error: %d", icuError);
if (pAnyToLatinTransliterator != nullptr)
delete pAnyToLatinTransliterator;
return input;
}
// Transliterate from any to latin
UnicodeString icuString(reinterpret_cast<const UChar*>(input.unicode()), input.length());
pAnyToLatinTransliterator->transliterate(icuString);
output = qMove(QString(reinterpret_cast<const QChar*>(icuString.getBuffer()), icuString.length()));
// If input and output differ at this point or accents/diacritics should be converted,
// normalize the output again
if ((input.compare(output, Qt::CaseInsensitive) != 0 || !keepAccentsAndDiacriticsInInput) && !keepAccentsAndDiacriticsInOutput)
{
const auto pIcuAccentsAndDiacriticsConverter = g_pIcuAccentsAndDiacriticsConverter->clone();
ok = pIcuAccentsAndDiacriticsConverter != nullptr && U_SUCCESS(icuError);
if (ok)
{
pIcuAccentsAndDiacriticsConverter->transliterate(icuString);
output = qMove(QString(reinterpret_cast<const QChar*>(icuString.getBuffer()), icuString.length()));
}
if (pIcuAccentsAndDiacriticsConverter != nullptr)
delete pIcuAccentsAndDiacriticsConverter;
}
if (pAnyToLatinTransliterator != nullptr)
delete pAnyToLatinTransliterator;
if (!ok)
{
LogPrintf(LogSeverityLevel::Error, "ICU error: %d", icuError);
return input;
}
return output;
}
OSMAND_CORE_API QString OSMAND_CORE_CALL OsmAnd::ICU::transliterateToLatin(const QString& input)
{
QString output;
UErrorCode icuError = U_ZERO_ERROR;
bool ok = true;
const auto pTransliterator = g_pIcuTransliterator->clone();
if(pTransliterator == nullptr || !U_SUCCESS(icuError))
{
LogPrintf(LogSeverityLevel::Error, "ICU error: %d", icuError);
if(pTransliterator != nullptr)
delete pTransliterator;
return input;
}
UnicodeString icuString(reinterpret_cast<const UChar*>(input.unicode()), input.length());
pTransliterator->transliterate(icuString);
output = qMove(QString(reinterpret_cast<const QChar*>(icuString.getBuffer()), icuString.length()));
if(pTransliterator != nullptr)
delete pTransliterator;
if(!ok)
{
LogPrintf(LogSeverityLevel::Error, "ICU error: %d", icuError);
return input;
}
return output;
}
keyword_t get_keyword(const char *s, const struct locale *lang) {
keyword_t result = NOKEYWORD;
assert(lang);
assert(s);
while (*s == '@') ++s;
if (*s) {
char buffer[64];
char *str = transliterate(buffer, sizeof(buffer) - sizeof(int), s);
if (str) {
int i;
void *match;
void **tokens = get_translations(lang, UT_KEYWORDS);
critbit_tree *cb = (critbit_tree *)*tokens;
if (cb && cb_find_prefix(cb, str, strlen(str), &match, 1, 0)) {
cb_get_kv(match, &i, sizeof(int));
result = (keyword_t)i;
return keyword_disabled(result) ? NOKEYWORD : result;
}
}
}
return NOKEYWORD;
}
void add_latin_alternatives(string_tree_t *tree, char *str, size_t len, uint64_t options) {
char *transliterated = NULL;
char *utf8_normalized = NULL;
char *prev_string = NULL;
if (options & NORMALIZE_STRING_LATIN_ASCII) {
transliterated = transliterate(LATIN_ASCII, str, len);
if (transliterated != NULL) {
utf8_normalized = normalize_string_utf8(transliterated, options);
free(transliterated);
transliterated = NULL;
}
if (utf8_normalized != NULL) {
string_tree_add_string(tree, utf8_normalized);
prev_string = utf8_normalized;
utf8_normalized = NULL;
}
}
char *str_copy = strndup(str, len);
utf8_normalized = normalize_string_utf8(str_copy, options);
free(str_copy);
if (options & NORMALIZE_STRING_LATIN_ASCII && utf8_normalized != NULL) {
transliterated = transliterate(LATIN_ASCII, utf8_normalized, strlen(utf8_normalized));
free(utf8_normalized);
} else {
transliterated = utf8_normalized;
}
if (transliterated != NULL) {
if (prev_string == NULL || strcmp(prev_string, transliterated) != 0) {
string_tree_add_string(tree, transliterated);
}
free(transliterated);
transliterated = NULL;
}
if (prev_string != NULL) {
free(prev_string);
}
}
void chatRecv(char *ip, char *lang) {
int recvSock, recvChatLen;
int i;
unsigned int clientAddrSize;
char recvChat[CHATMAX], code[CODELENGTH+1];
struct sockaddr_in servAddr;
struct sockaddr_in clientAddr;
clientAddrSize = sizeof(clientAddr);
/* Create a datagram/UDP socket */
if((recvSock = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0)
die("socket() failed");
/* Construct the server address structure */
memset(&servAddr, 0, sizeof(servAddr)); /* Zero out structure */
servAddr.sin_family = AF_INET; /* Internet addr family */
servAddr.sin_addr.s_addr = htonl(INADDR_ANY); /* Server IP address */
servAddr.sin_port = htons(CLIENT_RECVPORT); /* Server port */
/* Bind to the local address */
if(bind(recvSock, (struct sockaddr *)&servAddr, sizeof(servAddr)) < 0)
die("bind() failed");
while(1) {
recvChatLen = recvfrom(recvSock, recvChat, CHATMAX, 0, (struct sockaddr *)&clientAddr, &clientAddrSize);
recvChat[recvChatLen] = '\0';
for(i=0;i<CODELENGTH;i++) {
code[i] = recvChat[i];
}
code[CODELENGTH] = '\0';
for(i=CODELENGTH;i<recvChatLen;i++) {
recvChat[i-CODELENGTH] = recvChat[i];
}
switch(atoi(code)%10000) {
case 2:
transliterate(recvChat,getLang(lang));
printf("%s\n",recvChat);
break;
case 3:
printf("%s\n",recvChat);
exit(EXIT_SUCCESS);
break;
case 4:
printf("%s\n",recvChat);
break;
case 5:
printf("Error: %s\n",recvChat);
exit(EXIT_SUCCESS);
break;
default:
break;
}
}
close(recvSock);
return;
}
char *normalize_string_latin(char *str, size_t len, uint64_t options) {
char *transliterated = transliterate(LATIN_ASCII, str, len);
char *utf8_normalized;
if (transliterated == NULL) {
utf8_normalized = normalize_string_utf8(str, options);
} else {
utf8_normalized = normalize_string_utf8(transliterated, options);
free(transliterated);
transliterated = NULL;
}
return utf8_normalized;
}
skill_t get_skill(const char *s, const struct locale * lang)
{
skill_t result = NOSKILL;
char buffer[64];
if (s) {
char * str = transliterate(buffer, sizeof(buffer) - sizeof(int), s);
if (str) {
int i;
const void * match;
void **tokens = get_translations(lang, UT_SKILLS);
struct critbit_tree *cb = (critbit_tree *)*tokens;
if (cb && cb_find_prefix(cb, str, strlen(str), &match, 1, 0)) {
cb_get_kv(match, &i, sizeof(int));
result = (skill_t)i;
}
}
else {
log_warning("could not transliterate skill: %s", s);
}
}
return result;
}
void Transliterator::transliterate(Replaceable& text) const {
transliterate(text, 0, text.length());
}
/* This function, given these parameters, will determine the meaning of
* the English translation, and then creates a Panini function to
* match the headword to this meaning.
*
* char * headword - we will try to interpret this English word to get
* the meaning.
* char * reading - The phonemic shape of the Japanese word.
* char * ttemp - A length of text (looking at the EDICT file, this
* may be any of the strings delimited by /.
* kanji * klist - This is the list of kanji that was generated by
* the readkanjidic file.
* char * postag - The part-of-speech tag (eg. (n), (v), (adj-na) ...)
* char * incode - This string contains the Panini code the interpret
* the string in ttemp.
* char * jpos - This is the name of the Japanese part of speech
* (eg. noun, verb etc.)
*/
int learnentry_func(char * headword, char * reading, char * ttemp, \
kanji * klist, char * postag, char * incode, char * jpos) {
if(!klist) {
printf("No klist!\n");
return 0;
}
/* Remove the part-of-speech tag at the front of the translation. */
int poslen = strlen(postag);
if(!strncmp(ttemp, postag, poslen)) ttemp += poslen + 1;
/* Remove the slash at the end of the translation */
char * translation = strdup(ttemp);
char * slash = strstr(translation, "/");
if(!slash) {
free(translation);
return 0;
}
slash[0] = ' ';
slash[1] = '\0';
if(!strlen(headword)) return 0;
monad_map(pmonad, unlink_the_dead, (void*)0, -1);
monad_map(pmonad, set_intext, (void *)translation, -1);
monad_map(pmonad, set_stack, incode, -1);
if(!monad_map(pmonad, tranny_parse, (void *)0, 20)) return 0;
monad_map(pmonad, remove_ns, "rection", -1);
monad_map(pmonad, remove_ns, "record", -1);
monad_map(pmonad, remove_ns, "theta", -1);
monad_map(pmonad, remove_ns, "clues", -1);
monad_map(pmonad, remove_ns, "record", -1);
/* Find the kanji in the list */
while(klist) {
if(!klist->glyph) return 0;
if(strcmp(klist->glyph, headword)) {
klist = klist->next;
//printf("\n;strcmp(\"%s\", \"%s\")\n", klist->glyph, headword);
continue;
} else {
klist->used = 1;
break;
}
}
/* No kanji found? Then just stop here. */
if(!klist) return 0;
printf("; %s: %s\n", headword, translation);
/* Define the Japanese word */
printf("(df %s ", jpos);
/* It should call the right kanji,*/
char * r = transliterate(reading);
printf("(segments %s-%s) ", klist->jiscode, r);
free(r);
/* It should have the right meaning */
monad_map(pmonad, kill_least_confident, (void *)0, -1);
monad_map(pmonad, print_seme, stdout, -1);
printf(")\n");
return 1;
}
