GKbool GKTextEncoding::Convert( GKbyte* target, GKint32& nTargetLen, const GKbyte* source, GKint32 nSourceLen )
{
GKASSERT(target != NULL && source != NULL);
UErrorCode ErrorCode = U_ZERO_ERROR;
m_dstConverter = ucnv_open(m_dstCharset.Cstr(), &ErrorCode);
if(U_FAILURE(ErrorCode)){
GKFPRINTF(stdout, "%s\n",u_errorName(ErrorCode));
ucnv_close(m_dstConverter);
return false;
}
m_srcConverter = ucnv_open(m_srcCharset.Cstr(), &ErrorCode);
if(U_FAILURE(ErrorCode)){
GKFPRINTF(stdout, "%s\n",u_errorName(ErrorCode));
ucnv_close(m_srcConverter);
return false;
}
ucnv_convert(m_dstCharset.Cstr(), m_srcCharset.Cstr(), (char*)target, nTargetLen, (char*)source, nSourceLen,&ErrorCode);
if(U_FAILURE(ErrorCode)){
GKFPRINTF(stdout, "%s\n",u_errorName(ErrorCode));
ucnv_close(m_dstConverter);
ucnv_close(m_srcConverter);
return false;
}
ucnv_close(m_dstConverter);
ucnv_close(m_srcConverter);
return true;
}
mod_websocket_conv_t *
mod_websocket_conv_init(const char *locale) {
mod_websocket_conv_t *cnv;
UErrorCode err = U_ZERO_ERROR;
if (!locale) {
return NULL;
}
cnv = (mod_websocket_conv_t *)malloc(sizeof(mod_websocket_conv_t));
if (!cnv) {
return NULL;
}
if (strcasecmp(MOD_WEBSOCKET_UTF8_STR, locale) == 0) {
cnv->cli = NULL;
cnv->srv = NULL;
} else {
cnv->cli = ucnv_open(MOD_WEBSOCKET_UTF8_STR, &err);
if (U_FAILURE(err)) {
free(cnv);
return NULL;
}
cnv->srv = ucnv_open(locale, &err);
if (U_FAILURE(err)) {
ucnv_close(cnv->cli);
free(cnv);
return NULL;
}
}
return cnv;
}
void
XeTeXFontMgr_FC::initialize()
{
if (FcInit() == FcFalse) {
fprintf(stderr, "fontconfig initialization failed!\n");
exit(9);
}
if (gFreeTypeLibrary == 0 && FT_Init_FreeType(&gFreeTypeLibrary) != 0) {
fprintf(stderr, "FreeType initialization failed!\n");
exit(9);
}
UErrorCode err = U_ZERO_ERROR;
macRomanConv = ucnv_open("macintosh", &err);
utf16beConv = ucnv_open("UTF16BE", &err);
utf8Conv = ucnv_open("UTF8", &err);
if (err != 0) {
fprintf(stderr, "internal error; cannot read font names\n");
exit(3);
}
FcPattern* pat = FcNameParse((const FcChar8*)":outline=true");
FcObjectSet* os = FcObjectSetBuild(FC_FAMILY, FC_STYLE, FC_FILE, FC_INDEX,
FC_FULLNAME, FC_WEIGHT, FC_WIDTH, FC_SLANT, NULL);
allFonts = FcFontList(FcConfigGetCurrent(), pat, os);
FcObjectSetDestroy(os);
FcPatternDestroy(pat);
cachedAll = false;
}
UConverter *QIcuCodec::getConverter(QTextCodec::ConverterState *state) const
{
UConverter *conv = 0;
if (state) {
if (!state->d) {
// first time
state->flags |= QTextCodec::FreeFunction;
QTextCodecUnalignedPointer::encode(state->state_data, qIcuCodecStateFree);
UErrorCode error = U_ZERO_ERROR;
state->d = ucnv_open(m_name, &error);
ucnv_setSubstChars(static_cast<UConverter *>(state->d),
state->flags & QTextCodec::ConvertInvalidToNull ? "\0" : "?", 1, &error);
if (U_FAILURE(error))
qDebug() << "getConverter(state) ucnv_open failed" << m_name << u_errorName(error);
}
conv = static_cast<UConverter *>(state->d);
}
if (!conv) {
// stateless conversion
UErrorCode error = U_ZERO_ERROR;
conv = ucnv_open(m_name, &error);
ucnv_setSubstChars(conv, "?", 1, &error);
if (U_FAILURE(error))
qDebug() << "getConverter(no state) ucnv_open failed" << m_name << u_errorName(error);
}
return conv;
}
int initTxt(struct doc_descriptor *desc) {
UErrorCode err;
char *encoding = NULL;
int len, BOMlength = 0;
char buf[BUFSIZE];
UChar outbuf[4*BUFSIZE];
lseek(desc->fd, 0, SEEK_SET);
len = read(desc->fd, buf, BUFSIZE);
/* detect BOM */
err = U_ZERO_ERROR;
encoding = ucnv_detectUnicodeSignature(buf, BUFSIZE, &BOMlength, &err);
if(encoding != NULL) {
lseek(desc->fd, BOMlength, SEEK_SET);
/* initialize converter to encoding */
err = U_ZERO_ERROR;
desc->conv = ucnv_open(encoding, &err);
if (U_FAILURE(err)) {
fprintf(stderr, "unable to open ICU converter\n");
return ERR_ICU;
}
} else {
/* initialize converter to UTF-8 */
err = U_ZERO_ERROR;
desc->conv = ucnv_open("utf8", &err);
if (U_FAILURE(err)) {
fprintf(stderr, "unable to open ICU converter\n");
return ERR_ICU;
}
/* check the first 2048 bytes */
err = U_ZERO_ERROR;
ucnv_setToUCallBack(desc->conv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &err);
if (U_FAILURE(err)) {
fprintf(stderr, "error setToUCallback\n");
return ERR_ICU;
}
err = U_ZERO_ERROR;
ucnv_toUChars(desc->conv, outbuf, 4 * BUFSIZE, buf, len, &err);
if (U_FAILURE(err)) {
fprintf(stderr, "Unknown encoding\n");
return ERR_ICU;
}
lseek(desc->fd, 0, SEEK_SET);
}
return OK;
}
charsetFilteredOutputStream_icu::charsetFilteredOutputStream_icu
(const charset& source, const charset& dest, outputStream* os,
const charsetConverterOptions& opts)
: m_from(NULL), m_to(NULL), m_sourceCharset(source),
m_destCharset(dest), m_stream(*os), m_options(opts)
{
UErrorCode err = U_ZERO_ERROR;
m_from = ucnv_open(source.getName().c_str(), &err);
if (!U_SUCCESS(err))
{
throw exceptions::charset_conv_error
("Cannot initialize ICU converter for source charset '" + source.getName() + "' (error code: " + u_errorName(err) + ".");
}
m_to = ucnv_open(dest.getName().c_str(), &err);
if (!U_SUCCESS(err))
{
throw exceptions::charset_conv_error
("Cannot initialize ICU converter for destination charset '" + dest.getName() + "' (error code: " + u_errorName(err) + ".");
}
// Tell ICU what to do when encountering an illegal byte sequence
if (m_options.silentlyReplaceInvalidSequences)
{
// Set replacement chars for when converting from Unicode to codepage
icu::UnicodeString substString(m_options.invalidSequence.c_str());
ucnv_setSubstString(m_to, substString.getTerminatedBuffer(), -1, &err);
if (U_FAILURE(err))
throw exceptions::charset_conv_error("[ICU] Error when setting substitution string.");
}
else
{
// Tell ICU top stop (and return an error) on illegal byte sequences
ucnv_setToUCallBack
(m_to, UCNV_TO_U_CALLBACK_STOP, UCNV_SUB_STOP_ON_ILLEGAL, NULL, NULL, &err);
if (U_FAILURE(err))
throw exceptions::charset_conv_error("[ICU] Error when setting ToU callback.");
ucnv_setFromUCallBack
(m_to, UCNV_FROM_U_CALLBACK_STOP, UCNV_SUB_STOP_ON_ILLEGAL, NULL, NULL, &err);
if (U_FAILURE(err))
throw exceptions::charset_conv_error("[ICU] Error when setting FromU callback.");
}
}
char *aescstrdup(const UChar* unichars,int32_t length){
char *newString,*targetLimit,*target;
UConverterFromUCallback cb;
const void *p;
UErrorCode errorCode = U_ZERO_ERROR;
#if U_CHARSET_FAMILY==U_EBCDIC_FAMILY
# if U_PLATFORM == U_PF_OS390
static const char convName[] = "ibm-1047";
# else
static const char convName[] = "ibm-37";
# endif
#else
static const char convName[] = "US-ASCII";
#endif
UConverter* conv = ucnv_open(convName, &errorCode);
if(length==-1){
length = u_strlen( unichars);
}
newString = (char*)ctst_malloc ( sizeof(char) * 8 * (length +1));
target = newString;
targetLimit = newString+sizeof(char) * 8 * (length +1);
ucnv_setFromUCallBack(conv, UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_C, &cb, &p, &errorCode);
ucnv_fromUnicode(conv,&target,targetLimit, &unichars, (UChar*)(unichars+length),NULL,TRUE,&errorCode);
ucnv_close(conv);
*target = '\0';
return newString;
}
void convsample_50() {
printf("\n\n==============================================\n"
"Sample 50: C: ucnv_detectUnicodeSignature\n");
//! [ucnv_detectUnicodeSignature]
UErrorCode err = U_ZERO_ERROR;
UBool discardSignature = TRUE; /* set to TRUE to throw away the initial U+FEFF */
char input[] = { '\xEF','\xBB', '\xBF','\x41','\x42','\x43' };
int32_t signatureLength = 0;
const char *encoding = ucnv_detectUnicodeSignature(input,sizeof(input),&signatureLength,&err);
UConverter *conv = NULL;
UChar output[100];
UChar *target = output, *out;
const char *source = input;
if(encoding!=NULL && U_SUCCESS(err)){
// should signature be discarded ?
conv = ucnv_open(encoding, &err);
// do the conversion
ucnv_toUnicode(conv,
&target, output + UPRV_LENGTHOF(output),
&source, input + sizeof(input),
NULL, TRUE, &err);
out = output;
if (discardSignature){
++out; // ignore initial U+FEFF
}
while(out != target) {
printf("%04x ", *out++);
}
puts("");
}
//! [ucnv_detectUnicodeSignature]
puts("");
}
bool CSICU_charset_init(charset* cs,
const ASCII* charSetName)
{
UErrorCode status = U_ZERO_ERROR;
UConverter* conv = ucnv_open(charSetName, &status);
if (U_SUCCESS(status))
{
// charSetName comes from stack. Copy it.
ASCII* p = new ASCII[strlen(charSetName) + 1];
cs->charset_name = p;
strcpy(p, charSetName);
cs->charset_version = CHARSET_VERSION_1;
cs->charset_flags |= CHARSET_ASCII_BASED;
cs->charset_min_bytes_per_char = ucnv_getMinCharSize(conv);
cs->charset_max_bytes_per_char = ucnv_getMaxCharSize(conv);
cs->charset_fn_destroy = charset_destroy;
cs->charset_fn_well_formed = NULL;
const UChar unicodeSpace = 32;
BYTE* p2 = new BYTE[cs->charset_max_bytes_per_char];
cs->charset_space_character = p2;
cs->charset_space_length = ucnv_fromUChars(conv, reinterpret_cast<char*>(p2),
cs->charset_max_bytes_per_char, &unicodeSpace, 1, &status);
fb_assert(U_SUCCESS(status));
ucnv_close(conv);
CVICU_convert_init(cs);
}
return U_SUCCESS(status);
}
U_CDECL_BEGIN
static void U_CALLCONV
_HZOpen(UConverter *cnv, UConverterLoadArgs *pArgs, UErrorCode *errorCode){
UConverter *gbConverter;
if(pArgs->onlyTestIsLoadable) {
ucnv_canCreateConverter("GBK", errorCode); /* errorCode carries result */
return;
}
gbConverter = ucnv_open("GBK", errorCode);
if(U_FAILURE(*errorCode)) {
return;
}
cnv->toUnicodeStatus = 0;
cnv->fromUnicodeStatus= 0;
cnv->mode=0;
cnv->fromUChar32=0x0000;
cnv->extraInfo = uprv_calloc(1, sizeof(UConverterDataHZ));
if(cnv->extraInfo != NULL){
((UConverterDataHZ*)cnv->extraInfo)->gbConverter = gbConverter;
}
else {
ucnv_close(gbConverter);
*errorCode = U_MEMORY_ALLOCATION_ERROR;
return;
}
}
void TextCodecICU::createICUConverter() const
{
ASSERT(!m_converterICU);
UErrorCode err;
m_needsGBKFallbacks = !strcmp(m_encodingName, "GBK");
UConverter*& cachedConverter = cachedConverterICU();
if (cachedConverter) {
err = U_ZERO_ERROR;
const char* cachedConverterName = ucnv_getName(cachedConverter, &err);
if (U_SUCCESS(err) && !strcmp(m_canonicalConverterName, cachedConverterName)) {
m_converterICU = cachedConverter;
cachedConverter = 0;
return;
}
}
err = U_ZERO_ERROR;
m_converterICU = ucnv_open(m_canonicalConverterName, &err);
ASSERT(U_SUCCESS(err));
if (m_converterICU)
ucnv_setFallback(m_converterICU, TRUE);
}
FStringConverter::FStringConverter()
: ICUConverter(nullptr)
{
UErrorCode ICUStatus = U_ZERO_ERROR;
ICUConverter = ucnv_open(FPlatformString::GetEncodingName(), &ICUStatus);
check(U_SUCCESS(ICUStatus));
}
char *
ppb_char_set_utf16_to_char_set(PP_Instance instance, const uint16_t *utf16, uint32_t utf16_len,
const char *output_char_set,
enum PP_CharSet_ConversionError on_error, uint32_t *output_length)
{
// each character could take up to 4 bytes in UTF-8; with additional zero-terminator byte
const uint32_t output_buffer_length = (utf16_len + 1) * 4 + 1;
char *output = ppb_memory_mem_alloc(output_buffer_length);
if (!output) {
trace_error("%s, can't allocate memory, %u bytes\n", __func__, output_buffer_length);
goto err;
}
const char *charset = encoding_alias_get_canonical_name(output_char_set);
const UChar subst = '?';
UErrorCode st = U_ZERO_ERROR;
UConverter *u = ucnv_open(charset, &st);
if (!U_SUCCESS(st)) {
trace_error("%s, wrong charset %s\n", __func__, output_char_set);
goto err;
}
switch (on_error) {
default:
case PP_CHARSET_CONVERSIONERROR_FAIL:
st = U_ZERO_ERROR;
ucnv_setFromUCallBack(u, UCNV_FROM_U_CALLBACK_STOP, NULL, NULL, NULL, &st);
break;
case PP_CHARSET_CONVERSIONERROR_SKIP:
st = U_ZERO_ERROR;
ucnv_setFromUCallBack(u, UCNV_FROM_U_CALLBACK_SKIP, NULL, NULL, NULL, &st);
break;
case PP_CHARSET_CONVERSIONERROR_SUBSTITUTE:
st = U_ZERO_ERROR;
ucnv_setFromUCallBack(u, UCNV_FROM_U_CALLBACK_SUBSTITUTE, NULL, NULL, NULL, &st);
st = U_ZERO_ERROR;
ucnv_setSubstString(u, &subst, 1, &st);
break;
}
*output_length = ucnv_fromUChars(u, output, output_buffer_length, utf16, utf16_len, &st);
if (st != U_BUFFER_OVERFLOW_ERROR && !U_SUCCESS(st))
goto err;
ucnv_close(u);
return output;
err:
*output_length = 0;
ppb_memory_mem_free(output);
if (u)
ucnv_close(u);
return NULL;
}
UErrorCode convsample_12()
{
printf("\n\n==============================================\n"
"Sample 12: C: simple sjis -> unicode conversion\n");
// **************************** START SAMPLE *******************
char source[] = { 0x63, 0x61, 0x74, (char)0x94, 0x4C, (char)0x82, 0x6E, (char)0x82, 0x6A, 0x00 };
UChar target[100];
UErrorCode status = U_ZERO_ERROR;
UConverter *conv;
int32_t len;
// set up the converter
conv = ucnv_open("shift_jis", &status);
assert(U_SUCCESS(status));
// convert to Unicode
// Note: we can use strlen, we know it's an 8 bit null terminated codepage
target[6] = 0xFDCA;
len = ucnv_toUChars(conv, target, 100, source, strlen(source), &status);
U_ASSERT(status);
// close the converter
ucnv_close(conv);
// ***************************** END SAMPLE ********************
// Print it out
printBytes("src", source, strlen(source) );
printf("\n");
printUChars("targ", target, len);
return U_ZERO_ERROR;
}
UConverter* ICUUnicodeSupport::_openConverter<1>(ConstStringHolder<1> _encoding)
{
UErrorCode errorCode = U_ZERO_ERROR;
UConverter* cnv = ucnv_open( char_cast<const char*>( _encoding.c_str() ), &errorCode);
CHECK_ICU_ERROR_CODE(errorCode);
return cnv;
}
//------------------------------------------
size_t CFS_Sqlite::WriteUTF8(const wchar* string, FSFILE *fp)
{
if (!string || !*string) return 0;
// if not unicode
#ifndef UNICODE
return Write(string, 1, strlen(string), fp);
#else
unsigned int inputLen = wstrlen(string);
unsigned char *target;
target = new unsigned char[inputLen*4+1]; // make buffer long enough
UErrorCode status = U_ZERO_ERROR;
int32_t len;
//// set up the converter
UConverter *conv = ucnv_open("utf-8", &status);
assertd(U_SUCCESS(status)!=0, "Failed to open utf-8 converter!");
//// convert
len = ucnv_fromUChars(conv, (char*)target, inputLen*4, string, -1, &status);
assertd(U_SUCCESS(status)!=0, "Failed to convert string to utf-8!");
//// write to file
size_t wrlen = Write(target, 1, len, fp);
//// close and clean
delete[] target;
ucnv_close(conv);
return wrlen;
#endif
}
extern int
main(int argc, const char *argv[]) {
UErrorCode errorCode=U_ZERO_ERROR;
// Note: Using a global variable for any object is not exactly thread-safe...
// You can change this call to e.g. ucnv_open("UTF-8", &errorCode) if you pipe
// the output to a file and look at it with a Unicode-capable editor.
// This will currently affect only the printUString() function, see the code above.
// printUnicodeString() could use this, too, by changing to an extract() overload
// that takes a UConverter argument.
cnv=ucnv_open(NULL, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "error %s opening the default converter\n", u_errorName(errorCode));
return errorCode;
}
ucnv_setFromUCallBack(cnv, UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_C, NULL, NULL, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "error %s setting the escape callback in the default converter\n", u_errorName(errorCode));
ucnv_close(cnv);
return errorCode;
}
demo_utf_h_macros();
demo_C_Unicode_strings();
demoCaseMapInC();
demoCaseMapInCPlusPlus();
demoUnicodeStringStorage();
demoUnicodeStringInit();
ucnv_close(cnv);
return 0;
}
static UConverter *
GSStringOpenConverter (CFStringEncoding encoding, char lossByte)
{
const char *converterName;
UConverter *cnv;
UErrorCode err = U_ZERO_ERROR;
converterName = CFStringICUConverterName (encoding);
cnv = ucnv_open (converterName, &err);
if (U_FAILURE (err))
cnv = NULL;
if (lossByte)
{
/* FIXME: for some reason this is returning U_ILLEGAL_ARGUMENTS_ERROR */
ucnv_setSubstChars (cnv, &lossByte, 1, &err);
}
else
{
ucnv_setToUCallBack (cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL,
&err);
ucnv_setFromUCallBack (cnv, UCNV_FROM_U_CALLBACK_STOP, NULL, NULL, NULL,
&err);
}
return cnv;
}
UErrorCode convsample_02()
{
printf("\n\n==============================================\n"
"Sample 02: C: simple Unicode -> koi8-r conversion\n");
// **************************** START SAMPLE *******************
// "cat<cat>OK"
UChar source[] = { 0x041C, 0x043E, 0x0441, 0x043A, 0x0432,
0x0430, 0x0021, 0x0000 };
char target[100];
UErrorCode status = U_ZERO_ERROR;
UConverter *conv;
int32_t len;
// set up the converter
conv = ucnv_open("koi8-r", &status);
assert(U_SUCCESS(status));
// convert to koi8-r
len = ucnv_fromUChars(conv, target, 100, source, -1, &status);
assert(U_SUCCESS(status));
// close the converter
ucnv_close(conv);
// ***************************** END SAMPLE ********************
// Print it out
printUChars("src", source);
printf("\n");
printBytes("targ", target, len);
return U_ZERO_ERROR;
}
/* params : desc : the document descriptor
* return : an error code
*
* initializes the plugin by creating an xmlreader and
* storing it in desc
* This function also positions the meatreader at the beginning
* of the metadata
*/
int initPlugin(struct doc_descriptor *desc) {
UErrorCode err;
desc->myState = (struct ParserState *) malloc(sizeof(struct ParserState));
desc->fd = open(desc->filename, O_RDONLY);
desc->parser = XML_ParserCreate(NULL);
XML_SetUserData(desc->parser, desc->myState);
XML_SetElementHandler(desc->parser, startElement, endElement);
XML_SetCharacterDataHandler(desc->parser, characters);
((struct ParserState *)(desc->myState))->isTextContent = 0;
((struct ParserState *)(desc->myState))->suspended = 0;
((struct ParserState *)(desc->myState))->meta_suspended = 0;
((struct ParserState *)(desc->myState))->meta = NULL;
((struct ParserState *)(desc->myState))->pparser = &(desc->parser);
((struct ParserState *)(desc->myState))->size_adjusted = 0;
/* initialize converter ( content is utf8 ) */
err = U_ZERO_ERROR;
desc->conv = ucnv_open("utf8", &err);
if (U_FAILURE(err)) {
free(desc->myState);
desc->myState = NULL;
XML_ParserFree(desc->parser);
desc->parser = NULL;
close(desc->fd);
fprintf(stderr, "unable to open ICU converter\n");
return ERR_ICU;
}
((struct ParserState *)(desc->myState))->cnv = desc->conv;
return OK;
}
Bool
CodeSet_IsEncodingSupported(const char *name) // IN
{
UConverter *cv;
UErrorCode uerr;
/*
* Fallback if necessary.
*/
if (dontUseIcu) {
return CodeSetOld_IsEncodingSupported(name);
}
/*
* Try to open the encoding.
*/
uerr = U_ZERO_ERROR;
cv = ucnv_open(name, &uerr);
if (cv) {
ucnv_close(cv);
return TRUE;
}
return FALSE;
}
U_CAPI UConverter* U_EXPORT2
u_getDefaultConverter(UErrorCode *status)
{
UConverter *converter = NULL;
if (gDefaultConverter != NULL) {
umtx_lock(NULL);
/* need to check to make sure it wasn't taken out from under us */
if (gDefaultConverter != NULL) {
converter = gDefaultConverter;
gDefaultConverter = NULL;
}
umtx_unlock(NULL);
}
/* if the cache was empty, create a converter */
if(converter == NULL) {
converter = ucnv_open(NULL, status);
if(U_FAILURE(*status)) {
ucnv_close(converter);
converter = NULL;
}
}
return converter;
}
/*Extracts the UChar* to a char* and calls through createConverter */
UConverter* ucnv_openU (const UChar * name,
UErrorCode * err)
{
char asciiName[MAX_CONVERTER_NAME_LENGTH];
if (U_FAILURE (*err))
return NULL;
if (name == NULL)
return ucnv_open (NULL, err);
if (u_strlen (name) > MAX_CONVERTER_NAME_LENGTH)
{
*err = U_ILLEGAL_ARGUMENT_ERROR;
return NULL;
}
return ucnv_open (u_austrcpy (asciiName, name), err);
}
void TextCodecICU::createICUConverter() const
{
ASSERT(!m_converterICU);
#if defined(USING_SYSTEM_ICU)
const char* name = m_encoding.name();
m_needsGBKFallbacks = name[0] == 'G' && name[1] == 'B' && name[2] == 'K' && !name[3];
#endif
UErrorCode err;
UConverter*& cachedConverter = cachedConverterICU();
if (cachedConverter) {
err = U_ZERO_ERROR;
const char* cachedName = ucnv_getName(cachedConverter, &err);
if (U_SUCCESS(err) && m_encoding == cachedName) {
m_converterICU = cachedConverter;
cachedConverter = 0;
return;
}
}
err = U_ZERO_ERROR;
m_converterICU = ucnv_open(m_encoding.name(), &err);
#if !LOG_DISABLED
if (err == U_AMBIGUOUS_ALIAS_WARNING)
WTF_LOG_ERROR("ICU ambiguous alias warning for encoding: %s", m_encoding.name());
#endif
if (m_converterICU)
ucnv_setFallback(m_converterICU, TRUE);
}
//------------------------------------------------------------------------------------------
//
// printMatch Called when a matching line has been located.
// Print out the line from the file with the match, after
// converting it back to the default code page.
//
//------------------------------------------------------------------------------------------
void printMatch() {
char buf[2000];
UErrorCode status = U_ZERO_ERROR;
// If we haven't already created a converter for output, do it now.
if (outConverter == 0) {
outConverter = ucnv_open(NULL, &status);
if (U_FAILURE(status)) {
fprintf(stderr, "ugrep: Error opening default converter: \"%s\"\n",
u_errorName(status));
exit(-1);
}
};
// Convert the line to be printed back to the default 8 bit code page.
// If the line is too long for our buffer, just truncate it.
ucnv_fromUChars(outConverter,
buf, // destination buffer for conversion
sizeof(buf), // capacity of destination buffer
&ucharBuf[lineStart], // Input to conversion
lineEnd-lineStart, // number of UChars to convert
&status);
buf[sizeof(buf)-1] = 0; // Add null for use in case of too long lines.
// The converter null-terminates its output unless
// the buffer completely fills.
if (displayFileName) {
printf("%s:", fileName);
}
if (displayLineNum) {
printf("%d:", lineNum);
}
printf("%s", buf);
}
CF_INLINE UConverter *__CFStringEncodingConverterCreateICUConverter(const char *icuName, uint32_t flags, bool toUnicode) {
UConverter *converter;
UErrorCode errorCode = U_ZERO_ERROR;
uint8_t streamID = CFStringEncodingStreamIDFromMask(flags);
if (0 != streamID) { // this is a part of streaming previously created
__CFICUThreadData *data = __CFStringEncodingICUGetThreadData();
--streamID; // map to array index
if ((streamID < data->_numSlots) && (NULL != data->_converters[streamID])) return data->_converters[streamID];
}
converter = ucnv_open(icuName, &errorCode);
if (NULL != converter) {
char lossyByte = CFStringEncodingMaskToLossyByte(flags);
if ((0 == lossyByte) && (0 != (flags & kCFStringEncodingAllowLossyConversion))) lossyByte = '?';
if (0 ==lossyByte) {
if (toUnicode) {
ucnv_setToUCallBack(converter, &UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
} else {
ucnv_setFromUCallBack(converter, &UCNV_FROM_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
}
} else {
ucnv_setSubstChars(converter, &lossyByte, 1, &errorCode);
}
}
return converter;
}
U_STABLE UConverter* U_EXPORT2
ucnv_open_emoji(const char *converterName, UErrorCode *err) {
if (EmojiFont::IsAvailable()) {
if (strcmp(converterName, "Shift_JIS") == 0)
converterName = "docomo-emoji";
}
return ucnv_open(converterName, err);
}
std::unique_ptr<UConverter, UConverterDeleter> make_uconv(const std::string& encoding) {
icu::ErrorCode ec{};
UConverter* conv_ptr = ucnv_open(encoding.c_str(), ec);
if (!ec.isSuccess()) throw IcuUtilsException(TRACEMSG(std::string() +
"Error creating converter for encoding: [" + encoding + "],"
" error: [" + ec.errorName() + "]"));
return std::unique_ptr<UConverter, UConverterDeleter>{conv_ptr, UConverterDeleter{}};
}
static UConverter *my_ucnv_open(const char *cnv, UErrorCode *err)
{
if(cnv && cnv[0] == '@') {
return ucnv_openPackage("testdata", cnv+1, err);
} else {
return ucnv_open(cnv, err);
}
}
transcoder::transcoder (std::string const& encoding)
: ok_(false),
conv_(0)
{
UErrorCode err = U_ZERO_ERROR;
conv_ = ucnv_open(encoding.c_str(),&err);
if (U_SUCCESS(err)) ok_ = true;
// TODO ??
}
