void TextCodecICU::createICUConverter() const
{
ASSERT(!m_converterICU);
UErrorCode err;
m_needsGBKFallbacks = !strcmp(m_encodingName, "GBK");
UConverter*& cachedConverter = cachedConverterICU();
if (cachedConverter) {
err = U_ZERO_ERROR;
const char* cachedConverterName = ucnv_getName(cachedConverter, &err);
if (U_SUCCESS(err) && !strcmp(m_canonicalConverterName, cachedConverterName)) {
m_converterICU = cachedConverter;
cachedConverter = 0;
return;
}
}
err = U_ZERO_ERROR;
m_converterICU = ucnv_open(m_canonicalConverterName, &err);
ASSERT(U_SUCCESS(err));
if (m_converterICU)
ucnv_setFallback(m_converterICU, TRUE);
}
void TextCodecICU::createICUConverter() const
{
ASSERT(!m_converterICU);
#if defined(USING_SYSTEM_ICU)
const char* name = m_encoding.name();
m_needsGBKFallbacks = name[0] == 'G' && name[1] == 'B' && name[2] == 'K' && !name[3];
#endif
UErrorCode err;
UConverter*& cachedConverter = cachedConverterICU();
if (cachedConverter) {
err = U_ZERO_ERROR;
const char* cachedName = ucnv_getName(cachedConverter, &err);
if (U_SUCCESS(err) && m_encoding == cachedName) {
m_converterICU = cachedConverter;
cachedConverter = 0;
return;
}
}
err = U_ZERO_ERROR;
m_converterICU = ucnv_open(m_encoding.name(), &err);
#if !LOG_DISABLED
if (err == U_AMBIGUOUS_ALIAS_WARNING)
WTF_LOG_ERROR("ICU ambiguous alias warning for encoding: %s", m_encoding.name());
#endif
if (m_converterICU)
ucnv_setFallback(m_converterICU, TRUE);
}
int gtm_conv(UConverter* from, UConverter* to, mstr *src, char* dstbuff, int* bufflen)
{
char *dstptr, *dstbase, *srcptr;
const char *ichset;
int dstlen, src_charlen, srclen;
UErrorCode status, status1;
if (0 == src->len)
return 0;
if (NULL == dstbuff)
{
/* Compute the stringpool buffer space needed for conversion given that source
* is encoded in the ichset representation. The ICU functions ucnv_getMinCharSize()
* and ucnv_getMaxCharSize() are used to compute the minimum and maximum number of
* bytes required per UChar if converted from/to ichset/ochset respectively
*/
src_charlen = (src->len / ucnv_getMinCharSize(from)) + 1; /* number of UChar's from ichset */
dstlen = UCNV_GET_MAX_BYTES_FOR_STRING(src_charlen, ucnv_getMaxCharSize(to));
dstlen = (dstlen > MAX_STRLEN) ? MAX_STRLEN : dstlen;
ENSURE_STP_FREE_SPACE(dstlen);
dstbase = (char *)stringpool.free;
} else
{
dstbase = dstbuff;
dstlen = *bufflen;
}
srcptr = src->addr;
srclen = (int)src->len;
dstptr = dstbase;
status = U_ZERO_ERROR; /* initialization to "success" is required by ICU */
ucnv_convertEx(to, from, &dstptr, dstptr + dstlen, (const char**)&srcptr, srcptr + srclen,
NULL, NULL, NULL, NULL, TRUE, TRUE, &status);
if (U_FAILURE(status))
{
if (U_BUFFER_OVERFLOW_ERROR == status)
{ /* translation requires more space than the maximum allowed GT.M string size */
if (NULL == dstbuff)
rts_error_csa(NULL, VARLSTCNT(1) ERR_MAXSTRLEN);
else
{
/* Insufficient buffer passed. Return the required buffer length */
src_charlen = (srclen / ucnv_getMinCharSize(from)) + 1;
*bufflen = UCNV_GET_MAX_BYTES_FOR_STRING(src_charlen, ucnv_getMaxCharSize(to));
return -1;
}
}
status1 = U_ZERO_ERROR;
ichset = ucnv_getName(from, &status1);
assert(U_SUCCESS(status1));
UTF8_BADCHAR(1,(unsigned char *) (srcptr - 1), NULL,STRLEN(ichset), ichset);
}
return (int) (dstptr - dstbase);
}
U_CAPI const char* U_EXPORT2 /* U_CAPI ... U_EXPORT2 added by Peter Kirk 17 Nov 2001 */
u_fgetcodepage(UFILE *file)
{
UErrorCode status = U_ZERO_ERROR;
const char *codepage = NULL;
if (file->fConverter) {
codepage = ucnv_getName(file->fConverter, &status);
if(U_FAILURE(status))
return 0;
}
return codepage;
}
int
main(int argc,
char **argv)
{
UConverter *c;
UErrorCode status = U_ZERO_ERROR;
udata_setCommonData(NULL, &status);
printf("setCommonData(NULL) -> %s [should fail]\n", u_errorName(status));
if(status != U_ILLEGAL_ARGUMENT_ERROR)
{
printf("*** FAIL: should have returned U_ILLEGAL_ARGUMENT_ERROR\n");
return 1;
}
status = U_ZERO_ERROR;
udata_setCommonData(&U_ICUDATA_ENTRY_POINT, &status);
printf("setCommonData(%p) -> %s\n", (void*)&U_ICUDATA_ENTRY_POINT, u_errorName(status));
if(U_FAILURE(status))
{
printf("*** FAIL: should have returned U_ZERO_ERROR\n");
return 1;
}
status = U_ZERO_ERROR;
c = ucnv_open("iso-8859-3", &status);
printf("ucnv_open(iso-8859-3)-> %p, err = %s, name=%s\n",
(void *)c, u_errorName(status), (!c)?"?":ucnv_getName(c,&status) );
if(status != U_ZERO_ERROR)
{
printf("\n*** FAIL: should have returned U_ZERO_ERROR;\n");
return 1;
}
else
{
ucnv_close(c);
}
status = U_ZERO_ERROR;
udata_setCommonData(&U_ICUDATA_ENTRY_POINT, &status);
printf("setCommonData(%p) -> %s [should pass]\n", (void*) &U_ICUDATA_ENTRY_POINT, u_errorName(status));
if (U_FAILURE(status) || status == U_USING_DEFAULT_WARNING )
{
printf("\n*** FAIL: should pass and not set U_USING_DEFAULT_ERROR\n");
return 1;
}
printf("\n*** PASS PASS PASS, test PASSED!!!!!!!!\n");
return 0;
}
/* dump the contents of a converter */
static void UCNV_DEBUG_CNV(const UConverter *c, int line)
{
UErrorCode err = U_ZERO_ERROR;
fprintf(stderr, "%p\t:%d\t", c, line);
if(c!=NULL) {
const char *name = ucnv_getName(c, &err);
if (!name) {
name = "(null)";
}
fprintf(stderr, "%s\t", name);
fprintf(stderr, "shr=%p, ref=%x\n",
c->sharedData,
c->sharedData->referenceCounter);
} else {
fprintf(stderr, "DEMISED\n");
}
}
/**
* Sets current (default) ICU charset
*
* If given charset is unavailable, an error is raised
*
* @param enc new charset (single string)
* @return nothing (\code{R_NilValue})
*
* @version 0.1-?? (Marek Gagolewski)
*
* @version 0.2-1 (Marek Gagolewski)
* use StriUcnv; make StriException-friendly
*
* @version 0.3-1 (Marek Gagolewski, 2014-11-04)
* Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
*/
SEXP stri_enc_set(SEXP enc)
{
// here, the default encoding may not be requested:
const char* selected_enc
= stri__prepare_arg_enc(enc, "enc", false/*no default*/); /* this is R_alloc'ed */
STRI__ERROR_HANDLER_BEGIN(0)
StriUcnv uconv_obj(selected_enc);
// this will generate an error if selected_enc is not supported:
UConverter* uconv = uconv_obj.getConverter();
UErrorCode status = U_ZERO_ERROR;
// get "official" encoding name:
const char* name = ucnv_getName(uconv, &status);
STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
ucnv_setDefaultName(name); // set as default
return R_NilValue;
STRI__ERROR_HANDLER_END({/* no special action on error */})
}
/**
* Convert character vector between given encodings
*
* @param str input character vector or list of raw vectors
* @param from source encoding, \code{NULL} or \code{""} for default enc
* @param to target encoding, \code{NULL} or \code{""} for default enc
* @param to_raw single logical, should list of raw vectors be returned?
* @return a converted character vector or list of raw vectors
*
* @version 0.1 (Marek Gagolewski)
* @version 0.2 (Marek Gagolewski) arg to_raw_added, encoding marking
* @version 0.3 (Marek Gagolewski, 2013-06-16) make StriException-friendly
* @version 0.4 (Marek Gagolewski, 2013-08-08) use StriContainerListRaw
* @version 0.5 (Marek Gagolewski, 2013-11-20) BUGFIX call stri_encode_from_marked if necessary
*/
SEXP stri_encode(SEXP str, SEXP from, SEXP to, SEXP to_raw)
{
const char* selected_from = stri__prepare_arg_enc(from, "from", true);
if (!selected_from && Rf_isVectorAtomic(str))
return stri_encode_from_marked(str, to, to_raw);
str = stri_prepare_arg_list_raw(str, "str");
const char* selected_to = stri__prepare_arg_enc(to, "to", true);
bool to_raw_logical = stri__prepare_arg_logical_1_notNA(to_raw, "to_raw");
UConverter* uconv_from = NULL;
UConverter* uconv_to = NULL;
STRI__ERROR_HANDLER_BEGIN
StriContainerListRaw str_cont(str);
R_len_t str_n = str_cont.get_n();
// get the number of strings to convert; if == 0, then you know what's the result
if (str_n <= 0) return Rf_allocVector(to_raw_logical?VECSXP:STRSXP, 0);
// Open converters
uconv_from = stri__ucnv_open(selected_from);
uconv_to = stri__ucnv_open(selected_to);
// Get target encoding mark
UErrorCode err = U_ZERO_ERROR;
const char* uconv_to_name = ucnv_getName(uconv_to, &err);
if (U_FAILURE(err))
throw StriException(err);
cetype_t encmark_to = CE_BYTES; // all other cases than the below ones
// - bytes enc (this is reasonable, isn't it?)
if (!to_raw_logical) { // otherwise not needed
if (!strcmp(uconv_to_name, "US-ASCII") || !strcmp(uconv_to_name, "UTF-8"))
encmark_to = CE_UTF8; // no CE for ASCII, will be auto-detected by mkCharLenCE
else if (!strcmp(uconv_to_name, "ISO-8859-1"))
encmark_to = CE_LATIN1;
else if (!strcmp(uconv_to_name, ucnv_getDefaultName()))
encmark_to = CE_NATIVE;
}
// Prepare out val
SEXP ret;
PROTECT(ret = Rf_allocVector(to_raw_logical?VECSXP:STRSXP, str_n));
String8 buf(0); // will be extended in a moment
for (R_len_t i=0; i<str_n; ++i) {
if (str_cont.isNA(i)) {
if (to_raw_logical) SET_VECTOR_ELT(ret, i, R_NilValue);
else SET_STRING_ELT(ret, i, NA_STRING);
continue;
}
const char* curd = str_cont.get(i).c_str();
R_len_t curn = str_cont.get(i).length();
err = U_ZERO_ERROR;
UnicodeString encs(curd, curn, uconv_from, err); // FROM -> UTF-16 [this is the slow part]
if (U_FAILURE(err))
throw StriException(err);
R_len_t curn_tmp = encs.length();
const UChar* curs_tmp = encs.getBuffer(); // The buffer contents is (probably) not NUL-terminated.
if (!curs_tmp)
throw StriException(MSG__INTERNAL_ERROR);
R_len_t bufneed = UCNV_GET_MAX_BYTES_FOR_STRING(curn_tmp, ucnv_getMaxCharSize(uconv_to));
// "The calculated size is guaranteed to be sufficient for this conversion."
buf.resize(bufneed);
err = U_ZERO_ERROR;
// bufneed = encs.extract(buf.data(), buf.size(), uconv_to, err); // UTF-16 -> TO
ucnv_resetFromUnicode(uconv_to);
bufneed = ucnv_fromUChars(uconv_to, buf.data(), buf.size(), curs_tmp, curn_tmp, &err);
if (bufneed <= buf.size()) {
if (U_FAILURE(err))
throw StriException(err);
}
else {// larger buffer needed [this shouldn't happen?]
// warning("buf extending");
buf.resize(bufneed);
err = U_ZERO_ERROR;
bufneed = ucnv_fromUChars(uconv_to, buf.data(), buf.size(), curs_tmp, curn_tmp, &err);
if (U_FAILURE(err))
throw StriException(err);
if (bufneed > buf.size())
throw StriException(MSG__INTERNAL_ERROR);
}
if (to_raw_logical) {
SEXP outobj = Rf_allocVector(RAWSXP, bufneed);
memcpy(RAW(outobj), buf.data(), (size_t)bufneed);
SET_VECTOR_ELT(ret, i, outobj);
}
else {
SET_STRING_ELT(ret, i, Rf_mkCharLenCE(buf.data(), bufneed, encmark_to));
}
}
if (uconv_from) {
ucnv_close(uconv_from);
uconv_from = NULL;
}
if (uconv_to) {
ucnv_close(uconv_to);
uconv_to = NULL;
}
UNPROTECT(1);
return ret;
STRI__ERROR_HANDLER_END({
if (uconv_from)
ucnv_close(uconv_from);
if (uconv_to)
ucnv_close(uconv_to);
})
}
/** Fetch information on an encoding
*
* @param enc either NULL or "" for default encoding,
* or one string with encoding name
* @return R list object with many components (see R doc for details)
*
* @version 0.1-?? (Marek Gagolewski)
*
* @version 0.2-1 (Marek Gagolewski)
* use StriUcnv; make StriException-friendly
*
* @version 0.3-1 (Marek Gagolewski, 2014-11-04)
* Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
*/
SEXP stri_enc_info(SEXP enc)
{
const char* selected_enc = stri__prepare_arg_enc(enc, "enc", true/*default ok*/); /* this is R_alloc'ed */
STRI__ERROR_HANDLER_BEGIN(0)
StriUcnv uconv_obj(selected_enc);
//uconv_obj.setCallBackSubstitute(); // restore default callbacks (no warning)
UConverter* uconv = uconv_obj.getConverter(false);
UErrorCode status = U_ZERO_ERROR;
// get the list of available standards
vector<const char*> standards = StriUcnv::getStandards();
R_len_t standards_n = (R_len_t)standards.size();
// alloc output list
SEXP vals;
SEXP names;
const int nval = standards_n+2+5;
STRI__PROTECT(names = Rf_allocVector(STRSXP, nval));
SET_STRING_ELT(names, 0, Rf_mkChar("Name.friendly"));
SET_STRING_ELT(names, 1, Rf_mkChar("Name.ICU"));
for (R_len_t i=0; i<standards_n; ++i) {
if (standards[i])
SET_STRING_ELT(names, i+2, Rf_mkChar((string("Name.")+standards[i]).c_str()));
}
SET_STRING_ELT(names, nval-5, Rf_mkChar("ASCII.subset"));
SET_STRING_ELT(names, nval-4, Rf_mkChar("Unicode.1to1"));
SET_STRING_ELT(names, nval-3, Rf_mkChar("CharSize.8bit"));
SET_STRING_ELT(names, nval-2, Rf_mkChar("CharSize.min"));
SET_STRING_ELT(names, nval-1, Rf_mkChar("CharSize.max"));
STRI__PROTECT(vals = Rf_allocVector(VECSXP, nval));
// get canonical (ICU) name
status = U_ZERO_ERROR;
const char* canname = ucnv_getName(uconv, &status);
if (U_FAILURE(status) || !canname) {
SET_VECTOR_ELT(vals, 1, Rf_ScalarString(NA_STRING));
Rf_warning(MSG__ENC_ERROR_GETNAME);
}
else {
SET_VECTOR_ELT(vals, 1, stri__make_character_vector_char_ptr(1, canname));
// friendly name
const char* frname = StriUcnv::getFriendlyName(canname);
if (frname) SET_VECTOR_ELT(vals, 0, stri__make_character_vector_char_ptr(1, frname));
else SET_VECTOR_ELT(vals, 0, Rf_ScalarString(NA_STRING));
// has ASCII as its subset?
SET_VECTOR_ELT(vals, nval-5, Rf_ScalarLogical((int)uconv_obj.hasASCIIsubset()));
// min,max character size, is 8bit?
int mincharsize = (int)ucnv_getMinCharSize(uconv);
int maxcharsize = (int)ucnv_getMaxCharSize(uconv);
int is8bit = (mincharsize==1 && maxcharsize == 1);
SET_VECTOR_ELT(vals, nval-3, Rf_ScalarLogical(is8bit));
SET_VECTOR_ELT(vals, nval-2, Rf_ScalarInteger(mincharsize));
SET_VECTOR_ELT(vals, nval-1, Rf_ScalarInteger(maxcharsize));
// is there a one-to-one correspondence with Unicode?
if (!is8bit)
SET_VECTOR_ELT(vals, nval-4, Rf_ScalarLogical(NA_LOGICAL));
else
SET_VECTOR_ELT(vals, nval-4, Rf_ScalarLogical((int)uconv_obj.is1to1Unicode()));
// other standard names
for (R_len_t i=0; i<standards_n; ++i) {
if (!standards[i]) continue;
status = U_ZERO_ERROR;
const char* stdname = ucnv_getStandardName(canname, standards[i], &status);
if (U_FAILURE(status) || !stdname)
SET_VECTOR_ELT(vals, i+2, Rf_ScalarString(NA_STRING));
else
SET_VECTOR_ELT(vals, i+2, stri__make_character_vector_char_ptr(1, stdname));
}
}
Rf_setAttrib(vals, R_NamesSymbol, names);
STRI__UNPROTECT_ALL
return vals;
STRI__ERROR_HANDLER_END({/* no special action on error */})
}
/**
* Count the number of characters in a string
*
* Note that ICU permits only strings of length < 2^31.
* @param s R character vector
* @return integer vector
* @version 0.1 (Marcin Bujarski)
* @version 0.2 (Marek Gagolewski) Multiple input encoding support
* @version 0.3 (Marek Gagolewski, 2013-06-16) make StriException-friendly
*/
SEXP stri_length(SEXP str)
{
str = stri_prepare_arg_string(str, "str");
R_len_t ns = LENGTH(str);
SEXP ret;
UConverter* uconv = NULL;
bool uconv_8bit = false;
bool uconv_utf8 = false;
STRI__ERROR_HANDLER_BEGIN
/* Note: ICU50 permits only int-size strings in U8_NEXT and U8_FWD_1 */
#define STRI_LENGTH_CALCULATE_UTF8 \
const char* qc = CHAR(q); \
R_len_t j = 0; \
for (R_len_t i = 0; i < nq; j++) \
U8_FWD_1(qc, i, nq); \
retint[k] = j;
PROTECT(ret = Rf_allocVector(INTSXP, ns));
int* retint = INTEGER(ret);
for (R_len_t k = 0; k < ns; k++) {
SEXP q = STRING_ELT(str, k);
if (q == NA_STRING)
retint[k] = NA_INTEGER;
else {
R_len_t nq = LENGTH(q); // O(1) - stored by R
// We trust (is that a wise assumption?)
// R encoding marks; However, it there is no mark,
// the string may have any encoding (ascii, latin1, utf8, native)
if (IS_ASCII(q) || IS_LATIN1(q))
retint[k] = nq;
else if (IS_BYTES(q))
throw StriException(MSG__BYTESENC);
else if (IS_UTF8(q)) {
STRI_LENGTH_CALCULATE_UTF8
}
else { // Any encoding - detection needed
// UTF-8 strings can be fairly reliably recognized as such by a
// simple algorithm, i.e., the probability that a string of
// characters in any other encoding appears as valid UTF-8 is low,
// diminishing with increasing string length.
// We have two possibilities here:
// 1. Auto detect encoding: Is this ASCII or UTF-8? If not => use Native
// This won't work correctly in some cases.
// e.g. (c4,85) represents ("Polish a with ogonek") in UTF-8
// and ("A umlaut", "Ellipsis") in WINDOWS-1250
// 2. Assume it's Native; this assumes the user working in an 8-bit environment
// would convert strings to UTF-8 manually if needed - I think is's
// a more reasonable approach (Native --> input via keyboard)
if (!uconv) { // open ucnv on demand
uconv = stri__ucnv_open((const char*)NULL); // native decoder
if (!uconv) {
retint[k] = NA_INTEGER;
continue;
}
uconv_8bit = ((int)ucnv_getMaxCharSize(uconv) == 1);
if (!uconv_8bit) {
UErrorCode err = U_ZERO_ERROR;
const char* name = ucnv_getName(uconv, &err);
if (U_FAILURE(err))
throw StriException("could not query default converter");
uconv_utf8 = !strncmp("UTF-8", name, 5);
}
}
if (uconv_8bit) {
retint[k] = nq; // it's an 8-bit encoding :-)
}
else if (uconv_utf8) { // it's UTF-8
STRI_LENGTH_CALCULATE_UTF8
}
else { // native encoding which is neither 8-bit, nor UTF-8 (e.g. 'Big5')
UErrorCode err = U_ZERO_ERROR;
const char* source = CHAR(q);
const char* sourceLimit = source + nq;
R_len_t j;
for (j = 0; source != sourceLimit; j++) {
if (U_FAILURE(err)) break; // error from previous iteration
// iterate through each native-encoded character:
ucnv_getNextUChar(uconv, &source, sourceLimit, &err);
}
if (U_FAILURE(err)) { // error from last iteration
Rf_warning("error determining length for native, neither 8-bit- nor UTF-8-encoded string.");
retint[k] = NA_INTEGER;
}
else
retint[k] = j; // all right, we got it!
}
}
}
bool filter(const char_type*& src_begin, const char_type* src_end,
char_type*& dest_begin, char_type* dest_end,
bool flush)
{
if (! ucnv_from && ! ucnv_to) {
// no converter... simply copy!
const size_t copy_size = std::min(src_end - src_begin, dest_end - dest_begin);
std::copy(src_begin, src_begin + copy_size, dest_begin);
src_begin += copy_size;
dest_begin += copy_size;
return false;
}
UChar* pivot_end = pivot_start + boost::iostreams::default_device_buffer_size;
UErrorCode status = U_ZERO_ERROR;
if (pivot_target != pivot_end) {
const char_type* src_begin_prev = src_begin;
UChar* pivot_target_prev = pivot_target;
status = U_ZERO_ERROR;
ucnv_toUnicode(ucnv_from, &pivot_target, pivot_end, &src_begin, src_end, 0, flush, &status);
offset_from += src_begin - src_begin_prev;
offset_pivot_target += pivot_target - pivot_target_prev;
if (status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(status)) {
UErrorCode status_getname = U_ZERO_ERROR;
const char* encoding = ucnv_getName(ucnv_from, &status_getname);
std::ostringstream offset_stream;
offset_stream << offset_from;
std::ostringstream offset_stream_unicode;
offset_stream_unicode << offset_pivot_target;
message_from = (std::string("ucnv_toUnicode(): ") + u_errorName(status)
+ " from " + encoding
+ " offset: " + offset_stream.str()
+ " unicode offset: " + offset_stream_unicode.str());
throw BOOST_IOSTREAMS_FAILURE(message_from);
}
}
char_type* dest_begin_prev = dest_begin;
const UChar* pivot_source_prev = pivot_source;
status = U_ZERO_ERROR;
ucnv_fromUnicode(ucnv_to, &dest_begin, dest_end, &pivot_source, pivot_target, 0, flush, &status);
offset_to += dest_begin - dest_begin_prev;
offset_pivot_source += pivot_source - pivot_source_prev;
if (status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(status)) {
UErrorCode status_getname = U_ZERO_ERROR;
const char* encoding = ucnv_getName(ucnv_to, &status_getname);
std::ostringstream offset_stream;
offset_stream << offset_to;
std::ostringstream offset_stream_unicode;
offset_stream_unicode << offset_pivot_source;
message_to = (std::string("ucnv_fromUnicode(): ") + u_errorName(status)
+ " to " + encoding
+ " offset: " + offset_stream.str()
+ " unicode offset: " + offset_stream_unicode.str());
throw BOOST_IOSTREAMS_FAILURE(message_to);
}
if (pivot_source == pivot_target) {
pivot_source = pivot_start;
pivot_target = pivot_start;
}
return status == U_BUFFER_OVERFLOW_ERROR;
}
const char *__hs_ucnv_getName(const UConverter *converter, UErrorCode *err)
{
return ucnv_getName(converter, err);
}
EXPORT UnicodeString &PyBytes_AsUnicodeString(PyObject *object,
const char *encoding,
const char *mode,
UnicodeString &string)
{
UErrorCode status = U_ZERO_ERROR;
UConverter *conv = ucnv_open(encoding, &status);
if (U_FAILURE(status))
throw ICUException(status);
_STOPReason stop;
char *src;
Py_ssize_t len;
UChar *buffer, *target;
memset(&stop, 0, sizeof(stop));
if (!strcmp(mode, "strict"))
{
ucnv_setToUCallBack(conv, _stopDecode, &stop, NULL, NULL, &status);
if (U_FAILURE(status))
{
ucnv_close(conv);
throw ICUException(status);
}
}
PyBytes_AsStringAndSize(object, &src, &len);
stop.src = src;
stop.src_length = len;
buffer = target = new UChar[len];
if (buffer == NULL)
{
ucnv_close(conv);
PyErr_NoMemory();
throw ICUException();
}
ucnv_toUnicode(conv, &target, target + len,
(const char **) &src, src + len, NULL, true, &status);
if (U_FAILURE(status))
{
const char *reasonName;
switch (stop.reason) {
case UCNV_UNASSIGNED:
reasonName = "the code point is unassigned";
break;
case UCNV_ILLEGAL:
reasonName = "the code point is illegal";
break;
case UCNV_IRREGULAR:
reasonName = "the code point is not a regular sequence in the encoding";
break;
default:
reasonName = "unexpected reason code";
break;
}
status = U_ZERO_ERROR;
PyErr_Format(PyExc_ValueError, "'%s' codec can't decode byte 0x%x in position %d: reason code %d (%s)", ucnv_getName(conv, &status), (int) (unsigned char) stop.chars[0], stop.error_position, stop.reason, reasonName);
delete[] buffer;
ucnv_close(conv);
throw ICUException();
}
string.setTo(buffer, target - buffer);
delete[] buffer;
ucnv_close(conv);
return string;
}
