void UnicodeTestCase::ConversionUTF8()
{
static const StringConversionData utf8data[] =
{
#ifdef wxHAVE_U_ESCAPE
StringConversionData("\xc2\xa3", L"\u00a3"),
#endif
StringConversionData("\xc2", NULL),
};
wxCSConv conv(wxT("utf-8"));
for ( size_t n = 0; n < WXSIZEOF(utf8data); n++ )
{
const StringConversionData& d = utf8data[n];
d.Test(n, conv);
d.Test(n, wxConvUTF8);
}
static const char* const u25a6 = "\xe2\x96\xa6";
wxMBConvUTF8 c(wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL);
CPPUNIT_ASSERT_EQUAL( 2, c.ToWChar(NULL, 0, u25a6, wxNO_LEN) );
CPPUNIT_ASSERT_EQUAL( 0, c.ToWChar(NULL, 0, u25a6, 0) );
CPPUNIT_ASSERT_EQUAL( 1, c.ToWChar(NULL, 0, u25a6, 3) );
CPPUNIT_ASSERT_EQUAL( 2, c.ToWChar(NULL, 0, u25a6, 4) );
// Verify that converting a string with embedded NULs works.
CPPUNIT_ASSERT_EQUAL( 5, wxString::FromUTF8("abc\0\x32", 5).length() );
// Verify that converting a string containing invalid UTF-8 does not work,
// even if it happens after an embedded NUL.
CPPUNIT_ASSERT( wxString::FromUTF8("abc\xff").empty() );
CPPUNIT_ASSERT( wxString::FromUTF8("abc\0\xff", 5).empty() );
}
void UnicodeTestCase::ConversionUTF8()
{
static const StringConversionData utf8data[] =
{
#ifdef wxHAVE_U_ESCAPE
StringConversionData("\xc2\xa3", L"\u00a3"),
#endif
StringConversionData("\xc2", NULL),
};
wxCSConv conv(wxT("utf-8"));
for ( size_t n = 0; n < WXSIZEOF(utf8data); n++ )
{
const StringConversionData& d = utf8data[n];
d.Test(n, conv);
d.Test(n, wxConvUTF8);
}
static const char* const u25a6 = "\xe2\x96\xa6";
wxMBConvUTF8 c(wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL);
CPPUNIT_ASSERT_EQUAL( 2, c.ToWChar(NULL, 0, u25a6, wxNO_LEN) );
CPPUNIT_ASSERT_EQUAL( 0, c.ToWChar(NULL, 0, u25a6, 0) );
CPPUNIT_ASSERT_EQUAL( 1, c.ToWChar(NULL, 0, u25a6, 3) );
CPPUNIT_ASSERT_EQUAL( 2, c.ToWChar(NULL, 0, u25a6, 4) );
}
void UnicodeTestCase::ConversionUTF16()
{
static const StringConversionData utf16data[] =
{
#ifdef wxHAVE_U_ESCAPE
StringConversionData(
"\x04\x1f\x04\x40\x04\x38\x04\x32\x04\x35\x04\x42\0\0",
L"\u041f\u0440\u0438\u0432\u0435\u0442"),
StringConversionData(
"\x01\0\0b\x01\0\0a\x01\0\0r\0\0",
L"\u0100b\u0100a\u0100r"),
#endif
StringConversionData("\0f\0o\0o\0\0", L"foo"),
};
wxCSConv conv(wxFONTENCODING_UTF16BE);
for ( size_t n = 0; n < WXSIZEOF(utf16data); n++ )
{
const StringConversionData& d = utf16data[n];
d.Test(n, conv);
}
// special case: this string has consecutive NULs inside it which don't
// terminate the string, this exposed a bug in our conversion code which
// got confused in this case
size_t len;
conv.cMB2WC("\x01\0\0B\0C" /* A macron BC */, 6, &len);
CPPUNIT_ASSERT_EQUAL( 3, len );
// Another one: verify that the length of the resulting string is computed
// correctly when there is a surrogate in the input.
wxMBConvUTF16BE().cMB2WC("\xd8\x03\xdc\x01\0" /* OLD TURKIC LETTER YENISEI A */, wxNO_LEN, &len);
CPPUNIT_ASSERT_EQUAL( 1, len );
}
void UnicodeTestCase::ConversionUTF16()
{
static const StringConversionData utf16data[] =
{
#ifdef wxHAVE_U_ESCAPE
StringConversionData(
"\x04\x1f\x04\x40\x04\x38\x04\x32\x04\x35\x04\x42\0\0",
L"\u041f\u0440\u0438\u0432\u0435\u0442"),
StringConversionData(
"\x01\0\0b\x01\0\0a\x01\0\0r\0\0",
L"\u0100b\u0100a\u0100r"),
#endif
StringConversionData("\0f\0o\0o\0\0", L"foo"),
};
wxCSConv conv(wxFONTENCODING_UTF16BE);
for ( size_t n = 0; n < WXSIZEOF(utf16data); n++ )
{
const StringConversionData& d = utf16data[n];
d.Test(n, conv);
}
// special case: this string has consecutive NULs inside it which don't
// terminate the string, this exposed a bug in our conversion code which
// got confused in this case
size_t len;
conv.cMB2WC("\x01\0\0B\0C" /* A macron BC */, 6, &len);
CPPUNIT_ASSERT_EQUAL( 3, len );
// When using UTF-16 internally (i.e. MSW), we don't have any surrogate
// support, so the length of the string below is 2, not 1.
#if SIZEOF_WCHAR_T == 4
// Another one: verify that the length of the resulting string is computed
// correctly when there is a surrogate in the input.
wxMBConvUTF16BE().cMB2WC("\xd8\x03\xdc\x01\0" /* OLD TURKIC LETTER YENISEI A */, wxNO_LEN, &len);
CPPUNIT_ASSERT_EQUAL( 1, len );
#endif // UTF-32 internal representation
#if SIZEOF_WCHAR_T == 2
// Verify that the length of UTF-32 string is correct even when converting
// to it from a longer UTF-16 string with surrogates.
// Construct CAT FACE U+1F431 without using \U which is not supported by
// ancient compilers and without using \u with surrogates which is
// (correctly) flagged as an error by the newer ones.
wchar_t ws[2];
ws[0] = 0xd83d;
ws[1] = 0xdc31;
CPPUNIT_ASSERT_EQUAL( 4, wxMBConvUTF32BE().FromWChar(NULL, 0, ws, 2) );
#endif // UTF-16 internal representation
}
void UnicodeTestCase::ConversionUTF32()
{
static const StringConversionData utf32data[] =
{
#ifdef wxHAVE_U_ESCAPE
StringConversionData(
"\0\0\x04\x1f\0\0\x04\x40\0\0\x04\x38\0\0\x04\x32\0\0\x04\x35\0\0\x04\x42\0\0\0\0",
L"\u041f\u0440\u0438\u0432\u0435\u0442"),
#endif
StringConversionData("\0\0\0f\0\0\0o\0\0\0o\0\0\0\0", L"foo"),
};
wxCSConv conv(wxFONTENCODING_UTF32BE);
for ( size_t n = 0; n < WXSIZEOF(utf32data); n++ )
{
const StringConversionData& d = utf32data[n];
d.Test(n, conv);
}
size_t len;
conv.cMB2WC("\0\0\x01\0\0\0\0B\0\0\0C" /* A macron BC */, 12, &len);
CPPUNIT_ASSERT_EQUAL( 3, len );
}
void UnicodeTestCase::ConversionUTF7()
{
static const StringConversionData utf7data[] =
{
// normal fragments
StringConversionData("+AKM-", L"\xa3"),
StringConversionData("+AOk-t+AOk-", L"\xe9t\xe9"),
// this one is an alternative valid encoding of the same string
StringConversionData("+AOk-t+AOk", L"\xe9t\xe9",
StringConversionData::ONLY_MB2WC),
// some special cases
StringConversionData("+-", L"+"),
StringConversionData("+--", L"+-"),
// the following are invalid UTF-7 sequences
StringConversionData("\xa3", NULL),
StringConversionData("+", NULL),
StringConversionData("+~", NULL),
StringConversionData("a+", NULL),
};
for ( size_t n = 0; n < WXSIZEOF(utf7data); n++ )
{
const StringConversionData& d = utf7data[n];
// converting to/from UTF-7 using iconv() currently doesn't work
// because of several problems:
// - GetMBNulLen() doesn't return correct result (iconv converts L'\0'
// to an incomplete and anyhow nonsensical "+AA" string)
// - iconv refuses to convert "+-" (although it converts "+-\n" just
// fine, go figure)
//
// I have no idea how to fix this so just disable the test for now
#ifdef __WINDOWS__
wxCSConv conv("utf-7");
d.Test(n, conv);
#endif
d.Test(n, wxConvUTF7);
}
}
