void UnicodeTestCase::ConversionUTF8() { static const StringConversionData utf8data[] = { #ifdef wxHAVE_U_ESCAPE StringConversionData("\xc2\xa3", L"\u00a3"), #endif StringConversionData("\xc2", NULL), }; wxCSConv conv(wxT("utf-8")); for ( size_t n = 0; n < WXSIZEOF(utf8data); n++ ) { const StringConversionData& d = utf8data[n]; d.Test(n, conv); d.Test(n, wxConvUTF8); } static const char* const u25a6 = "\xe2\x96\xa6"; wxMBConvUTF8 c(wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL); CPPUNIT_ASSERT_EQUAL( 2, c.ToWChar(NULL, 0, u25a6, wxNO_LEN) ); CPPUNIT_ASSERT_EQUAL( 0, c.ToWChar(NULL, 0, u25a6, 0) ); CPPUNIT_ASSERT_EQUAL( 1, c.ToWChar(NULL, 0, u25a6, 3) ); CPPUNIT_ASSERT_EQUAL( 2, c.ToWChar(NULL, 0, u25a6, 4) ); // Verify that converting a string with embedded NULs works. CPPUNIT_ASSERT_EQUAL( 5, wxString::FromUTF8("abc\0\x32", 5).length() ); // Verify that converting a string containing invalid UTF-8 does not work, // even if it happens after an embedded NUL. CPPUNIT_ASSERT( wxString::FromUTF8("abc\xff").empty() ); CPPUNIT_ASSERT( wxString::FromUTF8("abc\0\xff", 5).empty() ); }
void UnicodeTestCase::ConversionUTF8() { static const StringConversionData utf8data[] = { #ifdef wxHAVE_U_ESCAPE StringConversionData("\xc2\xa3", L"\u00a3"), #endif StringConversionData("\xc2", NULL), }; wxCSConv conv(wxT("utf-8")); for ( size_t n = 0; n < WXSIZEOF(utf8data); n++ ) { const StringConversionData& d = utf8data[n]; d.Test(n, conv); d.Test(n, wxConvUTF8); } static const char* const u25a6 = "\xe2\x96\xa6"; wxMBConvUTF8 c(wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL); CPPUNIT_ASSERT_EQUAL( 2, c.ToWChar(NULL, 0, u25a6, wxNO_LEN) ); CPPUNIT_ASSERT_EQUAL( 0, c.ToWChar(NULL, 0, u25a6, 0) ); CPPUNIT_ASSERT_EQUAL( 1, c.ToWChar(NULL, 0, u25a6, 3) ); CPPUNIT_ASSERT_EQUAL( 2, c.ToWChar(NULL, 0, u25a6, 4) ); }
void UnicodeTestCase::ConversionUTF16() { static const StringConversionData utf16data[] = { #ifdef wxHAVE_U_ESCAPE StringConversionData( "\x04\x1f\x04\x40\x04\x38\x04\x32\x04\x35\x04\x42\0\0", L"\u041f\u0440\u0438\u0432\u0435\u0442"), StringConversionData( "\x01\0\0b\x01\0\0a\x01\0\0r\0\0", L"\u0100b\u0100a\u0100r"), #endif StringConversionData("\0f\0o\0o\0\0", L"foo"), }; wxCSConv conv(wxFONTENCODING_UTF16BE); for ( size_t n = 0; n < WXSIZEOF(utf16data); n++ ) { const StringConversionData& d = utf16data[n]; d.Test(n, conv); } // special case: this string has consecutive NULs inside it which don't // terminate the string, this exposed a bug in our conversion code which // got confused in this case size_t len; conv.cMB2WC("\x01\0\0B\0C" /* A macron BC */, 6, &len); CPPUNIT_ASSERT_EQUAL( 3, len ); // Another one: verify that the length of the resulting string is computed // correctly when there is a surrogate in the input. wxMBConvUTF16BE().cMB2WC("\xd8\x03\xdc\x01\0" /* OLD TURKIC LETTER YENISEI A */, wxNO_LEN, &len); CPPUNIT_ASSERT_EQUAL( 1, len ); }
void UnicodeTestCase::ConversionUTF16() { static const StringConversionData utf16data[] = { #ifdef wxHAVE_U_ESCAPE StringConversionData( "\x04\x1f\x04\x40\x04\x38\x04\x32\x04\x35\x04\x42\0\0", L"\u041f\u0440\u0438\u0432\u0435\u0442"), StringConversionData( "\x01\0\0b\x01\0\0a\x01\0\0r\0\0", L"\u0100b\u0100a\u0100r"), #endif StringConversionData("\0f\0o\0o\0\0", L"foo"), }; wxCSConv conv(wxFONTENCODING_UTF16BE); for ( size_t n = 0; n < WXSIZEOF(utf16data); n++ ) { const StringConversionData& d = utf16data[n]; d.Test(n, conv); } // special case: this string has consecutive NULs inside it which don't // terminate the string, this exposed a bug in our conversion code which // got confused in this case size_t len; conv.cMB2WC("\x01\0\0B\0C" /* A macron BC */, 6, &len); CPPUNIT_ASSERT_EQUAL( 3, len ); // When using UTF-16 internally (i.e. MSW), we don't have any surrogate // support, so the length of the string below is 2, not 1. #if SIZEOF_WCHAR_T == 4 // Another one: verify that the length of the resulting string is computed // correctly when there is a surrogate in the input. wxMBConvUTF16BE().cMB2WC("\xd8\x03\xdc\x01\0" /* OLD TURKIC LETTER YENISEI A */, wxNO_LEN, &len); CPPUNIT_ASSERT_EQUAL( 1, len ); #endif // UTF-32 internal representation #if SIZEOF_WCHAR_T == 2 // Verify that the length of UTF-32 string is correct even when converting // to it from a longer UTF-16 string with surrogates. // Construct CAT FACE U+1F431 without using \U which is not supported by // ancient compilers and without using \u with surrogates which is // (correctly) flagged as an error by the newer ones. wchar_t ws[2]; ws[0] = 0xd83d; ws[1] = 0xdc31; CPPUNIT_ASSERT_EQUAL( 4, wxMBConvUTF32BE().FromWChar(NULL, 0, ws, 2) ); #endif // UTF-16 internal representation }
void UnicodeTestCase::ConversionUTF32() { static const StringConversionData utf32data[] = { #ifdef wxHAVE_U_ESCAPE StringConversionData( "\0\0\x04\x1f\0\0\x04\x40\0\0\x04\x38\0\0\x04\x32\0\0\x04\x35\0\0\x04\x42\0\0\0\0", L"\u041f\u0440\u0438\u0432\u0435\u0442"), #endif StringConversionData("\0\0\0f\0\0\0o\0\0\0o\0\0\0\0", L"foo"), }; wxCSConv conv(wxFONTENCODING_UTF32BE); for ( size_t n = 0; n < WXSIZEOF(utf32data); n++ ) { const StringConversionData& d = utf32data[n]; d.Test(n, conv); } size_t len; conv.cMB2WC("\0\0\x01\0\0\0\0B\0\0\0C" /* A macron BC */, 12, &len); CPPUNIT_ASSERT_EQUAL( 3, len ); }
void UnicodeTestCase::ConversionUTF7() { static const StringConversionData utf7data[] = { // normal fragments StringConversionData("+AKM-", L"\xa3"), StringConversionData("+AOk-t+AOk-", L"\xe9t\xe9"), // this one is an alternative valid encoding of the same string StringConversionData("+AOk-t+AOk", L"\xe9t\xe9", StringConversionData::ONLY_MB2WC), // some special cases StringConversionData("+-", L"+"), StringConversionData("+--", L"+-"), // the following are invalid UTF-7 sequences StringConversionData("\xa3", NULL), StringConversionData("+", NULL), StringConversionData("+~", NULL), StringConversionData("a+", NULL), }; for ( size_t n = 0; n < WXSIZEOF(utf7data); n++ ) { const StringConversionData& d = utf7data[n]; // converting to/from UTF-7 using iconv() currently doesn't work // because of several problems: // - GetMBNulLen() doesn't return correct result (iconv converts L'\0' // to an incomplete and anyhow nonsensical "+AA" string) // - iconv refuses to convert "+-" (although it converts "+-\n" just // fine, go figure) // // I have no idea how to fix this so just disable the test for now #ifdef __WINDOWS__ wxCSConv conv("utf-7"); d.Test(n, conv); #endif d.Test(n, wxConvUTF7); } }