std::wstring widen_string(std::string in_str) { static std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter; return converter.from_bytes(in_str); }
void ResourceLoader::Queue(ResourceType type, const std::wstring& filename, const std::string& name, U32 subType) { // TEMP // Will do the real multithreaded thing once shit actually works static AssetLocator assLocator; switch (type) { case FONT: { static std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converterWtoA; auto filenameA = converterWtoA.to_bytes(filename); Font font; font.LoadFromFile((assLocator.GetAssetDirectoryA(AssetType::FONTS) + filenameA).c_str()); m_pResourceRegistry->AddResource(name, dynamic_cast<Renderer*>(m_pRenderer)->CacheFont(font)); break; } case SHADER: { m_pResourceRegistry->AddResource(name, m_pRenderer->CacheShader(static_cast<ShaderType>(subType), assLocator.GetAssetDirectory(AssetType::SHADERS) + filename)); break; } } }
void proc_cohort(int& pos, bool& first_err, const Cohort& c, std::ostringstream& text, std::ostream& os, const hfst::HfstTransducer& t, const msgmap& msgs, std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t>& utf16conv) { if(cohort_empty(c)) { return; } std::string wfs = utf16conv.to_bytes(c.form); if(!c.err.empty()) { if(!first_err) { os << ","; } first_err = false; os << cohort_errs_json(c, pos, t, msgs); } pos += c.form.size(); text << wfs; // TODO: wrapper for pos-increasing and text-adding, since they should always happen together }
#include "m1/utf8_decode.hpp" #include "m1/pragma_message.hpp" #include "utf8_test_data.hpp" #include "catch.hpp" #include <algorithm> #include <iterator> #include <vector> #include <string> #include <locale> #include <codecvt> TEST_CASE("Test m1::utf8_decode", "[m1]") { SECTION("utf8_decode_each -- test::utf8_test_data") { #pragma M1_MESSAGE("change to char32_t and u32string, this is a workaround for VS 2015 linker error") std::wstring_convert<std::codecvt_utf8<std::uint32_t>, std::uint32_t> utf32conv; std::basic_string<std::uint32_t> const utf32_test_data = utf32conv.from_bytes(reinterpret_cast<char const*>(test::utf8_test_data.begin()), reinterpret_cast<char const*>(test::utf8_test_data.end())); std::vector<m1::code_point> utf32_decode; m1::utf8_decode_copy(test::utf8_test_data.begin(), test::utf8_test_data.end(), std::back_inserter(utf32_decode)); CHECK(std::equal(utf32_decode.begin(), utf32_decode.end(), utf32_test_data.begin(), utf32_test_data.end())); } }
namespace Rejistry { std::wstring_convert<std::codecvt_utf16<wchar_t, 0x10ffff, std::little_endian>, wchar_t> conv; RegistryByteBuffer::RegistryByteBuffer(ByteBuffer * buffer) { if (buffer == NULL) { throw std::invalid_argument("Buffer must not be null."); } _byteBuffer = buffer; } RegistryByteBuffer::~RegistryByteBuffer() { if (_byteBuffer != NULL) { delete _byteBuffer; _byteBuffer = NULL; } } uint16_t RegistryByteBuffer::getWord(const uint32_t offset) const { return _byteBuffer->getShort(offset) & 0xFFFF; } uint32_t RegistryByteBuffer::getDWord(const uint32_t offset) const { return _byteBuffer->getInt(offset) & 0xFFFFFFFF; } uint64_t RegistryByteBuffer::getQWord(const uint32_t offset) const { return _byteBuffer->getLong(offset) & 0xFFFFFFFFFFFFFFFF; } std::string RegistryByteBuffer::getASCIIString() const { return getASCIIString(0, _byteBuffer->limit()); } std::string RegistryByteBuffer::getASCIIString(const uint32_t offset, const uint32_t length) const { if (length == 0) { return ""; } ByteBuffer::ByteArray &data = getData(offset, length); return std::string(data.begin(), data.end()); } std::wstring RegistryByteBuffer::getUTF16String() const { return getUTF16String(0, _byteBuffer->limit()); } std::wstring RegistryByteBuffer::getUTF16String(const uint32_t offset, const uint32_t length) const { if (length == 0) { return L""; } ByteBuffer::ByteArray &data = getData(offset, length); // If the size of the array is not a multiple of 2 it is // likely to not be UTF16 encoded. The most common case is that // the string is simply missing a terminating null so we add it. if (data.size() % 2 != 0) { data.push_back('\0'); } // Find UTF16 null terminator. uint32_t nullPos = 0; for (; nullPos < data.size(); nullPos += 2) { if (data[nullPos] == '\0' && data[nullPos+1] == '\0') { break; } } if (nullPos == 0) { return L""; } std::wstring result; try { result = conv.from_bytes(reinterpret_cast<const char*>(&data[0]), reinterpret_cast<const char*>(&data[nullPos])); } catch (std::exception&) { throw RegistryParseException("Error: Failed to convert string"); } return result; } ByteBuffer::ByteArray RegistryByteBuffer::getData() const { return getData(0, _byteBuffer->limit()); } ByteBuffer::ByteArray RegistryByteBuffer::getData(const uint32_t offset, const uint32_t length) const { uint32_t savedPosition = _byteBuffer->position(); _byteBuffer->position(offset); ByteBuffer::ByteArray data; data.resize(length); _byteBuffer->get(data, 0, length); _byteBuffer->position(savedPosition); return data; } std::vector<std::wstring> RegistryByteBuffer::getStringList() const { return getStringList(0, _byteBuffer->limit()); } std::vector<std::wstring> RegistryByteBuffer::getStringList(const uint32_t offset, const uint32_t length) const { std::vector<std::wstring> stringList; ByteBuffer::ByteArray data = getData(offset, length); uint32_t i = 0; uint32_t pos = 0; while (i < data.size()) { if (data[i] == '\0' && data[++i] == '\0') { stringList.push_back(std::wstring((wchar_t*)&data[pos])); pos = i; while (i < data.size() && data[i] == '\0') { pos++; i++; } } else { i++; } } return stringList; } };
void pal::to_stdstring(const pal::char_t* str, std::string* out) { out->assign(g_converter.to_bytes(str)); }
void pal::to_palstring(const char* str, pal::string_t* out) { out->assign(g_converter.from_bytes(str)); }
pal::string_t pal::to_palstring(const std::string& str) { return g_converter.from_bytes(str); }
std::string pal::to_stdstring(const string_t& str) { return g_converter.to_bytes(str); }
std::wstring Widen( const std::string& str ) { static std::wstring_convert< std::codecvt_utf8< wchar_t >, wchar_t > converter; return converter.from_bytes( str ); }
std::string Narrow( const std::wstring& wstr ) { static std::wstring_convert< std::codecvt_utf8< wchar_t >, wchar_t > converter; return converter.to_bytes( wstr ); }
std::wstring RegistryByteBuffer::getUTF16String(const uint32_t offset, const uint32_t length) const { if (length == 0) { return L""; } ByteBuffer::ByteArray &data = getData(offset, length); // If the size of the array is not a multiple of 2 it is // likely to not be UTF16 encoded. The most common case is that // the string is simply missing a terminating null so we add it. if (data.size() % 2 != 0) { data.push_back('\0'); } // Find UTF16 null terminator. uint32_t nullPos = 0; for (; nullPos < data.size(); nullPos += 2) { if (data[nullPos] == '\0' && data[nullPos+1] == '\0') { break; } } if (nullPos == 0) { return L""; } std::wstring result; try { result = conv.from_bytes(reinterpret_cast<const char*>(&data[0]), reinterpret_cast<const char*>(&data[nullPos])); } catch (std::exception&) { throw RegistryParseException("Error: Failed to convert string"); } return result; }