std::wstring widen_string(std::string in_str) {
static std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
return converter.from_bytes(in_str);
}
void ResourceLoader::Queue(ResourceType type, const std::wstring& filename, const std::string& name, U32 subType)
{
// TEMP
// Will do the real multithreaded thing once shit actually works
static AssetLocator assLocator;
switch (type)
{
case FONT:
{
static std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converterWtoA;
auto filenameA = converterWtoA.to_bytes(filename);
Font font;
font.LoadFromFile((assLocator.GetAssetDirectoryA(AssetType::FONTS) + filenameA).c_str());
m_pResourceRegistry->AddResource(name, dynamic_cast<Renderer*>(m_pRenderer)->CacheFont(font));
break;
}
case SHADER:
{
m_pResourceRegistry->AddResource(name, m_pRenderer->CacheShader(static_cast<ShaderType>(subType), assLocator.GetAssetDirectory(AssetType::SHADERS) + filename));
break;
}
}
}
void proc_cohort(int& pos,
bool& first_err,
const Cohort& c,
std::ostringstream& text,
std::ostream& os,
const hfst::HfstTransducer& t,
const msgmap& msgs,
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t>& utf16conv)
{
if(cohort_empty(c)) {
return;
}
std::string wfs = utf16conv.to_bytes(c.form);
if(!c.err.empty()) {
if(!first_err) {
os << ",";
}
first_err = false;
os << cohort_errs_json(c, pos, t, msgs);
}
pos += c.form.size();
text << wfs;
// TODO: wrapper for pos-increasing and text-adding, since they should always happen together
}
#include "m1/utf8_decode.hpp"
#include "m1/pragma_message.hpp"
#include "utf8_test_data.hpp"
#include "catch.hpp"
#include <algorithm>
#include <iterator>
#include <vector>
#include <string>
#include <locale>
#include <codecvt>
TEST_CASE("Test m1::utf8_decode", "[m1]")
{
SECTION("utf8_decode_each -- test::utf8_test_data")
{
#pragma M1_MESSAGE("change to char32_t and u32string, this is a workaround for VS 2015 linker error")
std::wstring_convert<std::codecvt_utf8<std::uint32_t>, std::uint32_t> utf32conv;
std::basic_string<std::uint32_t> const utf32_test_data = utf32conv.from_bytes(reinterpret_cast<char const*>(test::utf8_test_data.begin()),
reinterpret_cast<char const*>(test::utf8_test_data.end()));
std::vector<m1::code_point> utf32_decode;
m1::utf8_decode_copy(test::utf8_test_data.begin(),
test::utf8_test_data.end(),
std::back_inserter(utf32_decode));
CHECK(std::equal(utf32_decode.begin(), utf32_decode.end(),
utf32_test_data.begin(), utf32_test_data.end()));
}
}
namespace Rejistry {
std::wstring_convert<std::codecvt_utf16<wchar_t, 0x10ffff, std::little_endian>, wchar_t> conv;
RegistryByteBuffer::RegistryByteBuffer(ByteBuffer * buffer) {
if (buffer == NULL) {
throw std::invalid_argument("Buffer must not be null.");
}
_byteBuffer = buffer;
}
RegistryByteBuffer::~RegistryByteBuffer() {
if (_byteBuffer != NULL) {
delete _byteBuffer;
_byteBuffer = NULL;
}
}
uint16_t RegistryByteBuffer::getWord(const uint32_t offset) const {
return _byteBuffer->getShort(offset) & 0xFFFF;
}
uint32_t RegistryByteBuffer::getDWord(const uint32_t offset) const {
return _byteBuffer->getInt(offset) & 0xFFFFFFFF;
}
uint64_t RegistryByteBuffer::getQWord(const uint32_t offset) const {
return _byteBuffer->getLong(offset) & 0xFFFFFFFFFFFFFFFF;
}
std::string RegistryByteBuffer::getASCIIString() const {
return getASCIIString(0, _byteBuffer->limit());
}
std::string RegistryByteBuffer::getASCIIString(const uint32_t offset, const uint32_t length) const {
if (length == 0) {
return "";
}
ByteBuffer::ByteArray &data = getData(offset, length);
return std::string(data.begin(), data.end());
}
std::wstring RegistryByteBuffer::getUTF16String() const {
return getUTF16String(0, _byteBuffer->limit());
}
std::wstring RegistryByteBuffer::getUTF16String(const uint32_t offset, const uint32_t length) const {
if (length == 0) {
return L"";
}
ByteBuffer::ByteArray &data = getData(offset, length);
// If the size of the array is not a multiple of 2 it is
// likely to not be UTF16 encoded. The most common case is that
// the string is simply missing a terminating null so we add it.
if (data.size() % 2 != 0) {
data.push_back('\0');
}
// Find UTF16 null terminator.
uint32_t nullPos = 0;
for (; nullPos < data.size(); nullPos += 2) {
if (data[nullPos] == '\0' && data[nullPos+1] == '\0') {
break;
}
}
if (nullPos == 0) {
return L"";
}
std::wstring result;
try {
result = conv.from_bytes(reinterpret_cast<const char*>(&data[0]), reinterpret_cast<const char*>(&data[nullPos]));
}
catch (std::exception&)
{
throw RegistryParseException("Error: Failed to convert string");
}
return result;
}
ByteBuffer::ByteArray RegistryByteBuffer::getData() const {
return getData(0, _byteBuffer->limit());
}
ByteBuffer::ByteArray RegistryByteBuffer::getData(const uint32_t offset, const uint32_t length) const {
uint32_t savedPosition = _byteBuffer->position();
_byteBuffer->position(offset);
ByteBuffer::ByteArray data;
data.resize(length);
_byteBuffer->get(data, 0, length);
_byteBuffer->position(savedPosition);
return data;
}
std::vector<std::wstring> RegistryByteBuffer::getStringList() const {
return getStringList(0, _byteBuffer->limit());
}
std::vector<std::wstring> RegistryByteBuffer::getStringList(const uint32_t offset, const uint32_t length) const {
std::vector<std::wstring> stringList;
ByteBuffer::ByteArray data = getData(offset, length);
uint32_t i = 0;
uint32_t pos = 0;
while (i < data.size()) {
if (data[i] == '\0' && data[++i] == '\0') {
stringList.push_back(std::wstring((wchar_t*)&data[pos]));
pos = i;
while (i < data.size() && data[i] == '\0') {
pos++; i++;
}
}
else {
i++;
}
}
return stringList;
}
};
void pal::to_stdstring(const pal::char_t* str, std::string* out)
{
out->assign(g_converter.to_bytes(str));
}
void pal::to_palstring(const char* str, pal::string_t* out)
{
out->assign(g_converter.from_bytes(str));
}
pal::string_t pal::to_palstring(const std::string& str)
{
return g_converter.from_bytes(str);
}
std::string pal::to_stdstring(const string_t& str)
{
return g_converter.to_bytes(str);
}
std::wstring Widen( const std::string& str )
{
static std::wstring_convert< std::codecvt_utf8< wchar_t >, wchar_t > converter;
return converter.from_bytes( str );
}
std::string Narrow( const std::wstring& wstr )
{
static std::wstring_convert< std::codecvt_utf8< wchar_t >, wchar_t > converter;
return converter.to_bytes( wstr );
}
std::wstring RegistryByteBuffer::getUTF16String(const uint32_t offset, const uint32_t length) const {
if (length == 0) {
return L"";
}
ByteBuffer::ByteArray &data = getData(offset, length);
// If the size of the array is not a multiple of 2 it is
// likely to not be UTF16 encoded. The most common case is that
// the string is simply missing a terminating null so we add it.
if (data.size() % 2 != 0) {
data.push_back('\0');
}
// Find UTF16 null terminator.
uint32_t nullPos = 0;
for (; nullPos < data.size(); nullPos += 2) {
if (data[nullPos] == '\0' && data[nullPos+1] == '\0') {
break;
}
}
if (nullPos == 0) {
return L"";
}
std::wstring result;
try {
result = conv.from_bytes(reinterpret_cast<const char*>(&data[0]), reinterpret_cast<const char*>(&data[nullPos]));
}
catch (std::exception&)
{
throw RegistryParseException("Error: Failed to convert string");
}
return result;
}