std::pair<iterator, bool> insert_impl(trie_node_type& __node, const std::basic_string<T>& __word, size_t __at) {
if (__at == __word.size()) {
++__node._M_words;
++__node._M_prefix;
return std::make_pair(trie_iterator<trie_node_type>(__word, &__node), false);
}
++__node._M_prefix;
trie_node_type *_new_node = nullptr;
bool _new_insert = false;
auto _it = __node._M_children.find(__word[__at]);
if (_it == __node._M_children.end()) {
_new_node = new trie_node_type();
_new_insert = true;
__node._M_children[__word[__at]].reset(_new_node);
}
else {
_new_node = _it->second.get();
}
auto _ret = insert_impl(*_new_node, __word, ++__at);
_ret.second |= _new_insert;
return _ret;
}
inline void kbe_split(const std::basic_string<T>& s, T c, std::vector< std::basic_string<T> > &v)
{
if(s.size() == 0)
return;
typename std::basic_string< T >::size_type i = 0;
typename std::basic_string< T >::size_type j = s.find(c);
while(j != std::basic_string<T>::npos)
{
std::basic_string<T> buf = s.substr(i, j - i);
if(buf.size() > 0)
v.push_back(buf);
i = ++j; j = s.find(c, j);
}
if(j == std::basic_string<T>::npos)
{
std::basic_string<T> buf = s.substr(i, s.length() - i);
if(buf.size() > 0)
v.push_back(buf);
}
}
void test_normc(std::basic_string<Char> orig,std::basic_string<Char> normal,boost::locale::norm_type type)
{
std::locale l = boost::locale::generator().generate("en_US.UTF-8");
TEST(normalize(orig,type,l)==normal);
TEST(normalize(orig.c_str(),type,l)==normal);
TEST(normalize(orig.c_str(),orig.c_str()+orig.size(),type,l)==normal);
}
void TokenizerTest()
{
const std::basic_string<utf16_t> xmlText =
Transcoder::UTF8toUTF16("<token1><token2 />\nhello world.\nhello xml.</token1>");
XMLParser<utf16_t>::Tokenizer tokenizer(xmlText.c_str(), xmlText.c_str() + xmlText.size());
std::basic_string<utf16_t> result = tokenizer.getToken();
CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result),
result == Transcoder::UTF8toUTF16("<token1>"));
result = tokenizer.getToken();
CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result),
result == Transcoder::UTF8toUTF16("<token2 />"));
result = tokenizer.getToken();
CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result),
result == Transcoder::UTF8toUTF16("\nhello world.\nhello xml."));
result = tokenizer.getToken();
CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result),
result == Transcoder::UTF8toUTF16("</token1>"));
CPPUNIT_ASSERT(tokenizer.isEof());
result = tokenizer.getToken();
CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result),
result == Transcoder::UTF8toUTF16(""));
CPPUNIT_ASSERT(tokenizer.isEof());
}
void CPwQualityEst::_EnsureInitialized()
{
if(m_vCharTypes.size() == 0)
{
std::basic_string<WCHAR> strSpecial(PDCS_PRINTASCIISPEC);
if(strSpecial.find(L' ') != strSpecial.npos) { ASSERT(FALSE); }
else strSpecial += L' ';
const std::basic_string<WCHAR> strHigh =
PwCharSet::GetHighAnsiChars().ToString();
m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>(
new CQeCharType(QE_PAT_LOWERALPHA, PDCS_LOWER_CASE, true)));
m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>(
new CQeCharType(QE_PAT_UPPERALPHA, PDCS_UPPER_CASE, true)));
m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>(
new CQeCharType(QE_PAT_DIGIT, PDCS_NUMERIC, true)));
m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>(
new CQeCharType(QE_PAT_SPECIAL, strSpecial.c_str(), false)));
m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>(
new CQeCharType(QE_PAT_HIGH, strHigh.c_str(), false)));
m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>(
new CQeCharType(QE_PAT_OTHER, 0x10000U - (2U * 26U) - 10U -
strSpecial.size() - strHigh.size())));
}
}
static inline void split(_OutputIter iter,
const std::basic_string<_CharT, _Traits, _Alloc>& s,
int nsplits = -1) {
std::locale loc;
typedef std::basic_string<_CharT, _Traits, _Alloc> str_t;
int x = 0, rx = 0;
int y = (int)s.size() - 1;
while (std::isspace(s[x], loc))
x++;
rx = x;
while (y >= 0 && std::isspace(s[y], loc))
y--;
y++;
while (x < y) {
if (std::isspace(s[x], loc)) {
*iter++ = str_t(s, rx, x - rx);
while (x < y && std::isspace(s[x], loc))
x++;
rx = x;
if (!--nsplits) {
*iter++ = str_t(s, rx);
return;
}
} else {
x++;
}
}
if (rx != y) {
*iter++ = str_t(s, rx, y - rx);
}
}
std::vector<std::basic_string<T_Char>> split(
const std::basic_string<T_Char>& src,
const std::basic_string<T_Char>& delimit) {
std::vector<std::basic_string<T_Char>> array;
typedef typename std::basic_string<T_Char>::size_type size_type;
typedef typename std::basic_string<T_Char>::const_iterator const_iterator;
std::basic_string<T_Char> tmp;
if (src.empty())
return array;
if (delimit.empty()) {
array.reserve(array.size() + src.size());
for (const_iterator it = src.begin(); it != src.end(); ++it)
array.push_back(std::basic_string<T_Char>(1, *it));
return array;
}
size_type src_pos = 0;
while (src.begin() + src_pos != src.end()) {
size_type fnd_pos = src.find(delimit, src_pos);
if (fnd_pos == std::basic_string<T_Char>::npos) {
array.push_back(std::basic_string<T_Char>(src, src_pos));
break;
}
array.push_back(
std::basic_string<T_Char>(src, src_pos, fnd_pos - src_pos));
src_pos = fnd_pos + delimit.length();
}
return array;
}
Ret benchmark1(const std::basic_string<C>& str, const std::basic_string<C>& key, F f)
{
const int N = 1;
int val = 0;
f.set(str, key);
Xbyak::util::Clock clk;
for (int i = 0; i < N; i++) {
typename F::type p = f.begin();
typename F::type end = f.end();
for (;;) {
clk.begin();
typename F::type q = f.find(p);
clk.end();
if (q == end) break;
val += (int)(q - p);
p = q + 1;
}
}
if (val == 0) val = (int)(str.size()) * N;
Ret ret;
ret.val = val;
ret.clk = clk.getClock() / (double)val;
return ret;
}
testbuf(const std::basic_string<CharT>& str)
: str_(str)
{
base::setg(const_cast<CharT*>(str_.data()),
const_cast<CharT*>(str_.data()),
const_cast<CharT*>(str_.data() + str_.size()));
}
std::basic_string<Ch> encode_char_entities(const std::basic_string<Ch> &s)
{
// Don't do anything for empty strings.
if(s.empty()) return s;
typedef typename std::basic_string<Ch> Str;
Str r;
// To properly round-trip spaces and not uglify the XML beyond
// recognition, we have to encode them IF the text contains only spaces.
Str sp(1, Ch(' '));
if(s.find_first_not_of(sp) == Str::npos) {
// The first will suffice.
r = detail::widen<Ch>(" ");
r += Str(s.size() - 1, Ch(' '));
} else {
typename Str::const_iterator end = s.end();
for (typename Str::const_iterator it = s.begin(); it != end; ++it)
{
switch (*it)
{
case Ch('<'): r += detail::widen<Ch>("<"); break;
case Ch('>'): r += detail::widen<Ch>(">"); break;
case Ch('&'): r += detail::widen<Ch>("&"); break;
case Ch('"'): r += detail::widen<Ch>("""); break;
case Ch('\''): r += detail::widen<Ch>("'"); break;
default: r += *it; break;
}
}
}
return r;
}
void split(charT pat,
const std::basic_string<charT, traits, Alloc>& s, Iter out) {
unsigned int pos = 0;
for (unsigned int i = 0; i < s.size(); ++i) {
if (s[i] == pat) {
if (i - pos > 0) {
*(out++) =
std::basic_string<charT, traits, Alloc>(s, pos, i - pos);
}
pos = i + 1;
}
}
*(out++) =
std::basic_string<charT, traits, Alloc>(s, pos, s.size() - pos);
} // split
inline std::basic_string<typename re_detail::regex_iterator_traits<RandomAccessIterator>::value_type, traits, Allocator>
operator + (const sub_match<RandomAccessIterator>& m,
const std::basic_string<typename re_detail::regex_iterator_traits<RandomAccessIterator>::value_type, traits, Allocator>& s)
{
std::basic_string<typename re_detail::regex_iterator_traits<RandomAccessIterator>::value_type, traits, Allocator> result;
result.reserve(s.size() + m.length() + 1);
return result.append(m.first, m.second).append(s);
}
HGLOBAL CreateGlobalData(const std::basic_string<charT>& str)
{
HGLOBAL data = ::GlobalAlloc(GMEM_MOVEABLE, ((str.size() + 1) * sizeof(charT)));
if (data)
{
charT* raw_data = static_cast<charT*>(::GlobalLock(data));
if (!raw_data)
{
::GlobalUnlock(data);
return nullptr;
}
memcpy(raw_data, str.data(), str.size() * sizeof(charT));
raw_data[str.size()] = '\0';
::GlobalUnlock(data);
}
return data;
}
inline std::string u16tos(const std::basic_string<MIE_CHAR16>& str)
{
std::string ret;
for (size_t i = 0; i < str.size(); i++) {
ret += (char)str[i];
}
return ret;
}
static bool matches(
std::basic_string< CharT, StringTraitsT, AllocatorT > const& str,
boost::basic_regex< CharT, ReTraitsT > const& expr,
boost::regex_constants::match_flag_type flags = boost::regex_constants::match_default)
{
const CharT* p = str.c_str();
return boost::regex_match(p, p + str.size(), expr, flags);
}
int main()
{
std::locale l = std::locale::classic();
const F& f = std::use_facet<F>(l);
{
const std::basic_string<F::intern_type> from(L"some text");
std::vector<char> to(from.size()+1);
std::mbstate_t mbs = {};
const F::intern_type* from_next = 0;
char* to_next = 0;
F::result r = f.out(mbs, from.data(), from.data() + from.size(), from_next,
to.data(), to.data() + to.size(), to_next);
assert(r == F::ok);
assert(static_cast<std::size_t>(from_next - from.data()) == from.size());
assert(static_cast<std::size_t>(to_next - to.data()) == from.size());
assert(to.data() == std::string("some text"));
}
{
std::basic_string<F::intern_type> from(L"some text");
from[4] = '\0';
std::vector<char> to(from.size()+1);
std::mbstate_t mbs = {};
const F::intern_type* from_next = 0;
char* to_next = 0;
F::result r = f.out(mbs, from.data(), from.data() + from.size(), from_next,
to.data(), to.data() + to.size(), to_next);
assert(r == F::ok);
assert(static_cast<std::size_t>(from_next - from.data()) == from.size());
assert(static_cast<std::size_t>(to_next - to.data()) == from.size());
assert(memcmp(to.data(), "some\0text", from.size()) == 0);
}
{
std::basic_string<F::intern_type> from(L"some text");
std::vector<char> to(from.size()-1);
std::mbstate_t mbs = {};
const F::intern_type* from_next = 0;
char* to_next = 0;
F::result r = f.out(mbs, from.data(), from.data() + from.size(), from_next,
to.data(), to.data() + to.size()-1, to_next);
assert(r == F::partial);
assert(static_cast<std::size_t>(from_next - from.data()) == to.size()-1);
assert(static_cast<std::size_t>(to_next - to.data()) == to.size()-1);
assert(to.data() == std::string("some te"));
}
}
cstr(const std::basic_string<char_type, CharTraits, Allocator>& s)
{
if (s.empty()) {
clear();
} else {
buf_ = s.c_str();
size_ = s.size();
}
}
int main()
{
std::locale l = std::locale::classic();
const std::basic_string<F::extern_type> from("some text");
const std::basic_string<F::intern_type> expected(from.begin(), from.end());
std::basic_string<F::intern_type> to(from.size(), F::intern_type());
const F& f = std::use_facet<F>(l);
std::mbstate_t mbs = {};
const F::extern_type* from_next = 0;
F::intern_type* to_next = 0;
F::result r = f.in(mbs, from.data(), from.data() + from.size(), from_next,
&to[0], &to[0] + to.size(), to_next);
assert(r == F::ok);
assert(from_next - from.data() == from.size());
assert(to_next - to.data() == expected.size());
assert(to_next - to.data() == expected.size());
assert(to == expected);
}
static inline std::basic_string<_CharT, _Traits, _Alloc> rstrip(
const std::basic_string<_CharT, _Traits, _Alloc> &a) {
typedef std::basic_string<_CharT, _Traits, _Alloc> str_t;
typename str_t::size_type p = a.size();
std::locale loc;
while (p && std::isspace(a[p-1], loc)) p--;
if (!p) return "";
return a.substr(0, p);
}
inline int string_compare(const std::basic_string<C,T,A>& s, const C* p)
{
if(0 == *p)
{
if(s.empty() || ((s.size() == 1) && (s[0] == 0)))
return 0;
}
return s.compare(p);
}
std::basic_string<charT, traits, Alloc>
remove_space(const std::basic_string<charT, traits, Alloc>& s,
const std::locale& loc = std::locale::classic()) {
std::basic_string<charT, traits, Alloc> ns;
for (std::size_t i = 0; i < s.size(); ++i) {
if (std::isspace(s[i], loc) == false) ns.push_back(s[i]);
}
return ns;
} // remove_space
int operator()(const std::basic_string<charT, traits, Alloc>& s1,
const std::basic_string<charT, traits, Alloc>& s2) const {
std::size_t l1 = s1.size();
std::size_t l2 = s2.size();
if (l1 < l2) return -1; else if (l2 < l1) return 1;
charT c1, c2;
for (std::size_t i = 0; i < l1; ++i) {
c1 = ct_.toupper(s1[i]);
c2 = ct_.toupper(s2[i]);
if (c1 < c2) return -1; else if (c2 < c1) return 1;
}
return 0;
} // operator()
typename boost::disable_if<
std::is_same<typename std::decay<Sink>::type::element_type,
std::basic_string<Element>>,
typename error_type<Sink>::type>::type
append(Sink &&out, std::basic_string<Element> const &str)
{
return out.append(
make_iterator_range(str.data(), str.data() + str.size()));
}
OutputIterator regex_format(OutputIterator out,
const match_results<Iterator>& m,
const std::basic_string<charT>& fmt,
match_flag_type flags = format_all
)
{
re_detail::trivial_format_traits<charT> traits;
return re_detail::regex_format_imp(out, m, fmt.data(), fmt.data() + fmt.size(), flags, traits);
}
std::basic_string< wchar_t > str_convert< wchar_t, char >( std::basic_string< char > const & src )
{
std::basic_string< wchar_t > dst;
if ( !src.empty() )
{
try
{
std::unique_lock< std::mutex > lock( g_conversionMutex );
char * szloc = setlocale( LC_CTYPE, "" );
mbtowc( NULL, NULL, 0 );
size_t max = src.size();
int length = 1;
const char * in = src.c_str();
dst.reserve( src.size() );
while ( max > 0 && length >= 1 )
{
wchar_t wc;
length = mbtowc( &wc, in, max );
if ( length >= 1 )
{
dst += wc;
max -= length;
in += length;
}
}
setlocale( LC_CTYPE, szloc );
}
catch ( std::exception & exc )
{
CLogger::LogError( StringStream() << ERROR_DB_CONVERSION << exc.what() );
}
catch ( ... )
{
CLogger::LogError( StringStream() << ERROR_DB_CONVERSION << INFO_UNKNOWN );
}
}
return std::move( dst );
}
std::basic_string<charT> regex_format(const match_results<Iterator>& m,
const std::basic_string<charT>& fmt,
match_flag_type flags = format_all)
{
std::basic_string<charT> result;
re_detail::string_out_iterator<std::basic_string<charT> > i(result);
re_detail::trivial_format_traits<charT> traits;
re_detail::regex_format_imp(i, m, fmt.data(), fmt.data() + fmt.size(), flags, traits);
return result;
}
inline void WriteString(Process const& process,
PVOID address,
std::basic_string<T, Traits, Alloc> const& data)
{
HADESMEM_DETAIL_STATIC_ASSERT(detail::IsCharType<T>::value);
HADESMEM_DETAIL_ASSERT(address != nullptr);
return Write(process, address, data.c_str(), data.size() + 1);
}
auto GenRString(const RDI& rdi, size_t len, const std::basic_string<C>& src) {
std::basic_string<C> result;
result.resize(len);
auto srcLen = src.size();
for(size_t i=0 ; i<len ; i++) {
int cn = rdi({0, srcLen-1});
result[i] = src[cn];
}
return result;
}
static std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
{
std::string result;
iconv_t const conv_desc = iconv_open("UTF-8", fromcode);
if ((iconv_t)(-1) == conv_desc)
{
LOG_ERROR(Common, "Iconv initialization failure [%s]: %s", fromcode, strerror(errno));
iconv_close(conv_desc);
return {};
}
const size_t in_bytes = sizeof(T) * input.size();
// Multiply by 4, which is the max number of bytes to encode a codepoint
const size_t out_buffer_size = 4 * in_bytes;
std::string out_buffer;
out_buffer.resize(out_buffer_size);
auto src_buffer = &input[0];
size_t src_bytes = in_bytes;
auto dst_buffer = &out_buffer[0];
size_t dst_bytes = out_buffer.size();
while (0 != src_bytes)
{
size_t const iconv_result = iconv(conv_desc, (char**)(&src_buffer), &src_bytes,
&dst_buffer, &dst_bytes);
if (static_cast<size_t>(-1) == iconv_result)
{
if (EILSEQ == errno || EINVAL == errno)
{
// Try to skip the bad character
if (0 != src_bytes)
{
--src_bytes;
++src_buffer;
}
}
else
{
LOG_ERROR(Common, "iconv failure [%s]: %s", fromcode, strerror(errno));
break;
}
}
}
out_buffer.resize(out_buffer_size - dst_bytes);
out_buffer.swap(result);
iconv_close(conv_desc);
return result;
}
void test_from_neg(std::basic_string<Char> source,std::string target,std::string encoding)
{
using namespace boost::locale::conv;
boost::locale::generator g;
std::locale l=g("en_US."+encoding);
TEST(from_utf<Char>(source,encoding)==target);
TEST(from_utf<Char>(source.c_str(),encoding)==target);
TEST(from_utf<Char>(source.c_str(),source.c_str()+source.size(),encoding)==target);
TEST(from_utf<Char>(source,l)==target);
TEST(from_utf<Char>(source.c_str(),l)==target);
TEST(from_utf<Char>(source.c_str(),source.c_str()+source.size(),l)==target);
TESTF(from_utf<Char>(source,encoding,stop));
TESTF(from_utf<Char>(source.c_str(),encoding,stop));
TESTF(from_utf<Char>(source.c_str(),source.c_str()+source.size(),encoding,stop));
TESTF(from_utf<Char>(source,l,stop));
TESTF(from_utf<Char>(source.c_str(),l,stop));
TESTF(from_utf<Char>(source.c_str(),source.c_str()+source.size(),l,stop));
}
