std::pair<iterator, bool> insert_impl(trie_node_type& __node, const std::basic_string<T>& __word, size_t __at) { if (__at == __word.size()) { ++__node._M_words; ++__node._M_prefix; return std::make_pair(trie_iterator<trie_node_type>(__word, &__node), false); } ++__node._M_prefix; trie_node_type *_new_node = nullptr; bool _new_insert = false; auto _it = __node._M_children.find(__word[__at]); if (_it == __node._M_children.end()) { _new_node = new trie_node_type(); _new_insert = true; __node._M_children[__word[__at]].reset(_new_node); } else { _new_node = _it->second.get(); } auto _ret = insert_impl(*_new_node, __word, ++__at); _ret.second |= _new_insert; return _ret; }
inline void kbe_split(const std::basic_string<T>& s, T c, std::vector< std::basic_string<T> > &v) { if(s.size() == 0) return; typename std::basic_string< T >::size_type i = 0; typename std::basic_string< T >::size_type j = s.find(c); while(j != std::basic_string<T>::npos) { std::basic_string<T> buf = s.substr(i, j - i); if(buf.size() > 0) v.push_back(buf); i = ++j; j = s.find(c, j); } if(j == std::basic_string<T>::npos) { std::basic_string<T> buf = s.substr(i, s.length() - i); if(buf.size() > 0) v.push_back(buf); } }
void test_normc(std::basic_string<Char> orig,std::basic_string<Char> normal,boost::locale::norm_type type) { std::locale l = boost::locale::generator().generate("en_US.UTF-8"); TEST(normalize(orig,type,l)==normal); TEST(normalize(orig.c_str(),type,l)==normal); TEST(normalize(orig.c_str(),orig.c_str()+orig.size(),type,l)==normal); }
void TokenizerTest() { const std::basic_string<utf16_t> xmlText = Transcoder::UTF8toUTF16("<token1><token2 />\nhello world.\nhello xml.</token1>"); XMLParser<utf16_t>::Tokenizer tokenizer(xmlText.c_str(), xmlText.c_str() + xmlText.size()); std::basic_string<utf16_t> result = tokenizer.getToken(); CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result), result == Transcoder::UTF8toUTF16("<token1>")); result = tokenizer.getToken(); CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result), result == Transcoder::UTF8toUTF16("<token2 />")); result = tokenizer.getToken(); CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result), result == Transcoder::UTF8toUTF16("\nhello world.\nhello xml.")); result = tokenizer.getToken(); CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result), result == Transcoder::UTF8toUTF16("</token1>")); CPPUNIT_ASSERT(tokenizer.isEof()); result = tokenizer.getToken(); CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result), result == Transcoder::UTF8toUTF16("")); CPPUNIT_ASSERT(tokenizer.isEof()); }
void CPwQualityEst::_EnsureInitialized() { if(m_vCharTypes.size() == 0) { std::basic_string<WCHAR> strSpecial(PDCS_PRINTASCIISPEC); if(strSpecial.find(L' ') != strSpecial.npos) { ASSERT(FALSE); } else strSpecial += L' '; const std::basic_string<WCHAR> strHigh = PwCharSet::GetHighAnsiChars().ToString(); m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>( new CQeCharType(QE_PAT_LOWERALPHA, PDCS_LOWER_CASE, true))); m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>( new CQeCharType(QE_PAT_UPPERALPHA, PDCS_UPPER_CASE, true))); m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>( new CQeCharType(QE_PAT_DIGIT, PDCS_NUMERIC, true))); m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>( new CQeCharType(QE_PAT_SPECIAL, strSpecial.c_str(), false))); m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>( new CQeCharType(QE_PAT_HIGH, strHigh.c_str(), false))); m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>( new CQeCharType(QE_PAT_OTHER, 0x10000U - (2U * 26U) - 10U - strSpecial.size() - strHigh.size()))); } }
static inline void split(_OutputIter iter, const std::basic_string<_CharT, _Traits, _Alloc>& s, int nsplits = -1) { std::locale loc; typedef std::basic_string<_CharT, _Traits, _Alloc> str_t; int x = 0, rx = 0; int y = (int)s.size() - 1; while (std::isspace(s[x], loc)) x++; rx = x; while (y >= 0 && std::isspace(s[y], loc)) y--; y++; while (x < y) { if (std::isspace(s[x], loc)) { *iter++ = str_t(s, rx, x - rx); while (x < y && std::isspace(s[x], loc)) x++; rx = x; if (!--nsplits) { *iter++ = str_t(s, rx); return; } } else { x++; } } if (rx != y) { *iter++ = str_t(s, rx, y - rx); } }
std::vector<std::basic_string<T_Char>> split( const std::basic_string<T_Char>& src, const std::basic_string<T_Char>& delimit) { std::vector<std::basic_string<T_Char>> array; typedef typename std::basic_string<T_Char>::size_type size_type; typedef typename std::basic_string<T_Char>::const_iterator const_iterator; std::basic_string<T_Char> tmp; if (src.empty()) return array; if (delimit.empty()) { array.reserve(array.size() + src.size()); for (const_iterator it = src.begin(); it != src.end(); ++it) array.push_back(std::basic_string<T_Char>(1, *it)); return array; } size_type src_pos = 0; while (src.begin() + src_pos != src.end()) { size_type fnd_pos = src.find(delimit, src_pos); if (fnd_pos == std::basic_string<T_Char>::npos) { array.push_back(std::basic_string<T_Char>(src, src_pos)); break; } array.push_back( std::basic_string<T_Char>(src, src_pos, fnd_pos - src_pos)); src_pos = fnd_pos + delimit.length(); } return array; }
Ret benchmark1(const std::basic_string<C>& str, const std::basic_string<C>& key, F f) { const int N = 1; int val = 0; f.set(str, key); Xbyak::util::Clock clk; for (int i = 0; i < N; i++) { typename F::type p = f.begin(); typename F::type end = f.end(); for (;;) { clk.begin(); typename F::type q = f.find(p); clk.end(); if (q == end) break; val += (int)(q - p); p = q + 1; } } if (val == 0) val = (int)(str.size()) * N; Ret ret; ret.val = val; ret.clk = clk.getClock() / (double)val; return ret; }
testbuf(const std::basic_string<CharT>& str) : str_(str) { base::setg(const_cast<CharT*>(str_.data()), const_cast<CharT*>(str_.data()), const_cast<CharT*>(str_.data() + str_.size())); }
std::basic_string<Ch> encode_char_entities(const std::basic_string<Ch> &s) { // Don't do anything for empty strings. if(s.empty()) return s; typedef typename std::basic_string<Ch> Str; Str r; // To properly round-trip spaces and not uglify the XML beyond // recognition, we have to encode them IF the text contains only spaces. Str sp(1, Ch(' ')); if(s.find_first_not_of(sp) == Str::npos) { // The first will suffice. r = detail::widen<Ch>(" "); r += Str(s.size() - 1, Ch(' ')); } else { typename Str::const_iterator end = s.end(); for (typename Str::const_iterator it = s.begin(); it != end; ++it) { switch (*it) { case Ch('<'): r += detail::widen<Ch>("<"); break; case Ch('>'): r += detail::widen<Ch>(">"); break; case Ch('&'): r += detail::widen<Ch>("&"); break; case Ch('"'): r += detail::widen<Ch>("""); break; case Ch('\''): r += detail::widen<Ch>("'"); break; default: r += *it; break; } } } return r; }
void split(charT pat, const std::basic_string<charT, traits, Alloc>& s, Iter out) { unsigned int pos = 0; for (unsigned int i = 0; i < s.size(); ++i) { if (s[i] == pat) { if (i - pos > 0) { *(out++) = std::basic_string<charT, traits, Alloc>(s, pos, i - pos); } pos = i + 1; } } *(out++) = std::basic_string<charT, traits, Alloc>(s, pos, s.size() - pos); } // split
inline std::basic_string<typename re_detail::regex_iterator_traits<RandomAccessIterator>::value_type, traits, Allocator> operator + (const sub_match<RandomAccessIterator>& m, const std::basic_string<typename re_detail::regex_iterator_traits<RandomAccessIterator>::value_type, traits, Allocator>& s) { std::basic_string<typename re_detail::regex_iterator_traits<RandomAccessIterator>::value_type, traits, Allocator> result; result.reserve(s.size() + m.length() + 1); return result.append(m.first, m.second).append(s); }
HGLOBAL CreateGlobalData(const std::basic_string<charT>& str) { HGLOBAL data = ::GlobalAlloc(GMEM_MOVEABLE, ((str.size() + 1) * sizeof(charT))); if (data) { charT* raw_data = static_cast<charT*>(::GlobalLock(data)); if (!raw_data) { ::GlobalUnlock(data); return nullptr; } memcpy(raw_data, str.data(), str.size() * sizeof(charT)); raw_data[str.size()] = '\0'; ::GlobalUnlock(data); } return data; }
inline std::string u16tos(const std::basic_string<MIE_CHAR16>& str) { std::string ret; for (size_t i = 0; i < str.size(); i++) { ret += (char)str[i]; } return ret; }
static bool matches( std::basic_string< CharT, StringTraitsT, AllocatorT > const& str, boost::basic_regex< CharT, ReTraitsT > const& expr, boost::regex_constants::match_flag_type flags = boost::regex_constants::match_default) { const CharT* p = str.c_str(); return boost::regex_match(p, p + str.size(), expr, flags); }
int main() { std::locale l = std::locale::classic(); const F& f = std::use_facet<F>(l); { const std::basic_string<F::intern_type> from(L"some text"); std::vector<char> to(from.size()+1); std::mbstate_t mbs = {}; const F::intern_type* from_next = 0; char* to_next = 0; F::result r = f.out(mbs, from.data(), from.data() + from.size(), from_next, to.data(), to.data() + to.size(), to_next); assert(r == F::ok); assert(static_cast<std::size_t>(from_next - from.data()) == from.size()); assert(static_cast<std::size_t>(to_next - to.data()) == from.size()); assert(to.data() == std::string("some text")); } { std::basic_string<F::intern_type> from(L"some text"); from[4] = '\0'; std::vector<char> to(from.size()+1); std::mbstate_t mbs = {}; const F::intern_type* from_next = 0; char* to_next = 0; F::result r = f.out(mbs, from.data(), from.data() + from.size(), from_next, to.data(), to.data() + to.size(), to_next); assert(r == F::ok); assert(static_cast<std::size_t>(from_next - from.data()) == from.size()); assert(static_cast<std::size_t>(to_next - to.data()) == from.size()); assert(memcmp(to.data(), "some\0text", from.size()) == 0); } { std::basic_string<F::intern_type> from(L"some text"); std::vector<char> to(from.size()-1); std::mbstate_t mbs = {}; const F::intern_type* from_next = 0; char* to_next = 0; F::result r = f.out(mbs, from.data(), from.data() + from.size(), from_next, to.data(), to.data() + to.size()-1, to_next); assert(r == F::partial); assert(static_cast<std::size_t>(from_next - from.data()) == to.size()-1); assert(static_cast<std::size_t>(to_next - to.data()) == to.size()-1); assert(to.data() == std::string("some te")); } }
cstr(const std::basic_string<char_type, CharTraits, Allocator>& s) { if (s.empty()) { clear(); } else { buf_ = s.c_str(); size_ = s.size(); } }
int main() { std::locale l = std::locale::classic(); const std::basic_string<F::extern_type> from("some text"); const std::basic_string<F::intern_type> expected(from.begin(), from.end()); std::basic_string<F::intern_type> to(from.size(), F::intern_type()); const F& f = std::use_facet<F>(l); std::mbstate_t mbs = {}; const F::extern_type* from_next = 0; F::intern_type* to_next = 0; F::result r = f.in(mbs, from.data(), from.data() + from.size(), from_next, &to[0], &to[0] + to.size(), to_next); assert(r == F::ok); assert(from_next - from.data() == from.size()); assert(to_next - to.data() == expected.size()); assert(to_next - to.data() == expected.size()); assert(to == expected); }
static inline std::basic_string<_CharT, _Traits, _Alloc> rstrip( const std::basic_string<_CharT, _Traits, _Alloc> &a) { typedef std::basic_string<_CharT, _Traits, _Alloc> str_t; typename str_t::size_type p = a.size(); std::locale loc; while (p && std::isspace(a[p-1], loc)) p--; if (!p) return ""; return a.substr(0, p); }
inline int string_compare(const std::basic_string<C,T,A>& s, const C* p) { if(0 == *p) { if(s.empty() || ((s.size() == 1) && (s[0] == 0))) return 0; } return s.compare(p); }
std::basic_string<charT, traits, Alloc> remove_space(const std::basic_string<charT, traits, Alloc>& s, const std::locale& loc = std::locale::classic()) { std::basic_string<charT, traits, Alloc> ns; for (std::size_t i = 0; i < s.size(); ++i) { if (std::isspace(s[i], loc) == false) ns.push_back(s[i]); } return ns; } // remove_space
int operator()(const std::basic_string<charT, traits, Alloc>& s1, const std::basic_string<charT, traits, Alloc>& s2) const { std::size_t l1 = s1.size(); std::size_t l2 = s2.size(); if (l1 < l2) return -1; else if (l2 < l1) return 1; charT c1, c2; for (std::size_t i = 0; i < l1; ++i) { c1 = ct_.toupper(s1[i]); c2 = ct_.toupper(s2[i]); if (c1 < c2) return -1; else if (c2 < c1) return 1; } return 0; } // operator()
typename boost::disable_if< std::is_same<typename std::decay<Sink>::type::element_type, std::basic_string<Element>>, typename error_type<Sink>::type>::type append(Sink &&out, std::basic_string<Element> const &str) { return out.append( make_iterator_range(str.data(), str.data() + str.size())); }
OutputIterator regex_format(OutputIterator out, const match_results<Iterator>& m, const std::basic_string<charT>& fmt, match_flag_type flags = format_all ) { re_detail::trivial_format_traits<charT> traits; return re_detail::regex_format_imp(out, m, fmt.data(), fmt.data() + fmt.size(), flags, traits); }
std::basic_string< wchar_t > str_convert< wchar_t, char >( std::basic_string< char > const & src ) { std::basic_string< wchar_t > dst; if ( !src.empty() ) { try { std::unique_lock< std::mutex > lock( g_conversionMutex ); char * szloc = setlocale( LC_CTYPE, "" ); mbtowc( NULL, NULL, 0 ); size_t max = src.size(); int length = 1; const char * in = src.c_str(); dst.reserve( src.size() ); while ( max > 0 && length >= 1 ) { wchar_t wc; length = mbtowc( &wc, in, max ); if ( length >= 1 ) { dst += wc; max -= length; in += length; } } setlocale( LC_CTYPE, szloc ); } catch ( std::exception & exc ) { CLogger::LogError( StringStream() << ERROR_DB_CONVERSION << exc.what() ); } catch ( ... ) { CLogger::LogError( StringStream() << ERROR_DB_CONVERSION << INFO_UNKNOWN ); } } return std::move( dst ); }
std::basic_string<charT> regex_format(const match_results<Iterator>& m, const std::basic_string<charT>& fmt, match_flag_type flags = format_all) { std::basic_string<charT> result; re_detail::string_out_iterator<std::basic_string<charT> > i(result); re_detail::trivial_format_traits<charT> traits; re_detail::regex_format_imp(i, m, fmt.data(), fmt.data() + fmt.size(), flags, traits); return result; }
inline void WriteString(Process const& process, PVOID address, std::basic_string<T, Traits, Alloc> const& data) { HADESMEM_DETAIL_STATIC_ASSERT(detail::IsCharType<T>::value); HADESMEM_DETAIL_ASSERT(address != nullptr); return Write(process, address, data.c_str(), data.size() + 1); }
auto GenRString(const RDI& rdi, size_t len, const std::basic_string<C>& src) { std::basic_string<C> result; result.resize(len); auto srcLen = src.size(); for(size_t i=0 ; i<len ; i++) { int cn = rdi({0, srcLen-1}); result[i] = src[cn]; } return result; }
static std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input) { std::string result; iconv_t const conv_desc = iconv_open("UTF-8", fromcode); if ((iconv_t)(-1) == conv_desc) { LOG_ERROR(Common, "Iconv initialization failure [%s]: %s", fromcode, strerror(errno)); iconv_close(conv_desc); return {}; } const size_t in_bytes = sizeof(T) * input.size(); // Multiply by 4, which is the max number of bytes to encode a codepoint const size_t out_buffer_size = 4 * in_bytes; std::string out_buffer; out_buffer.resize(out_buffer_size); auto src_buffer = &input[0]; size_t src_bytes = in_bytes; auto dst_buffer = &out_buffer[0]; size_t dst_bytes = out_buffer.size(); while (0 != src_bytes) { size_t const iconv_result = iconv(conv_desc, (char**)(&src_buffer), &src_bytes, &dst_buffer, &dst_bytes); if (static_cast<size_t>(-1) == iconv_result) { if (EILSEQ == errno || EINVAL == errno) { // Try to skip the bad character if (0 != src_bytes) { --src_bytes; ++src_buffer; } } else { LOG_ERROR(Common, "iconv failure [%s]: %s", fromcode, strerror(errno)); break; } } } out_buffer.resize(out_buffer_size - dst_bytes); out_buffer.swap(result); iconv_close(conv_desc); return result; }
void test_from_neg(std::basic_string<Char> source,std::string target,std::string encoding) { using namespace boost::locale::conv; boost::locale::generator g; std::locale l=g("en_US."+encoding); TEST(from_utf<Char>(source,encoding)==target); TEST(from_utf<Char>(source.c_str(),encoding)==target); TEST(from_utf<Char>(source.c_str(),source.c_str()+source.size(),encoding)==target); TEST(from_utf<Char>(source,l)==target); TEST(from_utf<Char>(source.c_str(),l)==target); TEST(from_utf<Char>(source.c_str(),source.c_str()+source.size(),l)==target); TESTF(from_utf<Char>(source,encoding,stop)); TESTF(from_utf<Char>(source.c_str(),encoding,stop)); TESTF(from_utf<Char>(source.c_str(),source.c_str()+source.size(),encoding,stop)); TESTF(from_utf<Char>(source,l,stop)); TESTF(from_utf<Char>(source.c_str(),l,stop)); TESTF(from_utf<Char>(source.c_str(),source.c_str()+source.size(),l,stop)); }