Пример #1
0
  std::pair<iterator, bool> insert_impl(trie_node_type& __node, const std::basic_string<T>& __word, size_t __at) {
    if (__at == __word.size()) {
        ++__node._M_words;
        ++__node._M_prefix;
        return std::make_pair(trie_iterator<trie_node_type>(__word, &__node), false);
    }

    ++__node._M_prefix;

    trie_node_type *_new_node = nullptr;
    bool _new_insert = false;
    auto _it = __node._M_children.find(__word[__at]);
    if (_it == __node._M_children.end()) {
        _new_node = new trie_node_type();
        _new_insert = true;
        __node._M_children[__word[__at]].reset(_new_node);
    }
    else {
        _new_node = _it->second.get();
    }

    auto _ret = insert_impl(*_new_node, __word, ++__at);
    _ret.second |= _new_insert;
    return _ret;
  }
Пример #2
0
	inline void kbe_split(const std::basic_string<T>& s, T c, std::vector< std::basic_string<T> > &v)
	{
		if(s.size() == 0)
			return;

		typename std::basic_string< T >::size_type i = 0;
		typename std::basic_string< T >::size_type j = s.find(c);

		while(j != std::basic_string<T>::npos)
		{
			std::basic_string<T> buf = s.substr(i, j - i);

			if(buf.size() > 0)
				v.push_back(buf);

			i = ++j; j = s.find(c, j);
		}

		if(j == std::basic_string<T>::npos)
		{
			std::basic_string<T> buf = s.substr(i, s.length() - i);
			if(buf.size() > 0)
				v.push_back(buf);
		}
	}
Пример #3
0
void test_normc(std::basic_string<Char> orig,std::basic_string<Char> normal,boost::locale::norm_type type)
{
    std::locale l = boost::locale::generator().generate("en_US.UTF-8");
    TEST(normalize(orig,type,l)==normal);
    TEST(normalize(orig.c_str(),type,l)==normal);
    TEST(normalize(orig.c_str(),orig.c_str()+orig.size(),type,l)==normal);
}
Пример #4
0
	void TokenizerTest()
	{
		const std::basic_string<utf16_t> xmlText =
			Transcoder::UTF8toUTF16("<token1><token2 />\nhello world.\nhello xml.</token1>");
		XMLParser<utf16_t>::Tokenizer tokenizer(xmlText.c_str(), xmlText.c_str() + xmlText.size());

		std::basic_string<utf16_t> result = tokenizer.getToken();
		CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result),
							   result == Transcoder::UTF8toUTF16("<token1>"));

		result = tokenizer.getToken();
		CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result),
							   result == Transcoder::UTF8toUTF16("<token2 />"));

		result = tokenizer.getToken();
		CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result),
							   result == Transcoder::UTF8toUTF16("\nhello world.\nhello xml."));

		result = tokenizer.getToken();
		CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result),
							   result == Transcoder::UTF8toUTF16("</token1>"));

		CPPUNIT_ASSERT(tokenizer.isEof());

		result = tokenizer.getToken();
		CPPUNIT_ASSERT_MESSAGE(Transcoder::UTF16toUTF8(result),
							   result == Transcoder::UTF8toUTF16(""));

		CPPUNIT_ASSERT(tokenizer.isEof());
	}
Пример #5
0
void CPwQualityEst::_EnsureInitialized()
{
	if(m_vCharTypes.size() == 0)
	{
		std::basic_string<WCHAR> strSpecial(PDCS_PRINTASCIISPEC);
		if(strSpecial.find(L' ') != strSpecial.npos) { ASSERT(FALSE); }
		else strSpecial += L' ';

		const std::basic_string<WCHAR> strHigh =
			PwCharSet::GetHighAnsiChars().ToString();

		m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>(
			new CQeCharType(QE_PAT_LOWERALPHA, PDCS_LOWER_CASE, true)));
		m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>(
			new CQeCharType(QE_PAT_UPPERALPHA, PDCS_UPPER_CASE, true)));
		m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>(
			new CQeCharType(QE_PAT_DIGIT, PDCS_NUMERIC, true)));
		m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>(
			new CQeCharType(QE_PAT_SPECIAL, strSpecial.c_str(), false)));
		m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>(
			new CQeCharType(QE_PAT_HIGH, strHigh.c_str(), false)));
		m_vCharTypes.push_back(boost::shared_ptr<CQeCharType>(
			new CQeCharType(QE_PAT_OTHER, 0x10000U - (2U * 26U) - 10U -
			strSpecial.size() - strHigh.size())));
	}
}
Пример #6
0
static inline void split(_OutputIter iter,
                         const std::basic_string<_CharT, _Traits, _Alloc>& s,
                         int nsplits = -1) {
  std::locale loc;
  typedef std::basic_string<_CharT, _Traits, _Alloc> str_t;

  int x = 0, rx = 0;

  int y = (int)s.size() - 1;
  while (std::isspace(s[x], loc))
    x++;
  rx = x;
  while (y >= 0 && std::isspace(s[y], loc))
    y--;
  y++;

  while (x < y) {
    if (std::isspace(s[x], loc)) {
      *iter++ = str_t(s, rx, x - rx);
      while (x < y && std::isspace(s[x], loc))
        x++;
      rx = x;
      if (!--nsplits) {
        *iter++ = str_t(s, rx);
        return;
      }
    } else {
      x++;
    }
  }
  if (rx != y) {
    *iter++ = str_t(s, rx, y - rx);
  }
}
Пример #7
0
std::vector<std::basic_string<T_Char>> split(
                                        const std::basic_string<T_Char>& src,
const std::basic_string<T_Char>& delimit) {
    std::vector<std::basic_string<T_Char>> array;
    typedef typename std::basic_string<T_Char>::size_type size_type;
    typedef typename std::basic_string<T_Char>::const_iterator const_iterator;
    std::basic_string<T_Char> tmp;

    if (src.empty())
        return array;

    if (delimit.empty()) {
        array.reserve(array.size() + src.size());
        for (const_iterator it = src.begin(); it != src.end(); ++it)
            array.push_back(std::basic_string<T_Char>(1, *it));
        return array;
    }

    size_type src_pos = 0;
    while (src.begin() + src_pos != src.end()) {
        size_type fnd_pos = src.find(delimit, src_pos);
        if (fnd_pos == std::basic_string<T_Char>::npos) {
            array.push_back(std::basic_string<T_Char>(src, src_pos));
            break;
        }
        array.push_back(
            std::basic_string<T_Char>(src, src_pos, fnd_pos - src_pos));
        src_pos = fnd_pos + delimit.length();
    }
    return array;
}
Пример #8
0
Ret benchmark1(const std::basic_string<C>& str, const std::basic_string<C>& key, F f)
{
	const int N = 1;
	int val = 0;
	f.set(str, key);
	Xbyak::util::Clock clk;

	for (int i = 0; i < N; i++) {
		typename F::type p = f.begin();
		typename F::type end = f.end();
		for (;;) {
			clk.begin();
			typename F::type q = f.find(p);
			clk.end();

			if (q == end) break;
			val += (int)(q - p);
			p = q + 1;
		}
	}
 if (val == 0) val = (int)(str.size()) * N;
	Ret ret;
	ret.val = val;
	ret.clk = clk.getClock() / (double)val;
	return ret;
}
Пример #9
0
 testbuf(const std::basic_string<CharT>& str)
     : str_(str)
 {
     base::setg(const_cast<CharT*>(str_.data()),
                const_cast<CharT*>(str_.data()),
                const_cast<CharT*>(str_.data() + str_.size()));
 }
Пример #10
0
    std::basic_string<Ch> encode_char_entities(const std::basic_string<Ch> &s)
    {
        // Don't do anything for empty strings.
        if(s.empty()) return s;

        typedef typename std::basic_string<Ch> Str;
        Str r;
        // To properly round-trip spaces and not uglify the XML beyond
        // recognition, we have to encode them IF the text contains only spaces.
        Str sp(1, Ch(' '));
        if(s.find_first_not_of(sp) == Str::npos) {
            // The first will suffice.
            r = detail::widen<Ch>("&#32;");
            r += Str(s.size() - 1, Ch(' '));
        } else {
            typename Str::const_iterator end = s.end();
            for (typename Str::const_iterator it = s.begin(); it != end; ++it)
            {
                switch (*it)
                {
                    case Ch('<'): r += detail::widen<Ch>("&lt;"); break;
                    case Ch('>'): r += detail::widen<Ch>("&gt;"); break;
                    case Ch('&'): r += detail::widen<Ch>("&amp;"); break;
                    case Ch('"'): r += detail::widen<Ch>("&quot;"); break;
                    case Ch('\''): r += detail::widen<Ch>("&apos;"); break;
                    default: r += *it; break;
                }
            }
        }
        return r;
    }
Пример #11
0
  void split(charT pat,
	     const std::basic_string<charT, traits, Alloc>& s, Iter out) {
      unsigned int pos = 0;

      for (unsigned int i = 0; i < s.size(); ++i) {
	  if (s[i] == pat) {
	      if (i - pos > 0) {
		  *(out++) =
		      std::basic_string<charT, traits, Alloc>(s, pos, i - pos);
	      }
	      pos = i + 1;
	  }
      }

      *(out++) =
	  std::basic_string<charT, traits, Alloc>(s, pos, s.size() - pos);
  } // split
Пример #12
0
inline std::basic_string<typename re_detail::regex_iterator_traits<RandomAccessIterator>::value_type, traits, Allocator> 
operator + (const sub_match<RandomAccessIterator>& m,
            const std::basic_string<typename re_detail::regex_iterator_traits<RandomAccessIterator>::value_type, traits, Allocator>& s)
{
   std::basic_string<typename re_detail::regex_iterator_traits<RandomAccessIterator>::value_type, traits, Allocator> result;
   result.reserve(s.size() + m.length() + 1);
   return result.append(m.first, m.second).append(s);
}
Пример #13
0
HGLOBAL CreateGlobalData(const std::basic_string<charT>& str)
{
	HGLOBAL data = ::GlobalAlloc(GMEM_MOVEABLE, ((str.size() + 1) * sizeof(charT)));
	if (data) 
	{
		charT* raw_data = static_cast<charT*>(::GlobalLock(data));
		if (!raw_data)
		{
			::GlobalUnlock(data);
			return nullptr;
		}
		memcpy(raw_data, str.data(), str.size() * sizeof(charT));
		raw_data[str.size()] = '\0';
		::GlobalUnlock(data);
	}
	return data;
}
Пример #14
0
inline std::string u16tos(const std::basic_string<MIE_CHAR16>& str)
{
	std::string ret;
	for (size_t i = 0; i < str.size(); i++) {
		ret += (char)str[i];
	}
	return ret;
}
Пример #15
0
 static bool matches(
     std::basic_string< CharT, StringTraitsT, AllocatorT > const& str,
     boost::basic_regex< CharT, ReTraitsT > const& expr,
     boost::regex_constants::match_flag_type flags = boost::regex_constants::match_default)
 {
     const CharT* p = str.c_str();
     return boost::regex_match(p, p + str.size(), expr, flags);
 }
Пример #16
0
int main()
{
    std::locale l = std::locale::classic();
    const F& f = std::use_facet<F>(l);
    {
        const std::basic_string<F::intern_type> from(L"some text");
        std::vector<char> to(from.size()+1);
        std::mbstate_t mbs = {};
        const F::intern_type* from_next = 0;
        char* to_next = 0;
        F::result r = f.out(mbs, from.data(), from.data() + from.size(), from_next,
                                 to.data(), to.data() + to.size(), to_next);
        assert(r == F::ok);
        assert(static_cast<std::size_t>(from_next - from.data()) == from.size());
        assert(static_cast<std::size_t>(to_next - to.data()) == from.size());
        assert(to.data() == std::string("some text"));
    }
    {
        std::basic_string<F::intern_type> from(L"some text");
        from[4] = '\0';
        std::vector<char> to(from.size()+1);
        std::mbstate_t mbs = {};
        const F::intern_type* from_next = 0;
        char* to_next = 0;
        F::result r = f.out(mbs, from.data(), from.data() + from.size(), from_next,
                                 to.data(), to.data() + to.size(), to_next);
        assert(r == F::ok);
        assert(static_cast<std::size_t>(from_next - from.data()) == from.size());
        assert(static_cast<std::size_t>(to_next - to.data()) == from.size());
        assert(memcmp(to.data(), "some\0text", from.size()) == 0);
    }
    {
        std::basic_string<F::intern_type> from(L"some text");
        std::vector<char> to(from.size()-1);
        std::mbstate_t mbs = {};
        const F::intern_type* from_next = 0;
        char* to_next = 0;
        F::result r = f.out(mbs, from.data(), from.data() + from.size(), from_next,
                                 to.data(), to.data() + to.size()-1, to_next);
        assert(r == F::partial);
        assert(static_cast<std::size_t>(from_next - from.data()) == to.size()-1);
        assert(static_cast<std::size_t>(to_next - to.data()) == to.size()-1);
        assert(to.data() == std::string("some te"));
    }
}
Пример #17
0
 cstr(const std::basic_string<char_type, CharTraits, Allocator>& s)
 {
     if (s.empty()) {
         clear();
     } else {
         buf_ = s.c_str();
         size_ = s.size();
     }
 }
Пример #18
0
int main()
{
    std::locale l = std::locale::classic();
    const std::basic_string<F::extern_type> from("some text");
    const std::basic_string<F::intern_type> expected(from.begin(), from.end());
    std::basic_string<F::intern_type> to(from.size(), F::intern_type());
    const F& f = std::use_facet<F>(l);
    std::mbstate_t mbs = {};
    const F::extern_type* from_next = 0;
    F::intern_type* to_next = 0;
    F::result r = f.in(mbs, from.data(), from.data() + from.size(), from_next,
                            &to[0], &to[0] + to.size(), to_next);
    assert(r == F::ok);
    assert(from_next - from.data() == from.size());
    assert(to_next - to.data() == expected.size());
    assert(to_next - to.data() == expected.size());
    assert(to == expected);
}
Пример #19
0
 static inline std::basic_string<_CharT, _Traits, _Alloc> rstrip(
     const std::basic_string<_CharT, _Traits, _Alloc> &a) {
   typedef std::basic_string<_CharT, _Traits, _Alloc> str_t;
   typename str_t::size_type p = a.size();
   std::locale loc;
   while (p && std::isspace(a[p-1], loc)) p--;
   if (!p) return "";
   return a.substr(0, p);
 }
inline int string_compare(const std::basic_string<C,T,A>& s, const C* p)
{ 
   if(0 == *p)
   {
      if(s.empty() || ((s.size() == 1) && (s[0] == 0)))
         return 0;
   }
   return s.compare(p); 
}
Пример #21
0
  std::basic_string<charT, traits, Alloc>
  remove_space(const std::basic_string<charT, traits, Alloc>& s,
	       const std::locale& loc = std::locale::classic()) {
      std::basic_string<charT, traits, Alloc> ns;
      for (std::size_t i = 0; i < s.size(); ++i) {
	  if (std::isspace(s[i], loc) == false) ns.push_back(s[i]);
      }
      return ns;
  } // remove_space
Пример #22
0
      int operator()(const std::basic_string<charT, traits, Alloc>& s1,
		     const std::basic_string<charT, traits, Alloc>& s2) const {

	  std::size_t l1 = s1.size();
	  std::size_t l2 = s2.size();

	  if (l1 < l2) return -1; else if (l2 < l1) return 1;

	  charT c1, c2;

	  for (std::size_t i = 0; i < l1; ++i) {
	      c1 = ct_.toupper(s1[i]);
	      c2 = ct_.toupper(s2[i]);
	      if (c1 < c2) return -1; else if (c2 < c1) return 1;
	  }

	  return 0;
      } // operator()
Пример #23
0
 typename boost::disable_if<
     std::is_same<typename std::decay<Sink>::type::element_type,
                  std::basic_string<Element>>,
     typename error_type<Sink>::type>::type
 append(Sink &&out, std::basic_string<Element> const &str)
 {
     return out.append(
         make_iterator_range(str.data(), str.data() + str.size()));
 }
Пример #24
0
OutputIterator regex_format(OutputIterator out,
                          const match_results<Iterator>& m,
                          const std::basic_string<charT>& fmt,
                          match_flag_type flags = format_all
                         )
{
   re_detail::trivial_format_traits<charT> traits;
   return re_detail::regex_format_imp(out, m, fmt.data(), fmt.data() + fmt.size(), flags, traits);
}  
    std::basic_string< wchar_t > str_convert< wchar_t, char >( std::basic_string< char > const & src )
    {
        std::basic_string< wchar_t > dst;

        if ( !src.empty() )
        {
            try
            {
                std::unique_lock< std::mutex > lock( g_conversionMutex );
                char * szloc = setlocale( LC_CTYPE, "" );
                mbtowc( NULL, NULL, 0 );
                size_t max = src.size();
                int length = 1;
                const char * in = src.c_str();
                dst.reserve( src.size() );

                while ( max > 0 && length >= 1 )
                {
                    wchar_t wc;
                    length = mbtowc( &wc, in, max );

                    if ( length >= 1 )
                    {
                        dst += wc;
                        max -= length;
                        in += length;
                    }
                }

                setlocale( LC_CTYPE, szloc );
            }
            catch ( std::exception & exc )
            {
                CLogger::LogError( StringStream() << ERROR_DB_CONVERSION << exc.what() );
            }
            catch ( ... )
            {
                CLogger::LogError( StringStream() << ERROR_DB_CONVERSION << INFO_UNKNOWN );
            }
        }

        return std::move( dst );
    }
Пример #26
0
std::basic_string<charT> regex_format(const match_results<Iterator>& m, 
                                      const std::basic_string<charT>& fmt, 
                                      match_flag_type flags = format_all)
{
   std::basic_string<charT> result;
   re_detail::string_out_iterator<std::basic_string<charT> > i(result);
   re_detail::trivial_format_traits<charT> traits;
   re_detail::regex_format_imp(i, m, fmt.data(), fmt.data() + fmt.size(), flags, traits);
   return result;
}
Пример #27
0
inline void WriteString(Process const& process,
                        PVOID address,
                        std::basic_string<T, Traits, Alloc> const& data)
{
  HADESMEM_DETAIL_STATIC_ASSERT(detail::IsCharType<T>::value);

  HADESMEM_DETAIL_ASSERT(address != nullptr);

  return Write(process, address, data.c_str(), data.size() + 1);
}
Пример #28
0
		auto GenRString(const RDI& rdi, size_t len, const std::basic_string<C>& src) {
			std::basic_string<C> result;
			result.resize(len);
			auto srcLen = src.size();
			for(size_t i=0 ; i<len ; i++) {
				int cn = rdi({0, srcLen-1});
				result[i] = src[cn];
			}
			return result;
		}
Пример #29
0
static std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
{
    std::string result;

    iconv_t const conv_desc = iconv_open("UTF-8", fromcode);
    if ((iconv_t)(-1) == conv_desc)
    {
        LOG_ERROR(Common, "Iconv initialization failure [%s]: %s", fromcode, strerror(errno));
        iconv_close(conv_desc);
        return {};
    }

    const size_t in_bytes = sizeof(T) * input.size();
    // Multiply by 4, which is the max number of bytes to encode a codepoint
    const size_t out_buffer_size = 4 * in_bytes;

    std::string out_buffer;
    out_buffer.resize(out_buffer_size);

    auto src_buffer = &input[0];
    size_t src_bytes = in_bytes;
    auto dst_buffer = &out_buffer[0];
    size_t dst_bytes = out_buffer.size();

    while (0 != src_bytes)
    {
        size_t const iconv_result = iconv(conv_desc, (char**)(&src_buffer), &src_bytes,
            &dst_buffer, &dst_bytes);

        if (static_cast<size_t>(-1) == iconv_result)
        {
            if (EILSEQ == errno || EINVAL == errno)
            {
                // Try to skip the bad character
                if (0 != src_bytes)
                {
                    --src_bytes;
                    ++src_buffer;
                }
            }
            else
            {
                LOG_ERROR(Common, "iconv failure [%s]: %s", fromcode, strerror(errno));
                break;
            }
        }
    }

    out_buffer.resize(out_buffer_size - dst_bytes);
    out_buffer.swap(result);

    iconv_close(conv_desc);

    return result;
}
Пример #30
0
void test_from_neg(std::basic_string<Char> source,std::string target,std::string encoding)
{
    using namespace boost::locale::conv;
    boost::locale::generator g;
    std::locale l=g("en_US."+encoding);

    TEST(from_utf<Char>(source,encoding)==target);
    TEST(from_utf<Char>(source.c_str(),encoding)==target);
    TEST(from_utf<Char>(source.c_str(),source.c_str()+source.size(),encoding)==target);
    TEST(from_utf<Char>(source,l)==target);
    TEST(from_utf<Char>(source.c_str(),l)==target);
    TEST(from_utf<Char>(source.c_str(),source.c_str()+source.size(),l)==target);

    TESTF(from_utf<Char>(source,encoding,stop));
    TESTF(from_utf<Char>(source.c_str(),encoding,stop));
    TESTF(from_utf<Char>(source.c_str(),source.c_str()+source.size(),encoding,stop));
    TESTF(from_utf<Char>(source,l,stop));
    TESTF(from_utf<Char>(source.c_str(),l,stop));
    TESTF(from_utf<Char>(source.c_str(),source.c_str()+source.size(),l,stop));
}