Exemplo n.º 1
0
 void print_if_failed(char const* func, bool result
   , std::basic_string<Char> const& generated, T const& expected)
 {
     if (!result)
         std::cerr << "in " << func << ": result is false" << std::endl;
     else if (generated != expected)
         std::cerr << "in " << func << ": generated \""
             << std::string(generated.begin(), generated.end())
             << "\"" << std::endl;
 }
Exemplo n.º 2
0
std::basic_string<charT> regex_replace(const std::basic_string<charT>& s,
                         const basic_regex<charT, traits>& e, 
                         Formatter fmt,
                         match_flag_type flags = match_default)
{
   std::basic_string<charT> result;
   re_detail::string_out_iterator<std::basic_string<charT> > i(result);
   regex_replace(i, s.begin(), s.end(), e, fmt, flags);
   return result;
}
template< class CharT > bool IsPrintable( const std::basic_string< CharT >& str )
{
    const std::ctype< CharT > *pct = 0, &ct = tss::GetFacet( std::locale(), pct );
    for( std::basic_string< CharT >::const_iterator at = str.begin(); at != str.end(); at++ )
    {           
        if( ! ct.is( std::ctype_base::print, *at ) ) // if not printable
            return false;
    }

    return true;
}
Exemplo n.º 4
0
std::basic_string <CharType, TraitsType, AllocType>
lowercase( const std::basic_string <CharType, TraitsType, AllocType> & s )
{
    std::basic_string <CharType, TraitsType, AllocType> result( s.length(), '\0' );
    std::transform(
        s.begin(),
        s.end(),
        result.begin(),
        std::ptr_fun <int, int> ( std::tolower )
    );
    return result;
}
Exemplo n.º 5
0
int countUnique(const std::basic_string<T>& s) {
   using std::basic_string;

   basic_string<T> chars;

   for (typename basic_string<T>::const_iterator p = s.begin( );
        p != s.end( ); ++p) {
     if (chars.find(*p) == basic_string<T>::npos)
        chars += *p;
   }
   return(chars.length( ));
}
Exemplo n.º 6
0
inline std::basic_string<Char> regex_replace
(
    std::basic_string<Char> const &str
  , basic_regex<typename std::basic_string<Char>::const_iterator> const &re
  , std::basic_string<Char> const &fmt
  , regex_constants::match_flag_type flags = regex_constants::match_default
)
{
    std::basic_string<Char> result;
    result.reserve(fmt.length() * 2);
    regex_replace(std::back_inserter(result), str.begin(), str.end(), re, fmt, flags);
    return result;
}
Exemplo n.º 7
0
		std::basic_string < Elem, Traits > cell_encode(std::basic_string < Elem, Traits > Str, Elem Sep_, Elem Esc){
			if(Str.find(Sep_) < Str.size() || Str.find(Esc) < Str.size()){
				for(auto itr = Str.begin(); itr != Str.end(); ++itr){
					if(*itr == Esc){
						itr = Str.insert(++itr, Esc);
					}
				}

				Str.insert(Str.begin(), Esc);
				Str.push_back(Esc);
			}

			return std::move(Str);
		}
Exemplo n.º 8
0
void
test(const std::basic_string<CharT>& s)
{
    typedef std::basic_string<CharT> string;
    typedef std::sub_match<typename string::const_iterator> SM;
    typedef std::basic_ostringstream<CharT> ostringstream;
    SM sm;
    sm.first = s.begin();
    sm.second = s.end();
    sm.matched = true;
    ostringstream os;
    os << sm;
    assert(os.str() == s);
}
Exemplo n.º 9
0
 std::basic_string<Ch> trim(const std::basic_string<Ch> &s,
                            const std::locale &loc = std::locale())
 {
     typename std::basic_string<Ch>::const_iterator first = s.begin();
     typename std::basic_string<Ch>::const_iterator end = s.end();
     while (first != end && std::isspace(*first, loc))
         ++first;
     if (first == end)
         return std::basic_string<Ch>();
     typename std::basic_string<Ch>::const_iterator last = end;
     do --last; while (std::isspace(*last, loc));
     if (first != s.begin() || last + 1 != end)
         return std::basic_string<Ch>(first, last + 1);
     else
         return s;
 }
Exemplo n.º 10
0
 std::basic_string<Ch> encode_char_entities(const std::basic_string<Ch> &s)
 {
     typedef typename std::basic_string<Ch> Str;
     Str r;
     typename Str::const_iterator end = s.end();
     for (typename Str::const_iterator it = s.begin(); it != end; ++it)
     {
         switch (*it)
         {
             case Ch('<'): r += detail::widen<Ch>("&lt;"); break;
             case Ch('>'): r += detail::widen<Ch>("&gt;"); break;
             case Ch('&'): r += detail::widen<Ch>("&amp;"); break;
             default: r += *it; break;
         }
     }
     return r;
 }
Exemplo n.º 11
0
int main()
{
    std::locale l = std::locale::classic();
    const std::basic_string<F::extern_type> from("some text");
    const std::basic_string<F::intern_type> expected(from.begin(), from.end());
    std::basic_string<F::intern_type> to(from.size(), F::intern_type());
    const F& f = std::use_facet<F>(l);
    std::mbstate_t mbs = {};
    const F::extern_type* from_next = 0;
    F::intern_type* to_next = 0;
    F::result r = f.in(mbs, from.data(), from.data() + from.size(), from_next,
                            &to[0], &to[0] + to.size(), to_next);
    assert(r == F::ok);
    assert(from_next - from.data() == from.size());
    assert(to_next - to.data() == expected.size());
    assert(to_next - to.data() == expected.size());
    assert(to == expected);
}
Exemplo n.º 12
0
 std::basic_string<Ch> condense(const std::basic_string<Ch> &s)
 {
     std::basic_string<Ch> r;
     std::locale loc;
     bool space = false;
     typename std::basic_string<Ch>::const_iterator end = s.end();
     for (typename std::basic_string<Ch>::const_iterator it = s.begin();
          it != end; ++it)
     {
         if (isspace(*it, loc) || *it == Ch('\n'))
         {
             if (!space)
                 r += Ch(' '), space = true;
         }
         else
             r += *it, space = false;
     }
     return r;
 }
    std::basic_string<Ch> create_escapes(const std::basic_string<Ch> &s)
    {
        std::basic_string<Ch> result;
        typename std::basic_string<Ch>::const_iterator b = s.begin();
        typename std::basic_string<Ch>::const_iterator e = s.end();
        while (b != e)
        {
            // This assumes an ASCII superset. But so does everything in PTree.
            // We escape everything outside ASCII, because this code can't
            // handle high unicode characters.
            if (*b == 0x20 || *b == 0x21 || (*b >= 0x23 && *b <= 0x2E) ||
                (*b >= 0x30 && *b <= 0x5B) || (*b >= 0x5D && *b <= 0xFF) || 
				(*b >= -0x80 && *b < 0))  // PATCH!!! Patched by the "iCardClient" developing team, this will pass UTF-8 signed chars.
                result += *b;
            else if (*b == Ch('\b')) result += Ch('\\'), result += Ch('b');
            else if (*b == Ch('\f')) result += Ch('\\'), result += Ch('f');
            else if (*b == Ch('\n')) result += Ch('\\'), result += Ch('n');
            else if (*b == Ch('\r')) result += Ch('\\'), result += Ch('r');
            else if (*b == Ch('\t')) result += Ch('\\'), result += Ch('t');
            else if (*b == Ch('/')) result += Ch('\\'), result += Ch('/');
            else if (*b == Ch('"'))  result += Ch('\\'), result += Ch('"');
            else if (*b == Ch('\\')) result += Ch('\\'), result += Ch('\\');
            else
            {
                const char *hexdigits = "0123456789ABCDEF";
                typedef typename make_unsigned<Ch>::type UCh;
                unsigned long u = (std::min)(static_cast<unsigned long>(
                                                 static_cast<UCh>(*b)),
                                             0xFFFFul);
                int d1 = u / 4096; u -= d1 * 4096;
                int d2 = u / 256; u -= d2 * 256;
                int d3 = u / 16; u -= d3 * 16;
                int d4 = u;
                result += Ch('\\'); result += Ch('u');
                result += Ch(hexdigits[d1]); result += Ch(hexdigits[d2]);
                result += Ch(hexdigits[d3]); result += Ch(hexdigits[d4]);
				result += *b;
            }
            ++b;
        }
        return result;
    }
Exemplo n.º 14
0
    void IO::TrimLeft(std::basic_string<charType> & str, const char* chars2remove)
    {
        if (!str.empty())   //trim the characters in chars2remove from the left
        {
            std::string::size_type pos = 0;
            if (chars2remove != NULL)
            {
                pos = str.find_first_not_of(chars2remove);

                if (pos != std::string::npos)
                    str.erase(0,pos);
                else
                    str.erase( str.begin() , str.end() ); // make empty
            }
            else        //trim space
            {
                pos = std::string::npos;        //pos = -1
                for (size_t i = 0; i < str.size(); ++i)
                {
                    if (!isspace(str[i]))
                    {
                        pos = i;
                        break;
                    }
                }
                if (pos != std::string::npos)
                {
                    if (pos > 0)
                    {
                        size_t length = str.size() - pos;
                        for (size_t i = 0; i < length; ++i) str[i] = str[i+pos];
                        str.resize(length);
                    }
                }
                else
                {
                    str.clear();
                }
            }
        }
    }
Exemplo n.º 15
0
 std::vector<std::basic_string<charT> > 
 split_unix(
       const std::basic_string<charT>& cmdline, 
       const std::basic_string<charT>& seperator, 
       const std::basic_string<charT>& quote, 
       const std::basic_string<charT>& escape)
 {   
    typedef boost::tokenizer< boost::escaped_list_separator<charT>, 
          typename std::basic_string<charT>::const_iterator, 
          std::basic_string<charT> >  tokenizerT;
       
    tokenizerT tok(cmdline.begin(), cmdline.end(), 
              boost::escaped_list_separator< charT >(escape, seperator, quote));
       
    std::vector< std::basic_string<charT> > result;
    for (typename tokenizerT::iterator cur_token(tok.begin()), end_token(tok.end()); cur_token != end_token; ++cur_token) {
       if (!cur_token->empty())
          result.push_back(*cur_token);
    }
    return result;
 }
Exemplo n.º 16
0
std::string utf32_to_utf8(const std::basic_string<unsigned int> &s)
{
  std::string result;
  
  result.reserve(s.size()); // at least that long

  for(std::basic_string<unsigned int>::const_iterator
      it=s.begin();
      it!=s.end();
      it++)
  {  
    register unsigned int c=*it;
  
    if(c<=0x7f)           
      result+=char(c);
    else if(c<=0x7ff)
    {
      result+=char((c >> 6)   | 0xc0);
      result+=char((c & 0x3f) | 0x80);
    }
    else if(c<=0xffff)
Exemplo n.º 17
0
 std::basic_string<Ch> create_escapes(const std::basic_string<Ch> &s)
 {
     std::basic_string<Ch> result;
     typename std::basic_string<Ch>::const_iterator b = s.begin();
     typename std::basic_string<Ch>::const_iterator e = s.end();
     while (b != e)
     {
         if (*b == Ch('\0')) result += Ch('\\'), result += Ch('0');
         else if (*b == Ch('\a')) result += Ch('\\'), result += Ch('a');
         else if (*b == Ch('\b')) result += Ch('\\'), result += Ch('b');
         else if (*b == Ch('\f')) result += Ch('\\'), result += Ch('f');
         else if (*b == Ch('\n')) result += Ch('\\'), result += Ch('n');
         else if (*b == Ch('\r')) result += Ch('\\'), result += Ch('r');
         else if (*b == Ch('\v')) result += Ch('\\'), result += Ch('v');
         else if (*b == Ch('"')) result += Ch('\\'), result += Ch('"');
         else if (*b == Ch('\\')) result += Ch('\\'), result += Ch('\\');
         else
             result += *b;
         ++b;
     }
     return result;
 }
Exemplo n.º 18
0
		std::basic_string < Elem, Traits > cell_decode(std::basic_string < Elem, Traits > Str, Elem Sep_, Elem Esc){
			if(Str.size()>1 && Str.front()==Esc && Str.back()==Esc){
				Str.erase(Str.begin());
				Str.pop_back();

				bool EscFlag=false;
				for(auto itr = Str.begin(); itr != Str.end(); ++itr){
					if(*itr == Esc){
						if(EscFlag){
							itr = Str.erase(itr);
							--itr;
							EscFlag = false;
						} else{
							EscFlag = true;
						}
					} else{
						EscFlag = false;
					}
				}
			}

			return std::move(Str);
		}
Exemplo n.º 19
0
    void IO::TrimRight(std::basic_string<charType> & str, const char* chars2remove)
    {
        if (!str.empty())   //trim the characters in chars2remove from the right
        {
            std::string::size_type pos = 0;
            if (chars2remove != NULL)
            {
                pos = str.find_last_not_of(chars2remove);

                if (pos != std::string::npos)
                    str.erase(pos+1);
                else
                    str.erase( str.begin() , str.end() ); // make empty
            }
            else        //trim space
            {
                pos = std::string::npos;
                for (int i = str.size()-1; i >= 0; --i)
                {
                    if (!isspace(str[i]))
                    {
                        pos = i;
                        break;
                    }
                }
                if (pos != std::string::npos)
                {
                    if (pos+1 != str.size())
                        str.resize(pos+1);
                }
                else
                {
                    str.clear();
                }
            }
        }
    }
Exemplo n.º 20
0
 std::basic_string<Ch> create_escapes(const std::basic_string<Ch> &s,
                                      const std::locale &loc)
 {
     std::basic_string<Ch> result;
     typename std::basic_string<Ch>::const_iterator b = s.begin();
     typename std::basic_string<Ch>::const_iterator e = s.end();
     while (b != e)
     {
         if (*b == Ch('\0')) result += Ch('\\'), result += Ch('0');
         else if (*b == Ch('\b')) result += Ch('\\'), result += Ch('b');
         else if (*b == Ch('\f')) result += Ch('\\'), result += Ch('f');
         else if (*b == Ch('\n')) result += Ch('\\'), result += Ch('n');
         else if (*b == Ch('\r')) result += Ch('\\'), result += Ch('r');
         else if (*b == Ch('"')) result += Ch('\\'), result += Ch('"');
         else if (*b == Ch('\\')) result += Ch('\\'), result += Ch('\\');
         else
         {
             if (std::isprint(*b, loc))
                 result += *b;
             else
             {   
                 const char *hexdigits = "0123456789ABCDEF";
                 unsigned long u = (std::min)(static_cast<unsigned long>(*b), 0xFFFFul);
                 int d1 = u / 4096; u -= d1 * 4096;
                 int d2 = u / 256; u -= d2 * 256;
                 int d3 = u / 16; u -= d3 * 16;
                 int d4 = u;
                 result += Ch('\\'); result += Ch('u');
                 result += Ch(hexdigits[d1]); result += Ch(hexdigits[d2]);
                 result += Ch(hexdigits[d3]); result += Ch(hexdigits[d4]);
             }
         }
         ++b;
     }
     return result;
 }
Exemplo n.º 21
0
inline bool regex_search(const std::basic_string<charT, ST, SA>& s, 
                 const basic_regex<charT, traits>& e, 
                 match_flag_type flags = match_default)
{
   return regex_search(s.begin(), s.end(), e, flags);
}
Exemplo n.º 22
0
			CountT GetCount(const std::basic_string<CharT> & gram) const
			{
				std::map<std::basic_string<CharT>, CountT>::iterator iter = gram.find(gram);
				if (iter == gram.end()) return 0;
				else return iter->second;
			}
Exemplo n.º 23
0
std::basic_string<typename detail::switch_char<Ch>::type> convert(const std::basic_string<Ch> & src)
{
    return std::basic_string<typename detail::switch_char<Ch>::type>(src.begin(), src.end());
}
Exemplo n.º 24
0
 uuid operator()(std::basic_string<ch, char_traits, alloc> const& s) const {
     return operator()(s.begin(), s.end());
 }
Exemplo n.º 25
0
	static void addState(fsm_t& fsm,
						 const std::basic_string<char_t>& str,
						 typename token_t::TokenType type)
	{
		fsm.add(str.begin(), str.end(), type);
	}
Exemplo n.º 26
0
#include "m1/utf8_decode.hpp"
#include "m1/pragma_message.hpp"
#include "utf8_test_data.hpp"
#include "catch.hpp"
#include <algorithm>
#include <iterator>
#include <vector>
#include <string>
#include <locale>
#include <codecvt>

TEST_CASE("Test m1::utf8_decode", "[m1]")
{
    SECTION("utf8_decode_each -- test::utf8_test_data")
    {
        #pragma M1_MESSAGE("change to char32_t and u32string, this is a workaround for VS 2015 linker error")
        std::wstring_convert<std::codecvt_utf8<std::uint32_t>, std::uint32_t> utf32conv;
        std::basic_string<std::uint32_t> const utf32_test_data = utf32conv.from_bytes(reinterpret_cast<char const*>(test::utf8_test_data.begin()),
                                                                                      reinterpret_cast<char const*>(test::utf8_test_data.end()));

        std::vector<m1::code_point> utf32_decode;
        m1::utf8_decode_copy(test::utf8_test_data.begin(),
                             test::utf8_test_data.end(),
                             std::back_inserter(utf32_decode));

        CHECK(std::equal(utf32_decode.begin(), utf32_decode.end(),
                         utf32_test_data.begin(), utf32_test_data.end()));
    }
}
Exemplo n.º 27
0
static inline bool equals_ignore_case( std::basic_string<T> const& a, std::basic_string<T> const& b ){
  if( a.size() != b.size() )
    return false;
  return std::equal( b.begin(), b.end(), a.begin(), compare_ignore_case<T> );
}
inline u32regex_token_iterator<typename std::basic_string<charT, Traits, Alloc>::const_iterator> make_u32regex_token_iterator(const std::basic_string<charT, Traits, Alloc>& p, const u32regex& e, const std::vector<int>& submatch, regex_constants::match_flag_type m = regex_constants::match_default)
{
   typedef typename std::basic_string<charT, Traits, Alloc>::const_iterator iter_type;
   return u32regex_token_iterator<iter_type>(p.begin(), p.end(), e, submatch, m);
}
Exemplo n.º 29
0
inline std::basic_string<CharT> unescape_impl(std::basic_string<CharT> src, EscapeFlags flags)
{
    if (src.empty()) return {};
    typedef CharT char_type;
    typedef std::basic_string<char_type> string_type;
    typedef typename string_type::iterator iterator_type;
    typedef detail::named_character_references_traits<char_type> named_character_references_traits_type;
    typedef detail::unescape_traits<char_type> unescape_traits_type;

    boost::match_results<iterator_type> what;
    boost::match_flag_type rgx_flags = boost::match_default;
    iterator_type start = src.begin(), end = src.end();

    // this function must not fail.
    auto const perform_replace = [&](string_type const& replace_to) -> void {
        std::size_t const last_pos_i = std::distance(src.begin(), what[0].first);

//        std::cout << "replacing " << what.str(0) << " to " << replace_to << std::endl;

        src.replace(what[0].first, what[0].second, replace_to);
        start = src.begin() + last_pos_i + replace_to.size();
        end = src.end();
    };

    auto const process = [&]() -> bool {
        string_type const found_str(what.str(0));

        string_type found_code;
        bool is_numeric = false;
        bool is_hexadecimal = false;

        BOOST_ASSERT(what.size() == 5);

        // determine whether it's named, decimal or hexadecimal
        if (what.str(1).empty() && what.str(3).empty()) {
            // named
            is_numeric = false;

        } else {
            is_numeric = true;

            if (!what.str(1).empty()) {
                // hexadecimal
                BOOST_ASSERT(!what.str(2).empty());

                is_hexadecimal = true;
                found_code = what.str(2);

            } else {
                // decimal
                BOOST_ASSERT(!what.str(3).empty());
                BOOST_ASSERT(!what.str(4).empty());

                is_hexadecimal = false;
                found_code = what.str(4);
            }
        }

        try {
            // check for each types
            auto const& table = named_character_references_traits_type::table();

            if (is_numeric) {
                if ((flags & UNESCAPE_DECIMAL) && is_hexadecimal) {
                    UChar32 const cp = std::stoull(unescape_traits_type::hexadecimal_prefix() + found_code, nullptr, 16);
                    perform_replace(saya::to<string_type>(saya::ustring(cp)));
                    return true;

                } else if ((flags & UNESCAPE_DECIMAL) && !is_hexadecimal) {
                    UChar32 const cp = boost::lexical_cast<UChar32>(found_code);
                    perform_replace(saya::to<string_type>(saya::ustring(cp)));
                    return true;
                }
            } else {
                if ((flags & UNESCAPE_NAMED) && table.count(found_str)) {
                    perform_replace(table.at(found_str));
                    return true;
                }
            }

        } catch (std::out_of_range const&) {
            return false;

        } catch (std::invalid_argument const&) {
            return false;

        } catch (boost::bad_lexical_cast const&) {
            return false;
        }
        return false;
    };

    while (boost::regex_search(start, end, what, detail::unescape_traits<char_type>::character_reference_rgx(), rgx_flags)) {
        if (!process()) {
            // skip and continue
            start = what[0].second;
        }
        rgx_flags |= boost::match_prev_avail;
    }

    return src;
}
Exemplo n.º 30
0
inline regex_token_iterator<typename std::basic_string<charT, ST, SA>::const_iterator, charT, traits> make_regex_token_iterator(const std::basic_string<charT, ST, SA>& p, const basic_regex<charT, traits>& e, const std::vector<int>& submatch, regex_constants::match_flag_type m = regex_constants::match_default)
{
   return regex_token_iterator<typename std::basic_string<charT, ST, SA>::const_iterator, charT, traits>(p.begin(), p.end(), e, submatch, m);
}