bool CPreProcessor::emit_line_directive(ContextT const& ctx, ContainerT&pending, typename ContextT::token_type const& act_token)
{
//std::cout << "in emit_line_directive" << std::endl;
// emit a #line directive showing the relative filename instead
typename ContextT::position_type pos = act_token.get_position();
unsigned int column = 1;
typedef typename ContextT::token_type result_type;
pos.set_column(column);
std::string comments = "// From file: ";
pending.push_back(result_type(wave::T_STRINGLIT, comments.c_str(), pos));
pos.set_column(column += (unsigned int)comments.size()); // account for comments
std::string file("");
boost::filesystem::path filename(wave::util::create_path(ctx.get_current_filename().c_str()));
file += boost::wave::util::impl::escape_lit(wave::util::native_file_string(filename)) + "";
pending.push_back(result_type(wave::T_STRINGLIT, file.c_str(), pos));
pos.set_column(column += (unsigned int)file.size()); // account for filename
pending.push_back(result_type(wave::T_GENERATEDNEWLINE, "\n", pos));
return true;
}
void tokenize(
const string& str,
ContainerT& tokens,
const string& delimiters)
{
string::size_type pos, lastPos = 0;
typedef ContainerT Base;
typedef typename Base::value_type ValueType;
typedef typename ValueType::size_type SizeType;
while (true)
{
pos = str.find_first_of(delimiters, lastPos);
if (pos == std::string::npos)
{
pos = str.length();
tokens.push_back(ValueType(str.data()+lastPos,
static_cast<SizeType>(pos - lastPos)));
break;
}
else
{
tokens.push_back(ValueType(str.data()+lastPos,
static_cast<SizeType>(pos - lastPos)));
}
lastPos = pos + 1;
}
}
void tokenize(const std::string& str, ContainerT& tokens,
const std::string& delimiters = " ", bool trimEmpty = false)
{
std::string::size_type pos, lastPos = 0;
using value_type = typename ContainerT::value_type;
using size_type = typename ContainerT::size_type;
while(true)
{
pos = str.find_first_of(delimiters, lastPos);
if(pos == std::string::npos)
{
pos = str.length();
if(pos != lastPos || !trimEmpty)
tokens.push_back(value_type(str.data()+lastPos,
(size_type)pos-lastPos ));
break;
}
else
{
if(pos != lastPos || !trimEmpty)
tokens.push_back(value_type(str.data()+lastPos,
(size_type)pos-lastPos ));
}
lastPos = pos + 1;
}
}
void tokenize(const std::string& str, ContainerT& tokens,
const std::string& delimiters = " ", const bool trimEmpty = false)
{
std::string::size_type pos, lastPos = 0;
while(true)
{
pos = str.find_first_of(delimiters, lastPos);
if(pos == std::string::npos)
{
pos = str.length();
if(pos != lastPos || !trimEmpty)
tokens.push_back(typename ContainerT::value_type(str.data()+lastPos,(pos)-lastPos ));
//static_cast<ContainerT::value_type::size_type>*/(pos)-lastPos ));
break;
}
else
{
if(pos != lastPos || !trimEmpty)
tokens.push_back(typename ContainerT::value_type(str.data()+lastPos,(pos)-lastPos ));
//static_cast<ContainerT::value_type::size_type>(pos)-lastPos ));
}
lastPos = pos + 1;
}
};
void tokenize(const std::string& str, ContainerT& tokens,
const std::string& delimiters = " ", const bool trimEmpty = false)
{
typedef ContainerT Base;
typedef typename Base::value_type ValueType;
typedef typename ValueType::size_type SizeType;
SizeType pos, lastPos = 0;
while (true)
{
pos = str.find_first_of(delimiters, lastPos);
if (pos == std::string::npos)
{
pos = str.length();
if (pos != lastPos || !trimEmpty)
tokens.push_back(ValueType(str.data()+lastPos,(SizeType)pos-lastPos));
break;
}
else
{
if (pos != lastPos || !trimEmpty)
tokens.push_back(ValueType(str.data()+lastPos,(SizeType)pos-lastPos));
}
lastPos = pos + 1;
}
}
inline boost::wave::token_id
skip_whitespace(IteratorT &first, IteratorT const &last, ContainerT &queue)
{
queue.push_back (*first); // queue up the current token
token_id id = util::impl::next_token<IteratorT>::peek(first, last, false);
if (IS_CATEGORY(id, WhiteSpaceTokenType)) {
do {
queue.push_back(*++first); // queue up the next whitespace
id = util::impl::next_token<IteratorT>::peek(first, last, false);
} while (IS_CATEGORY(id, WhiteSpaceTokenType));
}
++first;
return id;
}
ContainerT
convert_list_node(const tree_node_t& node)
{
ContainerT list;
for(tree_node_iterator_t i = node.children.begin(); i != node.children.end(); ++i) //for each child node
{
list.push_back(convert_node<typename ContainerT::type>(*i));
}
return list;
}
void tokenize(const std::string& str, ContainerT& tokens, const std::string& delimiters, const bool trimEmpty)
{
/*
I usually choose to use std::vector<std::string> types as my second parameter
(ContainerT)... but list<> is way faster than vector<> for when direct access
is not needed, and you can even create your own string class and use something
like std::list<SubString> where SubString does not do any copies for incredible speed increases.
It's more than double as fast as the fastest tokenize on this page and almost 5 times
faster than some others. Also with the perfect parameter types you can eliminate all string and list copies.
Additionally it does not do the (extremely inefficient) return of result, but rather
it passes the tokens as a reference, thus also allowing you to build up tokens using
multiple calls if you so wished.Lastly it allows you to specify whether to trim empty
tokens from the results via a last optional parameter.All it needs is std::string...
the rest are optional. It does not use streams or the boost library, but is flexible
enough to be able to accept some of these foreign types naturally.
*/
std::string::size_type pos, lastPos = 0;
while(true)
{
pos = str.find_first_of(delimiters, lastPos);
if(pos == std::string::npos)
{
pos = str.length();
if(pos != lastPos || !trimEmpty)
{
tokens.push_back(typename ContainerT::value_type(str.data()+lastPos, typename ContainerT::value_type::size_type(pos-lastPos)));
}
break;
}
else
{
if(pos != lastPos || !trimEmpty)
tokens.push_back(typename ContainerT::value_type(str.data()+lastPos, typename ContainerT::value_type::size_type(pos-lastPos )));
}
lastPos = pos + 1;
}
}
void replace_ellipsis (std::vector<ContainerT> const &arguments,
typename ContainerT::size_type index,
ContainerT &expanded, PositionT const &pos)
{
using namespace cpplexer;
typedef typename ContainerT::value_type token_type;
token_type comma(T_COMMA, ",", pos);
for (/**/; index < arguments.size(); ++index) {
ContainerT const &arg = arguments[index];
std::copy(arg.begin(), arg.end(),
std::inserter(expanded, expanded.end()));
if (index < arguments.size()-1)
expanded.push_back(comma);
}
}
bool
emit_line_directive(ContextT const& ctx, ContainerT &pending,
typename ContextT::token_type const& act_token)
{
if (!need_emit_line_directives(ctx.get_language()) ||
!enable_relative_names_in_line_directives())
{
return false;
}
// emit a #line directive showing the relative filename instead
typename ContextT::position_type pos = act_token.get_position();
unsigned int column = 6;
typedef typename ContextT::token_type result_type;
using namespace boost::wave;
pos.set_column(1);
pending.push_back(result_type(T_PP_LINE, "#line", pos));
pos.set_column(column); // account for '#line'
pending.push_back(result_type(T_SPACE, " ", pos));
// 21 is the max required size for a 64 bit integer represented as a
// string
char buffer[22];
using namespace std; // for some systems sprintf is in namespace std
sprintf (buffer, "%d", pos.get_line());
pos.set_column(++column); // account for ' '
pending.push_back(result_type(T_INTLIT, buffer, pos));
pos.set_column(column += (unsigned int)strlen(buffer)); // account for <number>
pending.push_back(result_type(T_SPACE, " ", pos));
pos.set_column(++column); // account for ' '
std::string file("\"");
boost::filesystem::path filename(
boost::wave::util::create_path(ctx.get_current_relative_filename().c_str()));
using boost::wave::util::impl::escape_lit;
file += escape_lit(boost::wave::util::native_file_string(filename)) + "\"";
pending.push_back(result_type(T_STRINGLIT, file.c_str(), pos));
pos.set_column(column += (unsigned int)file.size()); // account for filename
pending.push_back(result_type(T_GENERATEDNEWLINE, "\n", pos));
return true;
}
forceinline void accumulate(const T& t)
{
container.push_back(t);
}