void readQuote(std::wistream& stream, std::wstring& str_out)//reads string from between double quotes.
{
wchar_t c;
str_out = L"";
//------------------------------------------------------------------------
//parse to first "
//------------------------------------------------------------------------
while(1)
{
stream.get(c);
if(!stream.good() || c == '"')
break;
}
//------------------------------------------------------------------------
//parse quoted text, to next "
//------------------------------------------------------------------------
while(1)
{
stream.get(c);
if(!stream.good() || c == '"')
break;
else
::concatWithChar(str_out, c);
}
/*stream >> str_out;
if(str_out.empty() || str_out[0] != '\"')
return;
str_out = getTailSubString(str_out, 1);//lop off quote
if(::hasSuffix(str_out, "\""))
{
str_out = str_out.substr(0, str_out.size() - 1);
return;
}
//------------------------------------------------------------------------
//read thru char by char until hit next quote
//------------------------------------------------------------------------
while(stream.good())
{
char c;
stream.get(c);
if(c == '\"')
break;
::concatWithChar(str_out, c);
}*/
}
void COptions::ImportOptions(std::wistream& in, std::vector<std::wstring>& optionVector)
{
wchar_t ch;
std::wstring currentToken;
while (in)
{
currentToken = L"";
do
{
in.get(ch);
if (in.eof() || ch == L' ' || ch == L'\t' || ch == L'' || ch == L'\n')
{
break;
}
else
{
currentToken.append(1, ch);
}
} while (in);
if (currentToken.length() != 0)
{
optionVector.push_back(currentToken);
}
}
}
uint64_t libmaus2::util::Utf8String::computeOctetLength(std::wistream & stream, uint64_t const len)
{
::libmaus2::util::CountPutObject CPO;
for ( uint64_t i = 0; i < len; ++i )
{
std::wistream::int_type const w = stream.get();
assert ( w != std::wistream::traits_type::eof() );
::libmaus2::util::UTF8::encodeUTF8(w,CPO);
}
return CPO.c;
}
//! Loads a config file from a stream.
bool CConfigReader::Load( std::wistream& stream )
{
wchar_t c;
while ( stream.get(c).good() )
{
Text += c;
}
TextPtr = Text.c_str();
return ( Text != L"" );
}
libmaus2::util::Utf8String::Utf8String(std::wistream & CIS, uint64_t const octetlength, uint64_t const symlength)
: A(octetlength,false)
{
::libmaus2::util::PutObject<uint8_t *> P(A.begin());
for ( uint64_t i = 0; i < symlength; ++i )
{
std::wistream::int_type const w = CIS.get();
assert ( w != std::wistream::traits_type::eof() );
::libmaus2::util::UTF8::encodeUTF8(w,P);
}
setup();
}
// Reads string from between double quotes.
void readQuote(std::wistream& stream, std::wstring& str_out)
{
wchar_t c;
str_out = L"";
// Parse to first "
while (1)
{
stream.get(c);
enforce(stream.good(), "Expected quoted string, got end of stream");
if (c == '"')
break;
enforce(isspace(c), std::wstring(L"Expected quoted string, got ") + c);
}
// Parse quoted text
bool escaping = false;
while (1)
{
stream.get(c);
enforce(stream.good(), "Unexpected end of stream while reading quoted string");
if (escaping)
{
str_out.push_back(c);
escaping = false;
}
else
{
if (c == '\\')
escaping = true;
else
if (c == '"')
break;
else
str_out.push_back(c);
}
}
}
void load_file2(std::wstring& wstrOut, std::wistream& wistreamIn)
{
wstrOut.erase();
//bad() 如果出现错误则返回true
if(wistreamIn.bad())
return;
wstrOut.reserve(wistreamIn.rdbuf()->in_avail());
wchar_t c;
//get() 读取字符
while(wistreamIn.get(c))
{
if(wstrOut.capacity() == wstrOut.size())
wstrOut.reserve(wstrOut.capacity() * 3);
wstrOut.append(1, c);
}
}
wchar_t Compiler::getNextCharacter(std::wistream &source, SourcePos &pos)
{
pos.column++;
pos.character++;
return source.get();
}
//! Parse source and build tokens vector
//! \param source source code
void Compiler::tokenize(std::wistream& source)
{
tokens.clear();
SourcePos pos(0, 0, 0);
const unsigned tabSize = 4;
// tokenize text source
while (source.good())
{
wchar_t c = source.get();
if (source.eof())
break;
pos.column++;
pos.character++;
switch (c)
{
// simple cases of one character
case ' ': break;
//case '\t': pos.column += tabSize - 1; break;
case '\t': break;
case '\n': pos.row++; pos.column = -1; break; // -1 so next call to pos.column++ result set 0
case '\r': pos.column = -1; break; // -1 so next call to pos.column++ result set 0
case '(': tokens.push_back(Token(Token::TOKEN_PAR_OPEN, pos)); break;
case ')': tokens.push_back(Token(Token::TOKEN_PAR_CLOSE, pos)); break;
case '[': tokens.push_back(Token(Token::TOKEN_BRACKET_OPEN, pos)); break;
case ']': tokens.push_back(Token(Token::TOKEN_BRACKET_CLOSE, pos)); break;
case ':': tokens.push_back(Token(Token::TOKEN_COLON, pos)); break;
case ',': tokens.push_back(Token(Token::TOKEN_COMMA, pos)); break;
// special case for comment
case '#':
{
// check if it's a comment block #* ... *#
if (source.peek() == '*')
{
// comment block
// record position of the begining
SourcePos begin(pos);
// move forward by 2 characters then search for the end
int step = 2;
while ((step > 0) || (c != '*') || (source.peek() != '#'))
{
if (step)
step--;
if (c == '\t')
pos.column += tabSize;
else if (c == '\n')
{
pos.row++;
pos.column = 0;
}
else
pos.column++;
c = source.get();
pos.character++;
if (source.eof())
{
// EOF -> unbalanced block
throw TranslatableError(begin, ERROR_UNBALANCED_COMMENT_BLOCK);
}
}
// fetch the #
getNextCharacter(source, pos);
}
else
{
// simple comment
while ((c != '\n') && (c != '\r') && (!source.eof()))
{
if (c == '\t')
pos.column += tabSize;
else
pos.column++;
c = source.get();
pos.character++;
}
if (c == '\n')
{
pos.row++;
pos.column = 0;
}
else if (c == '\r')
pos.column = 0;
}
}
break;
// cases that require one character look-ahead
case '+':
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_ADD_EQUAL))
break;
if (testNextCharacter(source, pos, '+', Token::TOKEN_OP_PLUS_PLUS))
break;
tokens.push_back(Token(Token::TOKEN_OP_ADD, pos));
break;
case '-':
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_NEG_EQUAL))
break;
if (testNextCharacter(source, pos, '-', Token::TOKEN_OP_MINUS_MINUS))
break;
tokens.push_back(Token(Token::TOKEN_OP_NEG, pos));
break;
case '*':
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_MULT_EQUAL))
break;
tokens.push_back(Token(Token::TOKEN_OP_MULT, pos));
break;
case '/':
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_DIV_EQUAL))
break;
tokens.push_back(Token(Token::TOKEN_OP_DIV, pos));
break;
case '%':
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_MOD_EQUAL))
break;
tokens.push_back(Token(Token::TOKEN_OP_MOD, pos));
break;
case '|':
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_BIT_OR_EQUAL))
break;
tokens.push_back(Token(Token::TOKEN_OP_BIT_OR, pos));
break;
case '^':
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_BIT_XOR_EQUAL))
break;
tokens.push_back(Token(Token::TOKEN_OP_BIT_XOR, pos));
break;
case '&':
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_BIT_AND_EQUAL))
break;
tokens.push_back(Token(Token::TOKEN_OP_BIT_AND, pos));
break;
case '~':
tokens.push_back(Token(Token::TOKEN_OP_BIT_NOT, pos));
break;
case '!':
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_NOT_EQUAL))
break;
throw TranslatableError(pos, ERROR_SYNTAX);
break;
case '=':
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_EQUAL))
break;
tokens.push_back(Token(Token::TOKEN_ASSIGN, pos));
break;
// cases that require two characters look-ahead
case '<':
if (source.peek() == '<')
{
// <<
getNextCharacter(source, pos);
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_SHIFT_LEFT_EQUAL))
break;
tokens.push_back(Token(Token::TOKEN_OP_SHIFT_LEFT, pos));
break;
}
// <
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_SMALLER_EQUAL))
break;
tokens.push_back(Token(Token::TOKEN_OP_SMALLER, pos));
break;
case '>':
if (source.peek() == '>')
{
// >>
getNextCharacter(source, pos);
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_SHIFT_RIGHT_EQUAL))
break;
tokens.push_back(Token(Token::TOKEN_OP_SHIFT_RIGHT, pos));
break;
}
// >
if (testNextCharacter(source, pos, '=', Token::TOKEN_OP_BIGGER_EQUAL))
break;
tokens.push_back(Token(Token::TOKEN_OP_BIGGER, pos));
break;
// cases that require to look for a while
default:
{
// check first character
if (!std::iswalnum(c) && (c != '_'))
throw TranslatableError(pos, ERROR_INVALID_IDENTIFIER).arg((unsigned)c, 0, 16);
// get a string
std::wstring s;
s += c;
wchar_t nextC = source.peek();
int posIncrement = 0;
while ((source.good()) && (std::iswalnum(nextC) || (nextC == '_') || (nextC == '.')))
{
s += nextC;
source.get();
posIncrement++;
nextC = source.peek();
}
// we now have a string, let's check what it is
if (std::iswdigit(s[0]))
{
// check if hex or binary
if ((s.length() > 1) && (s[0] == '0') && (!std::iswdigit(s[1])))
{
// check if we have a valid number
if (s[1] == 'x')
{
for (unsigned i = 2; i < s.size(); i++)
if (!std::iswxdigit(s[i]))
throw TranslatableError(pos, ERROR_INVALID_HEXA_NUMBER);
}
else if (s[1] == 'b')
{
for (unsigned i = 2; i < s.size(); i++)
if ((s[i] != '0') && (s[i] != '1'))
throw TranslatableError(pos, ERROR_INVALID_BINARY_NUMBER);
}
else
throw TranslatableError(pos, ERROR_NUMBER_INVALID_BASE);
}
else
{
// check if we have a valid number
for (unsigned i = 1; i < s.size(); i++)
if (!std::iswdigit(s[i]))
throw TranslatableError(pos, ERROR_IN_NUMBER);
}
tokens.push_back(Token(Token::TOKEN_INT_LITERAL, pos, s));
}
else
{
// check if it is a known keyword
// FIXME: clean-up that with a table
if (s == L"when")
tokens.push_back(Token(Token::TOKEN_STR_when, pos));
else if (s == L"emit")
tokens.push_back(Token(Token::TOKEN_STR_emit, pos));
else if (s == L"_emit")
tokens.push_back(Token(Token::TOKEN_STR_hidden_emit, pos));
else if (s == L"for")
tokens.push_back(Token(Token::TOKEN_STR_for, pos));
else if (s == L"in")
tokens.push_back(Token(Token::TOKEN_STR_in, pos));
else if (s == L"step")
tokens.push_back(Token(Token::TOKEN_STR_step, pos));
else if (s == L"while")
tokens.push_back(Token(Token::TOKEN_STR_while, pos));
else if (s == L"do")
tokens.push_back(Token(Token::TOKEN_STR_do, pos));
else if (s == L"if")
tokens.push_back(Token(Token::TOKEN_STR_if, pos));
else if (s == L"then")
tokens.push_back(Token(Token::TOKEN_STR_then, pos));
else if (s == L"else")
tokens.push_back(Token(Token::TOKEN_STR_else, pos));
else if (s == L"elseif")
tokens.push_back(Token(Token::TOKEN_STR_elseif, pos));
else if (s == L"end")
tokens.push_back(Token(Token::TOKEN_STR_end, pos));
else if (s == L"var")
tokens.push_back(Token(Token::TOKEN_STR_var, pos));
else if (s == L"const")
tokens.push_back(Token(Token::TOKEN_STR_const, pos));
else if (s == L"call")
tokens.push_back(Token(Token::TOKEN_STR_call, pos));
else if (s == L"sub")
tokens.push_back(Token(Token::TOKEN_STR_sub, pos));
else if (s == L"callsub")
tokens.push_back(Token(Token::TOKEN_STR_callsub, pos));
else if (s == L"onevent")
tokens.push_back(Token(Token::TOKEN_STR_onevent, pos));
else if (s == L"abs")
tokens.push_back(Token(Token::TOKEN_STR_abs, pos));
else if (s == L"return")
tokens.push_back(Token(Token::TOKEN_STR_return, pos));
else if (s == L"or")
tokens.push_back(Token(Token::TOKEN_OP_OR, pos));
else if (s == L"and")
tokens.push_back(Token(Token::TOKEN_OP_AND, pos));
else if (s == L"not")
tokens.push_back(Token(Token::TOKEN_OP_NOT, pos));
else
tokens.push_back(Token(Token::TOKEN_STRING_LITERAL, pos, s));
}
pos.column += posIncrement;
pos.character += posIncrement;
}
break;
} // switch (c)
} // while (source.good())
tokens.push_back(Token(Token::TOKEN_END_OF_STREAM, pos));
}