pfs::error_code ubjson_ostream<OStreamType, JsonType>::write_string (string_type const & s, bool with_prefix)
{
if (with_prefix) {
// Using size is safe here (no matter the string encoding)
if (s.size() == 1 && *s.cbegin() <= numeric_limits<int8_t>::max()) {
_os << static_cast<int8_t>('C');
_os << static_cast<int8_t>(*s.cbegin());
} else {
_os << static_cast<int8_t>('S');
write_integer(static_cast<typename json_type::integer_type>(s.size()), true);
_os << s.utf8();
}
} else {
write_integer(static_cast<typename json_type::integer_type>(s.size()), true);
_os << s;
}
return pfs::error_code();
}
/** Tokenize the expression.
@return a TokenList containing the tokens from 'expression'.
@param expression [in] The expression to tokenize.
@note Tokenizer dictionary may be updated if expression contains variables.
@note Will throws 'BadCharacter' if the expression contains an un-tokenizable character.
*/
TokenList Tokenizer::tokenize( string_type const& expression ) {
TokenList tokenizedExpression;
auto currentChar = expression.cbegin();
for(;;)
{
// strip whitespace
while( currentChar != end(expression) && isspace(*currentChar) )
++currentChar;
// check of end of expression
if( currentChar == end(expression) ) break;
// check for a number
if( isdigit( *currentChar ) ) {
tokenizedExpression.push_back( _get_number( currentChar, expression ) );
continue;
}
//check if left parenthesis
if (*currentChar == '(') {
tokenizedExpression.push_back(make<LeftParenthesis>());
++currentChar;
continue;
}
//check if right parenthesis
if (*currentChar == ')') {
tokenizedExpression.push_back(make<RightParenthesis>());
++currentChar;
continue;
}
//check if multiplication
if (*currentChar == '*') {
tokenizedExpression.push_back(make<Multiplication>());
++currentChar;
continue;
}
//check if division
if (*currentChar == '/') {
tokenizedExpression.push_back(make<Division>());
++currentChar;
continue;
}
//chck if argument seperator
if (*currentChar == ',') {
tokenizedExpression.push_back(make<ArgumentSeparator>());
++currentChar;
continue;
}
//check for factorial or inequality operator
if (*currentChar == '!') {
if (next(currentChar) != end(expression)) {
if (*next(currentChar) == '=') { //if next character is =, make inequality operator
tokenizedExpression.push_back(make<Inequality>());
currentChar++;
currentChar++;
continue;
}
}
//if is not inequality operator, must be factorial operator
tokenizedExpression.push_back(make<Factorial>());
currentChar++;
continue;
}
//check for addition or identity operator
if (*currentChar == '+') {
++currentChar;
if (!tokenizedExpression.empty()) {
if (is<RightParenthesis>(tokenizedExpression.back()) || is<Operand>(tokenizedExpression.back()) || is<Factorial>(tokenizedExpression.back())) {
//if + relates to another token, must be an addition operator
tokenizedExpression.push_back(make<Addition>());
}
else { //is an identity operator
tokenizedExpression.push_back(make<Identity>());
}
}
else { //in an empty expression must be identity operator
tokenizedExpression.push_back(make<Identity>());
}
continue;
}
//check for subtraction or negatino operator
if (*currentChar == '-') {
++currentChar;
if (!tokenizedExpression.empty()) {
if (is<RightParenthesis>(tokenizedExpression.back()) || is<Operand>(tokenizedExpression.back()) || is<Factorial>(tokenizedExpression.back())) {
tokenizedExpression.push_back(make<Subtraction>()); // same as addition, if related to operand, is subtraction
}
else {
tokenizedExpression.push_back(make<Negation>()); //must be negation
}
}
else {//empty expression, must be negation operator
tokenizedExpression.push_back(make<Negation>());
}
continue;
}
//check if power operator
if (*currentChar == '#') {
tokenizedExpression.push_back(make<Power>());
currentChar++;
continue;
}
//Check if modulus division operator
if (*currentChar == '%') {
tokenizedExpression.push_back(make<Modulus>());
currentChar++;
continue;
}
//check for equality or assignment operator
if (*currentChar == '=') {
if (next(currentChar) != end(expression)) {
//if next character is another =, must be equality operator
if (*next(currentChar) == '=') {
tokenizedExpression.push_back(make<Equality>());
currentChar++;
currentChar++;
continue;
}
}
//if single =, is assignment operator
tokenizedExpression.push_back(make<Assignment>());
currentChar++;
continue;
}
//check less than or less equal operator
if (*currentChar == '<') {
if (next(currentChar) != end(expression)) {
//if next character is =, make LessEqual operator
if (*next(currentChar) == '=') {
tokenizedExpression.push_back(make<LessEqual>());
currentChar++;
currentChar++;
continue;
}
}
//if no =, is less equal operator
tokenizedExpression.push_back(make<Less>());
currentChar++;
continue;
}
//check for greater than or greater equals operator
if (*currentChar == '>') {
if (next(currentChar) != end(expression)) {
//if next character is =, is greater equals operator
if (*next(currentChar) == '=') {
tokenizedExpression.push_back(make<GreaterEqual>());
currentChar++;
currentChar++;
continue;
}
}
//if lone >, mkae greater than operator
tokenizedExpression.push_back(make<Greater>());
currentChar++;
continue;
}
//is an identifier or variable
if (isalpha(*currentChar)) {
tokenizedExpression.push_back(_get_identifier(currentChar, expression));
continue;
}
// not a recognized token
throw XBadCharacter( expression, currentChar - begin(expression) );
}
return tokenizedExpression;
}
