bool parseOctalLiteral (CodeDocument::Iterator& source) noexcept
{
if (source.nextChar() != '0')
return false;
if (! isOctalDigit (source.nextChar()))
return false;
while (isOctalDigit (source.peekNextChar()))
source.skip();
return skipNumberSuffix (source);
}
/*
** Verify the validity of an octal string
*/
int checkOctalString(char const * octalString) {
int i=0, siz=0;
char digit = 0;
siz = strlen(octalString);
for(i=0; i<siz; i++) {
digit = octalString[i];
if (!isOctalDigit(digit)) {
return -1;
}
}
return 0;
}
int isIntegerConstant(unsigned char* s) {
int i = 0, iLen = 0;
if (isNonzeroDigit(s)) {
while (isDigit(s + i)) i++;
}
else if (isHexadecimalPerfix(s)) {
i += 2;
while (isHexadecimalDigit(s + i)) i++;
}
else if (*s == '0'){
i += 1;
while (isOctalDigit(s + i)) i++;
}
if (i && (iLen = isIntegerSuffix(s + i)))
i += iLen;
return i;
}
/*
** Convert an octal string to integer value
** This function stops converting either at the end of the input string
** or if it finds a non-octal digit.
*/
int octalValue(char const *octalString) {
int numberValue = 0;
int i=0, siz=0;
char digit = 0;
siz = strlen(octalString);
for(i=0; i<siz; i++) {
digit = octalString[i];
if (isOctalDigit(digit)) {
numberValue <<= 3; // numberValue = numberValue * 8
numberValue += digitValue(digit);
} else {
break;
}
}
return numberValue;
}
Token Scanner::readNumber()
{
if (!m_src.isEnd()) {
QChar ch = m_src.peek();
if (ch.toLower() == 'b') {
m_src.move();
while (isBinaryDigit(m_src.peek()))
m_src.move();
} else if (ch.toLower() == 'o') {
m_src.move();
while (isOctalDigit(m_src.peek()))
m_src.move();
} else if (ch.toLower() == 'x') {
m_src.move();
while (isHexDigit(m_src.peek()))
m_src.move();
} else { // either integer or float number
return readFloatNumber();
}
if (isValidIntegerSuffix(m_src.peek()))
m_src.move();
}
return Token(Token::Number, m_src.anchor(), m_src.length());
}
// Map escape sequences into their equivalent symbols. Return the equivalent
// ASCII character. s is advanced past the escape sequence. If no escape
// sequence is present, the current character is returned and s
// is advanced by one. The following are recognized:
//
// \b backspace
// \f formfeed
// \t tab
// \n newline
// \r carriage return
// \s space
// \e ASCII ESC character ('\033')
// \^C C = any letter. Control code ie C-'\x40'
// \xDD number formed of 1-2 hex digits
// \DDD number formed of 1-3 octal digits
UINT escape(const _TUCHAR *&s) {
UINT asciiVal;
if(*s != '\\') {
asciiVal = *(s++);
} else {
s++; // Skip the '\'
switch(toupper(*s)) {
case 'B' :
asciiVal = '\b';
break;
case 'F' :
asciiVal = '\f';
break;
case 'T' :
asciiVal = '\t';
break;
case 'N' :
asciiVal = '\n';
break;
case 'R' :
asciiVal = '\r';
break;
case 'S' :
asciiVal = ' ';
break;
case 'E' :
asciiVal = '\033';
break;
case '^' :
s++;
asciiVal = *s;
if (asciiVal >= '\40')
asciiVal = toupper(asciiVal) - '\40';
break;
case 'X' :
asciiVal = 0;
s++;
if (isxdigit(*s))
asciiVal = hexToBin(*(s++));
if (isxdigit(*s)) {
asciiVal <<= 4;
asciiVal |= hexToBin(*(s++));
}
s--;
break;
default :
if (!isOctalDigit(*s)) {
asciiVal = *s;
}
else {
s++;
asciiVal = octalToBin(*(s++));
if (isOctalDigit(*s)) {
asciiVal <<= 3;
asciiVal |= octalToBin(*(s++));
}
if (isOctalDigit(*s)) {
asciiVal <<= 3;
asciiVal |= octalToBin(*(s++));
}
s--;
}
break;
}
s++;
}
return asciiVal;
}
int QScript::Lexer::lex()
{
int token = 0;
state = Start;
ushort stringType = 0; // either single or double quotes
pos8 = pos16 = 0;
done = false;
terminator = false;
// did we push a token on the stack previously ?
// (after an automatic semicolon insertion)
if (stackToken >= 0) {
setDone(Other);
token = stackToken;
stackToken = -1;
}
while (!done) {
switch (state) {
case Start:
if (isWhiteSpace()) {
// do nothing
} else if (current == '/' && next1 == '/') {
recordStartPos();
shift(1);
state = InSingleLineComment;
} else if (current == '/' && next1 == '*') {
recordStartPos();
shift(1);
state = InMultiLineComment;
} else if (current == 0) {
syncProhibitAutomaticSemicolon();
if (!terminator && !delimited && !prohibitAutomaticSemicolon) {
// automatic semicolon insertion if program incomplete
token = QScriptGrammar::T_SEMICOLON;
stackToken = 0;
setDone(Other);
} else {
setDone(Eof);
}
} else if (isLineTerminator()) {
shiftWindowsLineBreak();
yylineno++;
yycolumn = 0;
bol = true;
terminator = true;
syncProhibitAutomaticSemicolon();
if (restrKeyword) {
token = QScriptGrammar::T_SEMICOLON;
setDone(Other);
}
} else if (current == '"' || current == '\'') {
recordStartPos();
state = InString;
stringType = current;
} else if (isIdentLetter(current)) {
recordStartPos();
record16(current);
state = InIdentifier;
} else if (current == '0') {
recordStartPos();
record8(current);
state = InNum0;
} else if (isDecimalDigit(current)) {
recordStartPos();
record8(current);
state = InNum;
} else if (current == '.' && isDecimalDigit(next1)) {
recordStartPos();
record8(current);
state = InDecimal;
} else {
recordStartPos();
token = matchPunctuator(current, next1, next2, next3);
if (token != -1) {
if (terminator && !delimited && !prohibitAutomaticSemicolon
&& (token == QScriptGrammar::T_PLUS_PLUS
|| token == QScriptGrammar::T_MINUS_MINUS)) {
// automatic semicolon insertion
stackToken = token;
token = QScriptGrammar::T_SEMICOLON;
}
setDone(Other);
}
else {
setDone(Bad);
err = IllegalCharacter;
errmsg = QLatin1String("Illegal character");
}
}
break;
case InString:
if (current == stringType) {
shift(1);
setDone(String);
} else if (current == 0 || isLineTerminator()) {
setDone(Bad);
err = UnclosedStringLiteral;
errmsg = QLatin1String("Unclosed string at end of line");
} else if (current == '\\') {
state = InEscapeSequence;
} else {
record16(current);
}
break;
// Escape Sequences inside of strings
case InEscapeSequence:
if (isOctalDigit(current)) {
if (current >= '0' && current <= '3' &&
isOctalDigit(next1) && isOctalDigit(next2)) {
record16(convertOctal(current, next1, next2));
shift(2);
state = InString;
} else if (isOctalDigit(current) &&
isOctalDigit(next1)) {
record16(convertOctal('0', current, next1));
shift(1);
state = InString;
} else if (isOctalDigit(current)) {
record16(convertOctal('0', '0', current));
state = InString;
} else {
setDone(Bad);
err = IllegalEscapeSequence;
errmsg = QLatin1String("Illegal escape squence");
}
} else if (current == 'x')
state = InHexEscape;
else if (current == 'u')
state = InUnicodeEscape;
else {
if (isLineTerminator()) {
shiftWindowsLineBreak();
yylineno++;
yycolumn = 0;
bol = true;
} else {
record16(singleEscape(current));
}
state = InString;
}
break;
case InHexEscape:
if (isHexDigit(current) && isHexDigit(next1)) {
state = InString;
record16(QLatin1Char(convertHex(current, next1)));
shift(1);
} else if (current == stringType) {
record16(QLatin1Char('x'));
shift(1);
setDone(String);
} else {
record16(QLatin1Char('x'));
record16(current);
state = InString;
}
break;
case InUnicodeEscape:
if (isHexDigit(current) && isHexDigit(next1) &&
isHexDigit(next2) && isHexDigit(next3)) {
record16(convertUnicode(current, next1, next2, next3));
shift(3);
state = InString;
} else if (current == stringType) {
record16(QLatin1Char('u'));
shift(1);
setDone(String);
} else {
setDone(Bad);
err = IllegalUnicodeEscapeSequence;
errmsg = QLatin1String("Illegal unicode escape sequence");
}
break;
case InSingleLineComment:
if (isLineTerminator()) {
shiftWindowsLineBreak();
yylineno++;
yycolumn = 0;
terminator = true;
bol = true;
if (restrKeyword) {
token = QScriptGrammar::T_SEMICOLON;
setDone(Other);
} else
state = Start;
} else if (current == 0) {
setDone(Eof);
}
break;
case InMultiLineComment:
if (current == 0) {
setDone(Bad);
err = UnclosedComment;
errmsg = QLatin1String("Unclosed comment at end of file");
} else if (isLineTerminator()) {
shiftWindowsLineBreak();
yylineno++;
} else if (current == '*' && next1 == '/') {
state = Start;
shift(1);
}
break;
case InIdentifier:
if (isIdentLetter(current) || isDecimalDigit(current)) {
record16(current);
break;
}
setDone(Identifier);
break;
case InNum0:
if (current == 'x' || current == 'X') {
record8(current);
state = InHex;
} else if (current == '.') {
record8(current);
state = InDecimal;
} else if (current == 'e' || current == 'E') {
record8(current);
state = InExponentIndicator;
} else if (isOctalDigit(current)) {
record8(current);
state = InOctal;
} else if (isDecimalDigit(current)) {
record8(current);
state = InDecimal;
} else {
setDone(Number);
}
break;
case InHex:
if (isHexDigit(current))
record8(current);
else
setDone(Hex);
break;
case InOctal:
if (isOctalDigit(current)) {
record8(current);
} else if (isDecimalDigit(current)) {
record8(current);
state = InDecimal;
} else {
setDone(Octal);
}
break;
case InNum:
if (isDecimalDigit(current)) {
record8(current);
} else if (current == '.') {
record8(current);
state = InDecimal;
} else if (current == 'e' || current == 'E') {
record8(current);
state = InExponentIndicator;
} else {
setDone(Number);
}
break;
case InDecimal:
if (isDecimalDigit(current)) {
record8(current);
} else if (current == 'e' || current == 'E') {
record8(current);
state = InExponentIndicator;
} else {
setDone(Number);
}
break;
case InExponentIndicator:
if (current == '+' || current == '-') {
record8(current);
} else if (isDecimalDigit(current)) {
record8(current);
state = InExponent;
} else {
setDone(Bad);
err = IllegalExponentIndicator;
errmsg = QLatin1String("Illegal syntax for exponential number");
}
break;
case InExponent:
if (isDecimalDigit(current)) {
record8(current);
} else {
setDone(Number);
}
break;
default:
Q_ASSERT_X(0, "Lexer::lex", "Unhandled state in switch statement");
}
// move on to the next character
if (!done)
shift(1);
if (state != Start && state != InSingleLineComment)
bol = false;
}
// no identifiers allowed directly after numeric literal, e.g. "3in" is bad
if ((state == Number || state == Octal || state == Hex)
&& isIdentLetter(current)) {
state = Bad;
err = IllegalIdentifier;
errmsg = QLatin1String("Identifier cannot start with numeric literal");
}
// terminate string
buffer8[pos8] = '\0';
double dval = 0;
if (state == Number) {
dval = qstrtod(buffer8, 0, 0);
} else if (state == Hex) { // scan hex numbers
dval = QScript::integerFromString(buffer8, pos8, 16);
state = Number;
} else if (state == Octal) { // scan octal number
dval = QScript::integerFromString(buffer8, pos8, 8);
state = Number;
}
restrKeyword = false;
delimited = false;
switch (parenthesesState) {
case IgnoreParentheses:
break;
case CountParentheses:
if (token == QScriptGrammar::T_RPAREN) {
--parenthesesCount;
if (parenthesesCount == 0)
parenthesesState = BalancedParentheses;
} else if (token == QScriptGrammar::T_LPAREN) {
++parenthesesCount;
}
break;
case BalancedParentheses:
parenthesesState = IgnoreParentheses;
break;
}
switch (state) {
case Eof:
return 0;
case Other:
if(token == QScriptGrammar::T_RBRACE || token == QScriptGrammar::T_SEMICOLON)
delimited = true;
return token;
case Identifier:
if ((token = findReservedWord(buffer16, pos16)) < 0) {
/* TODO: close leak on parse error. same holds true for String */
if (driver) {
Q_ASSERT_X(false, Q_FUNC_INFO, "not implemented");
qsyylval.ustr = 0; // driver->intern(buffer16, pos16);
} else
qsyylval.ustr = 0;
return QScriptGrammar::T_IDENTIFIER;
}
if (token == QScriptGrammar::T_CONTINUE || token == QScriptGrammar::T_BREAK
|| token == QScriptGrammar::T_RETURN || token == QScriptGrammar::T_THROW) {
restrKeyword = true;
} else if (token == QScriptGrammar::T_IF || token == QScriptGrammar::T_FOR
|| token == QScriptGrammar::T_WHILE || token == QScriptGrammar::T_WITH) {
parenthesesState = CountParentheses;
parenthesesCount = 0;
} else if (token == QScriptGrammar::T_DO) {
parenthesesState = BalancedParentheses;
}
return token;
case String:
if (driver) {
Q_ASSERT_X(false, Q_FUNC_INFO, "not implemented");
qsyylval.ustr = 0; // driver->intern(buffer16, pos16);
} else
qsyylval.ustr = 0;
return QScriptGrammar::T_STRING_LITERAL;
case Number:
qsyylval.dval = dval;
return QScriptGrammar::T_NUMERIC_LITERAL;
case Bad:
return -1;
default:
Q_ASSERT(!"unhandled numeration value in switch");
return -1;
}
}
bool MathLib::isInt(const std::string & s)
{
// perform prechecks:
// ------------------
// first check, if a point is found, it is an floating point value
if (s.find(".", 0) != std::string::npos) return false;
// check for scientific notation e.g. NumberE-Number this is obvious an floating point value
else if (s.find("E-", 0) != std::string::npos || s.find("e-", 0) != std::string::npos) return false;
// prechecking has nothing found,...
// gather information
enum Representation
{
eScientific = 0 // NumberE+Number or NumberENumber
, eOctal // starts with 0
, eHex // starts with 0x
, eDefault // Numbers with a (possible) trailing u or U or l or L for unsigned or long datatypes
};
// create an instance
Representation Mode = eDefault;
// remember position
unsigned long n = 0;
// eat up whitespace
while (std::isspace(s[n])) ++n;
// determine type
if (s.find("E", 0) != std::string::npos)
{
Mode = eScientific;
}
else if (s.find("0x", n, 2) != std::string::npos)
{
Mode = eHex;
}
else if (s.length() > 1 && s[0] == '0' && std::isdigit(s[1]))
{
Mode = eOctal;
}
// check sign
if (s[n] == '-' || s[n] == '+') ++n;
// check scientific notation
if (Mode == eScientific)
{
// check digits
while (std::isdigit(s[n])) ++n;
// check scientific notation
if (std::tolower(s[n]) == 'e')
{
++n;
// check positive exponent
if (s[n] == '+') ++n;
// floating pointer number e.g. 124E-2
if (s[n] == '-') return false;
// check digits of the exponent
while (std::isdigit(s[n])) ++n;
}
}
// check hex notation
else if (Mode == eHex)
{
++n; // 0
++n; // x
while (std::isxdigit(s[n]))
++n;
}
// check octal notation
else if (Mode == eOctal)
{
while (isOctalDigit(s[n]))
++n;
}
else if (Mode == eDefault)
{
// starts with digit
bool bStartsWithDigit=false;
while (std::isdigit(s[n]))
{
bStartsWithDigit=true;
++n;
};
// unsigned or long
while (std::tolower(s[n]) == 'u' || std::tolower(s[n]) == 'l') ++n;
if (bStartsWithDigit==false)
return false;
}
// eat up whitespace
while (std::isspace(s[n]))
++n;
// if everything goes good, we are at the end of the string and no digits/character
// is here --> return true, but if something was found eg. 12E+12AA return false
if (s[n])
return false;
return true;
}
Char PushbackString::nextOctal() {
Char c = next();
if ( c == 0 ) return 0;
if ( isOctalDigit( c ) ) return c;
return 0;
}
bool Tokenizer::consumeNumber(std::streambuf::int_type ch)
{
int count = 0;
if (isSign(ch))
{
bool number = false;
while (!eof())
{
++count;
auto nch = mpInput->get();
if (isWhitespace(nch))
continue;
number = isDecimalDigit(nch);
break;
}
if (count)
mpInput->clear();
for (int i = 0; i < count; ++i)
mpInput->unget();
if (!number)
return false;
}
absorbed(ch);
mpCurrent->addChar(ch);
for (int i = 0; i < count; ++i)
{
auto nch = mpInput->get();
absorbed(nch);
mpCurrent->addChar(nch);
}
bool bDot = false;
if (isDot(ch))
{
bDot = true;
mpCurrent->setType(Token::TYPE_REAL_LITERAL);
}
else
{
mpCurrent->setType(Token::TYPE_INTEGER_LITERAL);
if (!eof() && isZero(ch))
{
ch = mpInput->get();
if (isHexIndicator(ch))
{
if (eof())
{
mpMessageCollector->addMessage(
Message::SEVERITY_ERROR, Message::t_invalidIntegerLiteral,
mpSourceId, mpCurrent->line(), mpCurrent->beginColumn());
mpCurrent.reset();
return false;
}
mpCurrent->addChar(ch);
absorbed(ch);
while (!eof())
{
ch = mpInput->get();
if (isNonHexicalLetter(ch) || isUnderscore(ch) || isDot(ch))
{
mpMessageCollector->addMessage(Message::SEVERITY_ERROR,
Message::t_invalidIntegerLiteral,
mpSourceId, mpCurrent->line(), mpCurrent->beginColumn());
mpCurrent.reset();
return false;
}
if (!isHexicalDigit(ch))
{
mpInput->clear();
mpInput->unget();
break;
}
absorbed(ch);
mpCurrent->addChar(ch);
}
mpCurrent->setEndColumn(column());
return true;
}
for (;;)
{
if ( isLetter(ch)
|| isDot(ch)
|| isNonOctalDecimalDigit(ch)
|| isUnderscore(ch))
{
mpMessageCollector->addMessage(
Message::SEVERITY_ERROR, Message::t_invalidIntegerLiteral,
mpSourceId, mpCurrent->line(), mpCurrent->beginColumn());
mpCurrent.reset();
return false;
}
if (!isOctalDigit(ch))
{
mpInput->clear();
mpInput->unget();
break;
}
absorbed(ch);
mpCurrent->addChar(ch);
if (eof())
{
break;
}
ch = mpInput->get();
}
mpCurrent->setEndColumn(column());
return true;
}
}
bool bExponent = false;
while (!eof())
{
ch = mpInput->get();
if (isExponent(ch))
{
if (bExponent)
{
mpMessageCollector->addMessage(
Message::SEVERITY_ERROR, Message::t_invalidIntegerLiteral,
mpSourceId, mpCurrent->line(), mpCurrent->beginColumn());
mpCurrent.reset();
return false;
}
bExponent = true;
mpCurrent->setType(Token::TYPE_REAL_LITERAL);
absorbed(ch);
mpCurrent->addChar(ch);
ch = mpInput->get();
if (isSign(ch))
{
mpCurrent->setType(Token::TYPE_REAL_LITERAL);
absorbed(ch);
mpCurrent->addChar(ch);
}
else
{
mpInput->clear();
mpInput->unget();
}
continue;
}
if (isLetter(ch) || isUnderscore(ch))
{
mpMessageCollector->addMessage(
Message::SEVERITY_ERROR, Message::t_invalidIntegerLiteral,
mpSourceId, mpCurrent->line(), mpCurrent->beginColumn());
mpCurrent.reset();
return false;
}
if (isDot(ch))
{
if (bExponent || bDot)
{
mpMessageCollector->addMessage(
Message::SEVERITY_ERROR, Message::t_invalidIntegerLiteral,
mpSourceId, mpCurrent->line(), mpCurrent->beginColumn());
mpCurrent.reset();
return false;
}
bDot = true;
mpCurrent->setType(Token::TYPE_REAL_LITERAL);
absorbed(ch);
mpCurrent->addChar(ch);
continue;
}
if (!isDecimalDigit(ch))
{
mpInput->clear();
mpInput->unget();
break;
}
absorbed(ch);
mpCurrent->addChar(ch);
}
mpCurrent->setEndColumn(column());
return true;
}
