bool parseValue(File& file, Value& out_value) { skipWhitespaces(file); char c = *file.str; uint32_t line_num = file.current_line_num; if(c == '{') { Object obj; if(!parseObject(file, obj)) return false; out_value = Value(std::move(obj), line_num); } else if(c == '[') { Array arr; if(!parseArray(file, arr)) return false; out_value = Value(std::move(arr), line_num); } else if(c == '"') { std::string str; if(consume(file, "\"\"\"")) { if(!parseCode(file, str)) return false; } else if(!parseString(file, str)) return false; out_value = Value(std::move(str), line_num); } else if(c == '-' || (c >= '0' && c <= '9')) { double num; if(!parseNumber(file, num)) return false; out_value = Value(num, line_num); } else if(isAlpha(*file.str)) { if(*file.str == 't' && consume(file, "true")) { out_value = Value(true, line_num); return true; } else if(*file.str == 'f' && consume(file, "false")) { out_value = Value(false, line_num); return true; } else if(*file.str == 'n' && consume(file, "null")) { out_value = Value(line_num); return true; } std::string str; if(!parseIdentifier(file, str)) return false; out_value = Value(std::move(str), line_num); } else if(c == '\0') { std::cout << "Line " << file.current_line_num << ": Unexpected end-of-file!\n"; return false; } else { std::cout << "Line " << file.current_line_num << ": Unexpected char '" << c << "'!\n"; return false; } return true; }
static void readToken (tokenInfo *const token) { int c; token->type = TOKEN_UNDEFINED; token->keyword = KEYWORD_NONE; vStringClear (token->string); getNextChar: if (! InPhp) { c = findPhpStart (); if (c != EOF) InPhp = TRUE; } else c = getcFromInputFile (); c = skipWhitespaces (c); token->lineNumber = getInputLineNumber (); token->filePosition = getInputFilePosition (); switch (c) { case EOF: token->type = TOKEN_EOF; break; case '(': token->type = TOKEN_OPEN_PAREN; break; case ')': token->type = TOKEN_CLOSE_PAREN; break; case ';': token->type = TOKEN_SEMICOLON; break; case ',': token->type = TOKEN_COMMA; break; case '.': token->type = TOKEN_PERIOD; break; case ':': token->type = TOKEN_COLON; break; case '{': token->type = TOKEN_OPEN_CURLY; break; case '}': token->type = TOKEN_CLOSE_CURLY; break; case '[': token->type = TOKEN_OPEN_SQUARE; break; case ']': token->type = TOKEN_CLOSE_SQUARE; break; case '&': token->type = TOKEN_AMPERSAND; break; case '\\': token->type = TOKEN_BACKSLASH; break; case '=': { int d = getcFromInputFile (); if (d == '=' || d == '>') token->type = TOKEN_OPERATOR; else { ungetcToInputFile (d); token->type = TOKEN_EQUAL_SIGN; } break; } case '\'': case '"': token->type = TOKEN_STRING; parseString (token->string, c); token->lineNumber = getInputLineNumber (); token->filePosition = getInputFilePosition (); break; case '<': { int d = getcFromInputFile (); if (d == '/') { /* </script[:white:]*> */ if (tolower ((d = getcFromInputFile ())) == 's' && tolower ((d = getcFromInputFile ())) == 'c' && tolower ((d = getcFromInputFile ())) == 'r' && tolower ((d = getcFromInputFile ())) == 'i' && tolower ((d = getcFromInputFile ())) == 'p' && tolower ((d = getcFromInputFile ())) == 't' && (d = skipWhitespaces (getcFromInputFile ())) == '>') { InPhp = FALSE; goto getNextChar; } else { ungetcToInputFile (d); token->type = TOKEN_UNDEFINED; } } else if (d == '<' && (d = getcFromInputFile ()) == '<') { token->type = TOKEN_STRING; parseHeredoc (token->string); } else { ungetcToInputFile (d); token->type = TOKEN_UNDEFINED; } break; } case '#': /* comment */ skipSingleComment (); goto getNextChar; break; case '+': case '-': case '*': case '%': { int d = getcFromInputFile (); if (d != '=' && ! (c == '-' && d == '>')) ungetcToInputFile (d); token->type = TOKEN_OPERATOR; break; } case '/': /* division or comment start */ { int d = getcFromInputFile (); if (d == '/') /* single-line comment */ { skipSingleComment (); goto getNextChar; } else if (d == '*') { do { c = skipToCharacter ('*'); if (c != EOF) { c = getcFromInputFile (); if (c == '/') break; else ungetcToInputFile (c); } } while (c != EOF && c != '\0'); goto getNextChar; } else { if (d != '=') ungetcToInputFile (d); token->type = TOKEN_OPERATOR; } break; } case '$': /* variable start */ { int d = getcFromInputFile (); if (! isIdentChar (d)) { ungetcToInputFile (d); token->type = TOKEN_UNDEFINED; } else { parseIdentifier (token->string, d); token->type = TOKEN_VARIABLE; } break; } case '?': /* maybe the end of the PHP chunk */ { int d = getcFromInputFile (); if (d == '>') { InPhp = FALSE; goto getNextChar; } else { ungetcToInputFile (d); token->type = TOKEN_UNDEFINED; } break; } default: if (! isIdentChar (c)) token->type = TOKEN_UNDEFINED; else { parseIdentifier (token->string, c); token->keyword = analyzeToken (token->string, getInputLanguage ()); if (token->keyword == KEYWORD_NONE) token->type = TOKEN_IDENTIFIER; else token->type = TOKEN_KEYWORD; } break; } if (token->type == TOKEN_SEMICOLON || token->type == TOKEN_OPEN_CURLY || token->type == TOKEN_CLOSE_CURLY) { /* reset current statement details on statement end, and when entering * a deeper scope. * it is a bit ugly to do this in readToken(), but it makes everything * a lot simpler. */ CurrentStatement.access = ACCESS_UNDEFINED; CurrentStatement.impl = IMPL_UNDEFINED; } }
void Lexer::skipWhitespacesAndComments() { while(skipWhitespaces() || skipBlockComment() || skipComment()) {} }
BOOL ParseLine::ParseFunction(LPWSTR &strCurr, BOOL bRHS) { BOOL bResult = FALSE; skipWhitespaces(strCurr); LPWSTR startOfFunction = strCurr; LPWSTR endOfToken = findEndOfTokenName(strCurr); if (endOfToken == strCurr) return SetErrorMessage(L"Invalid character found", strCurr-m_strLine); if (bRHS) strCopy(RHS.FunctionName, strCurr, endOfToken - strCurr); else strCopy(LHS.FunctionName, strCurr, endOfToken - strCurr); strCurr = endOfToken; skipWhitespaces(strCurr); if (strCurr[0] == '(') { strCurr++; skipWhitespaces(strCurr); if (strCurr[0] != ')') { strCurr--; do { strCurr++; skipWhitespaces(strCurr); if (strCurr[0] == '"') { strCurr++; LPWSTR nextChr = strCurr; do { if (nextChr[0] == '\\') { if (nextChr[1] == '\0') return SetErrorMessage(L"Line continuation by using a backslash is not supported.", strCurr-m_strLine); nextChr += 2; } nextChr = wcspbrk(nextChr, L"\\\""); if (nextChr == NULL) return SetErrorMessage(L"Cannot find end of string", strCurr-m_strLine); } while (nextChr[0] != '"'); if (bRHS) AddParameter(RHS, strCurr, nextChr - strCurr); else AddParameter(LHS, strCurr, nextChr - strCurr); strCurr = nextChr+1; } else { LPWSTR nextChr = wcspbrk(strCurr, L",)"); if (nextChr == NULL) return SetErrorMessage(L"Cannot find next ',' or ')' in function", strCurr-m_strLine); for (nextChr-- ; ((nextChr > strCurr) && (isWhitespace(nextChr[0]))) ; nextChr--); if (nextChr >= strCurr) nextChr++; if (nextChr <= strCurr) return SetErrorMessage(L"Missing parameter", strCurr-m_strLine); if (bRHS) AddParameter(RHS, strCurr, nextChr - strCurr); else AddParameter(LHS, strCurr, nextChr - strCurr); strCurr = nextChr; } skipWhitespaces(strCurr); } while (strCurr[0] == ','); } if (strCurr[0] == '\0') return SetErrorMessage(L"End of line found. Expecting ',' or ')'", strCurr-m_strLine); if (strCurr[0] != ')') return SetErrorMessage(L"Invalid character found. Expecting ',' or ')'", strCurr-m_strLine); strCurr++; bResult = TRUE; } if (bRHS) strCopy(RHS.Function, startOfFunction, strCurr-startOfFunction); else strCopy(LHS.Function, startOfFunction, strCurr-startOfFunction); skipWhitespaces(strCurr); if (strCurr[0] == '=') { if (bRHS) return SetErrorMessage(L"Cannot assign twice. Unexpected '=' found", strCurr-m_strLine); m_bHasRHS = TRUE; strCurr++; LPWSTR endOfFunction = strCurr; return ParseFunction(endOfFunction, TRUE); } if (strCurr[0] == '\0') return TRUE; else if (strCurr[0] == '#') return TRUE; else return SetErrorMessage(L"Invalid character found. Expecting '=' or '('", strCurr-m_strLine); }
std::unique_ptr<Layout> create(RenderBlockFlow& flow) { RenderText& textRenderer = toRenderText(*flow.firstChild()); ASSERT(!textRenderer.firstTextBox()); const RenderStyle& style = flow.style(); const unsigned textLength = textRenderer.textLength(); ETextAlign textAlign = style.textAlign(); float wordTrailingSpaceWidth = style.font().width(TextRun(&space, 1)); LazyLineBreakIterator lineBreakIterator(textRenderer.text(), style.locale()); int nextBreakable = -1; Layout::RunVector runs; unsigned lineCount = 0; unsigned lineEndOffset = 0; while (lineEndOffset < textLength) { lineEndOffset = skipWhitespaces(textRenderer, lineEndOffset, textLength); unsigned lineStartOffset = lineEndOffset; unsigned wordEndOffset = lineEndOffset; LineWidth lineWidth(flow, false, DoNotIndentText); Vector<Run, 4> lineRuns; lineRuns.uncheckedAppend(Run(lineStartOffset, 0)); while (wordEndOffset < textLength) { ASSERT(!isWhitespace(textRenderer.characterAt(wordEndOffset))); bool previousWasSpaceBetweenWords = wordEndOffset > lineStartOffset && isWhitespace(textRenderer.characterAt(wordEndOffset - 1)); unsigned wordStartOffset = previousWasSpaceBetweenWords ? wordEndOffset - 1 : wordEndOffset; ++wordEndOffset; while (wordEndOffset < textLength) { if (wordEndOffset > lineStartOffset && isBreakable(lineBreakIterator, wordEndOffset, nextBreakable, false)) break; ++wordEndOffset; } unsigned wordLength = wordEndOffset - wordStartOffset; bool includeEndSpace = wordEndOffset < textLength && textRenderer.characterAt(wordEndOffset) == ' '; float wordWidth; if (includeEndSpace) wordWidth = textWidth(textRenderer, wordStartOffset, wordLength + 1, lineWidth.committedWidth(), style) - wordTrailingSpaceWidth; else wordWidth = textWidth(textRenderer, wordStartOffset, wordLength, lineWidth.committedWidth(), style); lineWidth.addUncommittedWidth(wordWidth); // Move to the next line if the current one is full and we have something on it. if (!lineWidth.fitsOnLine() && lineWidth.committedWidth()) break; if (wordStartOffset > lineEndOffset) { // There were more than one consecutive whitespace. ASSERT(previousWasSpaceBetweenWords); // Include space to the end of the previous run. lineRuns.last().textLength++; lineRuns.last().right += wordTrailingSpaceWidth; // Start a new run on the same line. lineRuns.append(Run(wordStartOffset + 1, lineRuns.last().right)); } lineWidth.commit(); lineRuns.last().right = lineWidth.committedWidth(); lineRuns.last().textLength = wordEndOffset - lineRuns.last().textOffset; lineEndOffset = wordEndOffset; wordEndOffset = skipWhitespaces(textRenderer, wordEndOffset, textLength); if (!lineWidth.fitsOnLine()) { // The first run on the line overflows. ASSERT(lineRuns.size() == 1); break; } } if (lineStartOffset == lineEndOffset) continue; adjustRunOffsets(lineRuns, textAlign, lineWidth.committedWidth(), lineWidth.availableWidth()); for (unsigned i = 0; i < lineRuns.size(); ++i) runs.append(lineRuns[i]); runs.last().isEndOfLine = true; ++lineCount; } textRenderer.clearNeedsLayout(); return Layout::create(runs, lineCount); }
static void readToken (tokenInfo *const token) { int c; token->type = TOKEN_UNDEFINED; vStringClear (token->string); getNextChar: c = getcFromInputFile (); c = skipWhitespaces (c); token->lineNumber = getInputLineNumber (); token->filePosition = getInputFilePosition (); switch (c) { case EOF: token->type = TOKEN_EOF; break; case '(': token->type = TOKEN_OPEN_PAREN; break; case ')': token->type = TOKEN_CLOSE_PAREN; break; case ';': token->type = TOKEN_SEMICOLON; break; case ',': token->type = TOKEN_COMMA; break; case '.': token->type = TOKEN_PERIOD; break; case ':': token->type = TOKEN_COLON; break; case '{': token->type = TOKEN_OPEN_CURLY; break; case '}': token->type = TOKEN_CLOSE_CURLY; break; case '[': token->type = TOKEN_OPEN_SQUARE; break; case ']': token->type = TOKEN_CLOSE_SQUARE; break; case '=': token->type = TOKEN_EQUAL_SIGN; break; case '\'': case '"': token->type = TOKEN_STRING; parseString (token->string, c); token->lineNumber = getInputLineNumber (); token->filePosition = getInputFilePosition (); break; case '<': { int d = getcFromInputFile (); if (d == '#') { /* <# ... #> multiline comment */ do { c = skipToCharacterInInputFile ('#'); if (c != EOF) { c = getcFromInputFile (); if (c == '>') break; else ungetcToInputFile (c); } } while (c != EOF); goto getNextChar; } else { ungetcToInputFile (d); token->type = TOKEN_UNDEFINED; } break; } case '#': /* comment */ skipSingleComment (); goto getNextChar; break; case '+': case '-': case '*': case '/': case '%': { int d = getcFromInputFile (); if (d != '=') ungetcToInputFile (d); token->type = TOKEN_OPERATOR; break; } case '$': /* variable start */ { int d = getcFromInputFile (); if (! isIdentChar (d)) { ungetcToInputFile (d); token->type = TOKEN_UNDEFINED; } else { parseIdentifier (token->string, d); token->type = TOKEN_VARIABLE; } break; } default: if (! isIdentChar (c)) token->type = TOKEN_UNDEFINED; else { parseIdentifier (token->string, c); if (isTokenFunction (token->string)) token->type = TOKEN_KEYWORD; else token->type = TOKEN_IDENTIFIER; } break; } }