//-------------------------------------------------------------------------------------------------- bool Lexer_t::IsMatch ( parseTree::Token_t::Type_t type ///< The type of token to pull from the file. ) //-------------------------------------------------------------------------------------------------- { switch (type) { case parseTree::Token_t::END_OF_FILE: return (nextChar == EOF); case parseTree::Token_t::OPEN_CURLY: return (nextChar == '{'); case parseTree::Token_t::CLOSE_CURLY: return (nextChar == '}'); case parseTree::Token_t::OPEN_PARENTHESIS: return (nextChar == '('); case parseTree::Token_t::CLOSE_PARENTHESIS: return (nextChar == ')'); case parseTree::Token_t::COLON: return (nextChar == ':'); case parseTree::Token_t::EQUALS: return (nextChar == '='); case parseTree::Token_t::DOT: return (nextChar == '.'); case parseTree::Token_t::STAR: return (nextChar == '*'); case parseTree::Token_t::ARROW: return ((nextChar == '-') && (inputStream.peek() == '>')); case parseTree::Token_t::WHITESPACE: return IsWhitespace(nextChar); case parseTree::Token_t::COMMENT: if (nextChar == '/') { int secondChar = inputStream.peek(); return ((secondChar == '/') || (secondChar == '*')); } else { return false; } case parseTree::Token_t::FILE_PERMISSIONS: case parseTree::Token_t::SERVER_IPC_OPTION: case parseTree::Token_t::CLIENT_IPC_OPTION: return (nextChar == '['); case parseTree::Token_t::ARG: // Can be anything in a FILE_PATH, plus the equals sign (=). if (nextChar == '=') { return true; } // *** FALL THROUGH *** case parseTree::Token_t::FILE_PATH: // Can be anything in a FILE_NAME, plus the forward slash (/). // If it starts with a slash, it could be a comment or a file path. if (nextChar == '/') { // If it's not a comment, then it's a file path. int secondChar = inputStream.peek(); return ((secondChar != '/') && (secondChar != '*')); } // *** FALL THROUGH *** case parseTree::Token_t::FILE_NAME: return ( IsFileNameChar(nextChar) || (nextChar == '\'') // Could be in single-quotes. || (nextChar == '"') ); // Could be in quotes. case parseTree::Token_t::IPC_AGENT: // Can start with the same characters as a NAME or GROUP_NAME, plus '<'. if (nextChar == '<') { return true; } // *** FALL THROUGH *** case parseTree::Token_t::NAME: case parseTree::Token_t::GROUP_NAME: return ( islower(nextChar) || isupper(nextChar) || (nextChar == '_') ); case parseTree::Token_t::INTEGER: return (isdigit(nextChar)); case parseTree::Token_t::SIGNED_INTEGER: return ( (nextChar == '+') || (nextChar == '-') || isdigit(nextChar)); case parseTree::Token_t::BOOLEAN: throw mk::Exception_t("Internal bug: BOOLEAN lookahead not implemented."); case parseTree::Token_t::FLOAT: throw mk::Exception_t("Internal bug: FLOAT lookahead not implemented."); case parseTree::Token_t::STRING: throw mk::Exception_t("Internal bug: STRING lookahead not implemented."); } throw mk::Exception_t("Internal bug: IsMatch(): Invalid token type requested."); }
void Tokenizer::SkipSpaces() { while (Good() && IsWhitespace(PeekNextChar())) { GetNextChar(); } }
static int Demux( demux_t *p_demux ) { int i_ret = -1; xml_reader_t *p_xml_reader = NULL; char *psz_elname = NULL; const char *node; input_item_t *p_input; char *psz_mrl = NULL, *psz_title = NULL, *psz_genre = NULL; char *psz_now = NULL, *psz_listeners = NULL, *psz_bitrate = NULL; input_item_node_t *p_subitems = NULL; input_item_t *p_current_input = GetCurrentItem(p_demux); free( stream_ReadLine( p_demux->s ) ); p_xml_reader = xml_ReaderCreate( p_demux, p_demux->s ); if( !p_xml_reader ) return -1; /* xml */ /* check root node */ if( xml_ReaderNextNode( p_xml_reader, &node ) != XML_READER_STARTELEM ) { msg_Err( p_demux, "invalid file (no root node)" ); goto end; } if( strcmp( node, "WinampXML" ) ) { msg_Err( p_demux, "invalid root node: %s", node ); goto end; } /* root node should not have any attributes, and should only * contain the "playlist node */ /* Skip until 1st child node */ while( (i_ret = xml_ReaderNextNode( p_xml_reader, &node )) != XML_READER_STARTELEM ) if( i_ret <= 0 ) { msg_Err( p_demux, "invalid file (no child node)" ); goto end; } if( strcmp( node, "playlist" ) ) { msg_Err( p_demux, "invalid child node %s", node ); goto end; } // Read the attributes const char *attr, *value; while( (attr = xml_ReaderNextAttr( p_xml_reader, &value )) != NULL ) { if( !strcmp( attr, "num_entries" ) ) msg_Dbg( p_demux, "playlist has %d entries", atoi(value) ); else if( !strcmp( attr, "label" ) ) input_item_SetName( p_current_input, value ); else msg_Warn( p_demux, "stray attribute %s with value %s in element" " <playlist>", attr, value ); } p_subitems = input_item_node_Create( p_current_input ); while( (i_ret = xml_ReaderNextNode( p_xml_reader, &node )) > 0 ) { // Get the node type switch( i_ret ) { case XML_READER_STARTELEM: { // Read the element name free( psz_elname ); psz_elname = strdup( node ); if( unlikely(!psz_elname) ) goto end; // Read the attributes while( (attr = xml_ReaderNextAttr( p_xml_reader, &value )) ) { if( !strcmp( psz_elname, "entry" ) && !strcmp( attr, "Playstring" ) ) { free( psz_mrl ); psz_mrl = strdup( value ); } else { msg_Warn( p_demux, "unexpected attribute %s in <%s>", attr, psz_elname ); } } break; } case XML_READER_TEXT: { char **p; if( psz_elname == NULL ) break; if( IsWhitespace( node ) ) break; if( !strcmp( psz_elname, "Name" ) ) p = &psz_title; else if( !strcmp( psz_elname, "Genre" ) ) p = &psz_genre; else if( !strcmp( psz_elname, "Nowplaying" ) ) p = &psz_now; else if( !strcmp( psz_elname, "Listeners" ) ) p = &psz_listeners; else if( !strcmp( psz_elname, "Bitrate" ) ) p = &psz_bitrate; else { msg_Warn( p_demux, "unexpected text in element <%s>", psz_elname ); break; } free( *p ); *p = strdup( node ); break; } // End element case XML_READER_ENDELEM: { // Read the element name if( !strcmp( node, "entry" ) ) { p_input = input_item_New( psz_mrl, psz_title ); if( psz_now ) input_item_SetNowPlaying( p_input, psz_now ); if( psz_genre ) input_item_SetGenre( p_input, psz_genre ); if( psz_listeners ) msg_Err( p_demux, "Unsupported meta listeners" ); if( psz_bitrate ) msg_Err( p_demux, "Unsupported meta bitrate" ); input_item_node_AppendItem( p_subitems, p_input ); vlc_gc_decref( p_input ); FREENULL( psz_title ); FREENULL( psz_mrl ); FREENULL( psz_genre ); FREENULL( psz_bitrate ); FREENULL( psz_listeners ); FREENULL( psz_now ); } FREENULL( psz_elname ); break; } } } if( i_ret < 0 ) { msg_Warn( p_demux, "error while parsing data" ); i_ret = 0; /* Needed for correct operation of go back */ } end: free( psz_elname ); if( p_subitems ) input_item_node_PostAndDelete( p_subitems ); vlc_gc_decref( p_current_input ); if( p_xml_reader ) xml_ReaderDelete( p_xml_reader ); return i_ret; }
/** * Lexical analyzing. Break up the input stream into elements. * * @param in The stream to parse from. * @param tokenStream The token stream that gets filled up. * * @return the lex result from the action. */ LexResult Parser::Lex(std::istream& in, std::vector<Element>& tokenStream) { size_t const NUMBER_OF_LEXABLES = this->lexables.size(); // Create the character stream. CharacterStream cs(in); // Clear out the existing contents first. tokenStream.clear(); // Store a local result to add errors to. LexResult result; // Loop over all characters. while(cs.HasCharacters()) { // Consume White Space. char c = cs.NextCharacter(); if (IsWhitespace(c)) { cs.Consume(); continue; } cs.Rollback1(); c = cs.NextCharacter(); if ( c == '/') { c = cs.NextCharacter(); if (c == '/') { cs.Consume(); while(cs.HasCharacters()) { c = cs.NextCharacter(); cs.Consume(); if (c == '\n' || c == '\r') { break; } } continue; } else { cs.Rollback1(); cs.Rollback1(); } } else { cs.Rollback1(); } bool got_one = false; int row = cs.Row(); int column = cs.Column(); // Iterate through each lexable, figure out which one is current. for(size_t i=0; i<NUMBER_OF_LEXABLES; ++i) { std::shared_ptr<LexableElement_> lexable = this->lexables[i]; // Element to read in. strine::Element element; bool success = lexable->Process(cs, element); if (success) { element.SetSourceLocation(SourceLocation(row, column)); tokenStream.push_back(element); got_one = true; break; } } if (false == got_one) { result.AddError("Unrecognized sequence of lexable characters."); return result; } } result.SetSuccess(true); return result; }
// Tokenise a user's typed expression string void CExpression::ParseUserString(const char* exp, int* pchpos, bool editorMode) { toks.clear(); whitespaceCount = 0; // Loop every char for (int i = 0; exp[i] != NULL; i++) { if (pchpos) *pchpos = i; char NextCh = exp[i+1]; char CurCh = exp[i]; char PrevCh = exp[(i == 0 ? 0 : i-1)]; char strbuf[128]; // Check this char switch (exp[i]) { case '+': AppendToken(T_ADD, "+"); break; case '-': // If previous token is not operand, use as negative sign // Fix 25/4/07: random(4) - random(5) interprets '-' as integer: explicit check for right bracket if (toks.size() == 0 || ((!TokenFitsRule(toks.back().t, T_ANY_VALUE) || toks.back().t == T_LEFTBRACKET) && toks.back().t != T_RIGHTBRACKET)) { i += ConsumeDigits(strbuf, &(exp[i])); // Contains a dot: float if (StrContains(strbuf, '.')) AppendToken(T_FLOAT, strbuf); else // Else integer AppendToken(T_INTEGER, strbuf); } else AppendToken(T_SUBTRACT, "-"); break; case '*': AppendToken(T_MULTIPLY, "*"); break; case '/': AppendToken(T_DIVIDE, "/"); break; case '^': AppendToken(T_POWER, "^"); break; case '%': AppendToken(T_MOD, "%"); break; case '(': AppendToken(T_LEFTBRACKET, "("); break; case ')': AppendToken(T_RIGHTBRACKET, ")"); break; case '{': AppendToken(T_LEFTCURLY, "{"); break; case '}': AppendToken(T_RIGHTCURLY, "}"); break; case '@': AppendToken(T_AT, "@"); break; case ',': AppendToken(T_COMMA, ","); break; case '.': AppendToken(T_DOT, "."); break; case '"': i += AppendString(&(exp[i]), editorMode); break; case '\'': i += AppendString(&(exp[i]), editorMode, true); break; case '=': AppendToken(T_EQUAL, "="); break; case '<': if (NextCh == '=') { AppendToken(T_LESSEQUAL, "<="); i++; } else if (NextCh == '>') { AppendToken(T_NOTEQUAL, "<>"); i++; } else AppendToken(T_LESS, "<"); break; // Alternative not equal operator != <> case '!': if (NextCh == '=') { AppendToken(T_NOTEQUAL, "!="); i++; } else NotifyError("Syntax error: '!'"); break; case '&': AppendToken(T_AND, "&"); break; case '|': AppendToken(T_OR, "|"); break; case '>': if (NextCh == '=') { AppendToken(T_GREATEREQUAL, ">="); i++; } else AppendToken(T_GREATER, ">"); break; case '?': AppendToken(T_CONDITIONAL, "?"); break; case ':': AppendToken(T_COLON, ":"); break; default: // Parse numbers and idents if (IsChAlphaNumeric(CurCh)) i += ConsumeAppendIdentifier(strbuf, &(exp[i])); // Skip over whitespace else if (IsWhitespace(CurCh)) { // In editor mode add a T_WHITESPACE token if (editorMode) { Token t; t.length = 0; t.t = T_WHITESPACE; while (IsWhitespace(exp[i])) { t.str += exp[i]; i++; } // We want the next for loop iteration to see the next char i--; toks.push_back(t); } // Add to last token length if (toks.size() > 0) toks[toks.size() - 1].length++; } // Else unknown character else { string error = "Unknown character '"; error += CurCh; error += "'"; throw runtime_error(error.c_str()); } }//switch }//for // Only perform preprocessing when not in edittime mode if (!editorMode) PreProcessTokStream(); /////////////////////////// // Final step: parse to tree // Determine if the overall expression is constant, collapsing constant parts along // the way. // Runtime only #ifdef RUNTIME // Make the tree int step = 0; ExpTree = DoTree(step); isConstant = ExpTree->Constant(); // If constant, get the constant value. if (isConstant) { CollapseConstantTree(ExpTree); ExpTree->Evaluate(&constantValue); } #endif }
vector<Token> LexFileContents(char* fileContents, int fileLength){ Token currToken; currToken.start = fileContents; currToken.length = 0; currToken.type = SourceTokenType::EMPTY; vector<Token> tokens; for(int i = 0; i < fileLength; i++){ char currChar = fileContents[i]; bool isWhitespace = IsWhitespace(currChar); bool isQuote = (currChar == '"'); bool isOperator = IsOperator(currChar); bool isNumber = IsNumber(currChar); switch (currToken.type){ case SourceTokenType::EMPTY:{ if(!isWhitespace){ currToken.start = &fileContents[i]; currToken.length = 1; if(isQuote){ currToken.start++; currToken.length = 0; currToken.type = SourceTokenType::STRING; } else if(isOperator){ currToken.type = SourceTokenType::OPERATOR; } else if(isNumber){ currToken.type = SourceTokenType::NUMBER; } else{ currToken.type = SourceTokenType::IDENTIFIER; } } }break; case SourceTokenType::NUMBER: case SourceTokenType::IDENTIFIER:{ if(isWhitespace){ tokens.push_back(currToken); currToken.length = 0; currToken.type = SourceTokenType::EMPTY; } else if(isOperator){ tokens.push_back(currToken); currToken.start = &fileContents[i]; currToken.length = 1; currToken.type = SourceTokenType::OPERATOR; } else{ currToken.length++; } }break; case SourceTokenType::STRING:{ if(isQuote){ tokens.push_back(currToken); currToken.length = 0; currToken.type = SourceTokenType::EMPTY; } else{ currToken.length++; } }break; case SourceTokenType::OPERATOR:{ tokens.push_back(currToken); currToken.length = 1; currToken.start = &fileContents[i]; if(isWhitespace){ currToken.length = 0; currToken.type = SourceTokenType::EMPTY; } else if(isNumber){ currToken.type = SourceTokenType::NUMBER; } else if(isOperator){ currToken.type = SourceTokenType::OPERATOR; } else if(isQuote){ currToken.start++; currToken.length = 0; currToken.type = SourceTokenType::STRING; } else{ currToken.type = SourceTokenType::IDENTIFIER; } }break; default: break; } } return tokens; }
void CppScanner::SkipWhitespace() { while (IsWhitespace(f.Get())) f.GetNext(); }
// // Skip past all spaces and tabs in the input stream. // TCHAR FBaseParser::GetLeadingChar() { TCHAR TrailingCommentNewline = 0; for (;;) { bool MultipleNewlines = false; TCHAR c; // Skip blanks. do { c = GetChar(); // Check if we've encountered another newline since the last one if (c == TrailingCommentNewline) { MultipleNewlines = true; } } while (IsWhitespace(c)); if (c != TEXT('/') || PeekChar() != TEXT('/')) { return c; } // Clear the comment if we've encountered newlines since the last comment if (MultipleNewlines) { ClearComment(); } // Record the first slash. The first iteration of the loop will get the second slash. PrevComment += c; do { c = GetChar(true); if (c == 0) return c; PrevComment += c; } while (!IsEOL(c)); TrailingCommentNewline = c; for (;;) { c = GetChar(); if (c == 0) return c; if (c == TrailingCommentNewline || !IsEOL(c)) { UngetChar(); break; } PrevComment += c; } } }
/** * Construct a new line with a maximum width. * @param max_width The maximum width of the string. * @return A Line, or NULL when at the end of the paragraph. */ const ParagraphLayouter::Line *FallbackParagraphLayout::NextLine(int max_width) { /* Simple idea: * - split a line at a newline character, or at a space where we can break a line. * - split for a visual run whenever a new line happens, or the font changes. */ if (this->buffer == NULL) return NULL; FallbackLine *l = new FallbackLine(); if (*this->buffer == '\0') { /* Only a newline. */ this->buffer = NULL; *l->Append() = new FallbackVisualRun(this->runs.Begin()->second, this->buffer, 0, 0); return l; } const WChar *begin = this->buffer; const WChar *last_space = NULL; const WChar *last_char = begin; int width = 0; int offset = this->buffer - this->buffer_begin; FontMap::iterator iter = this->runs.Begin(); while (iter->first <= offset) { iter++; assert(iter != this->runs.End()); } const FontCache *fc = iter->second->fc; const WChar *next_run = this->buffer_begin + iter->first; for (;;) { WChar c = *this->buffer; last_char = this->buffer; if (c == '\0') { this->buffer = NULL; break; } if (this->buffer == next_run) { int w = l->GetWidth(); *l->Append() = new FallbackVisualRun(iter->second, begin, this->buffer - begin, w); iter++; assert(iter != this->runs.End()); next_run = this->buffer_begin + iter->first; begin = this->buffer; last_space = NULL; } if (IsWhitespace(c)) last_space = this->buffer; if (IsPrintable(c) && !IsTextDirectionChar(c)) { int char_width = GetCharacterWidth(fc->GetSize(), c); width += char_width; if (width > max_width) { /* The string is longer than maximum width so we need to decide * what to do with it. */ if (width == char_width) { /* The character is wider than allowed width; don't know * what to do with this case... bail out! */ this->buffer = NULL; return l; } if (last_space == NULL) { /* No space has been found. Just terminate at our current * location. This usually happens for languages that do not * require spaces in strings, like Chinese, Japanese and * Korean. For other languages terminating mid-word might * not be the best, but terminating the whole string instead * of continuing the word at the next line is worse. */ last_char = this->buffer; } else { /* A space is found; perfect place to terminate */ this->buffer = last_space + 1; last_char = last_space; } break; } } this->buffer++; } if (l->Length() == 0 || last_char - begin != 0) { int w = l->GetWidth(); *l->Append() = new FallbackVisualRun(iter->second, begin, last_char - begin, w); } return l; }