//--------------------------------------------------------------------------------------------------
bool Lexer_t::IsMatch
(
parseTree::Token_t::Type_t type ///< The type of token to pull from the file.
)
//--------------------------------------------------------------------------------------------------
{
switch (type)
{
case parseTree::Token_t::END_OF_FILE:
return (nextChar == EOF);
case parseTree::Token_t::OPEN_CURLY:
return (nextChar == '{');
case parseTree::Token_t::CLOSE_CURLY:
return (nextChar == '}');
case parseTree::Token_t::OPEN_PARENTHESIS:
return (nextChar == '(');
case parseTree::Token_t::CLOSE_PARENTHESIS:
return (nextChar == ')');
case parseTree::Token_t::COLON:
return (nextChar == ':');
case parseTree::Token_t::EQUALS:
return (nextChar == '=');
case parseTree::Token_t::DOT:
return (nextChar == '.');
case parseTree::Token_t::STAR:
return (nextChar == '*');
case parseTree::Token_t::ARROW:
return ((nextChar == '-') && (inputStream.peek() == '>'));
case parseTree::Token_t::WHITESPACE:
return IsWhitespace(nextChar);
case parseTree::Token_t::COMMENT:
if (nextChar == '/')
{
int secondChar = inputStream.peek();
return ((secondChar == '/') || (secondChar == '*'));
}
else
{
return false;
}
case parseTree::Token_t::FILE_PERMISSIONS:
case parseTree::Token_t::SERVER_IPC_OPTION:
case parseTree::Token_t::CLIENT_IPC_OPTION:
return (nextChar == '[');
case parseTree::Token_t::ARG:
// Can be anything in a FILE_PATH, plus the equals sign (=).
if (nextChar == '=')
{
return true;
}
// *** FALL THROUGH ***
case parseTree::Token_t::FILE_PATH:
// Can be anything in a FILE_NAME, plus the forward slash (/).
// If it starts with a slash, it could be a comment or a file path.
if (nextChar == '/')
{
// If it's not a comment, then it's a file path.
int secondChar = inputStream.peek();
return ((secondChar != '/') && (secondChar != '*'));
}
// *** FALL THROUGH ***
case parseTree::Token_t::FILE_NAME:
return ( IsFileNameChar(nextChar)
|| (nextChar == '\'') // Could be in single-quotes.
|| (nextChar == '"') ); // Could be in quotes.
case parseTree::Token_t::IPC_AGENT:
// Can start with the same characters as a NAME or GROUP_NAME, plus '<'.
if (nextChar == '<')
{
return true;
}
// *** FALL THROUGH ***
case parseTree::Token_t::NAME:
case parseTree::Token_t::GROUP_NAME:
return ( islower(nextChar)
|| isupper(nextChar)
|| (nextChar == '_') );
case parseTree::Token_t::INTEGER:
return (isdigit(nextChar));
case parseTree::Token_t::SIGNED_INTEGER:
return ( (nextChar == '+')
|| (nextChar == '-')
|| isdigit(nextChar));
case parseTree::Token_t::BOOLEAN:
throw mk::Exception_t("Internal bug: BOOLEAN lookahead not implemented.");
case parseTree::Token_t::FLOAT:
throw mk::Exception_t("Internal bug: FLOAT lookahead not implemented.");
case parseTree::Token_t::STRING:
throw mk::Exception_t("Internal bug: STRING lookahead not implemented.");
}
throw mk::Exception_t("Internal bug: IsMatch(): Invalid token type requested.");
}
void Tokenizer::SkipSpaces()
{
while (Good() && IsWhitespace(PeekNextChar())) {
GetNextChar();
}
}
static int Demux( demux_t *p_demux )
{
int i_ret = -1;
xml_reader_t *p_xml_reader = NULL;
char *psz_elname = NULL;
const char *node;
input_item_t *p_input;
char *psz_mrl = NULL, *psz_title = NULL, *psz_genre = NULL;
char *psz_now = NULL, *psz_listeners = NULL, *psz_bitrate = NULL;
input_item_node_t *p_subitems = NULL;
input_item_t *p_current_input = GetCurrentItem(p_demux);
free( stream_ReadLine( p_demux->s ) );
p_xml_reader = xml_ReaderCreate( p_demux, p_demux->s );
if( !p_xml_reader )
return -1;
/* xml */
/* check root node */
if( xml_ReaderNextNode( p_xml_reader, &node ) != XML_READER_STARTELEM )
{
msg_Err( p_demux, "invalid file (no root node)" );
goto end;
}
if( strcmp( node, "WinampXML" ) )
{
msg_Err( p_demux, "invalid root node: %s", node );
goto end;
}
/* root node should not have any attributes, and should only
* contain the "playlist node */
/* Skip until 1st child node */
while( (i_ret = xml_ReaderNextNode( p_xml_reader, &node )) != XML_READER_STARTELEM )
if( i_ret <= 0 )
{
msg_Err( p_demux, "invalid file (no child node)" );
goto end;
}
if( strcmp( node, "playlist" ) )
{
msg_Err( p_demux, "invalid child node %s", node );
goto end;
}
// Read the attributes
const char *attr, *value;
while( (attr = xml_ReaderNextAttr( p_xml_reader, &value )) != NULL )
{
if( !strcmp( attr, "num_entries" ) )
msg_Dbg( p_demux, "playlist has %d entries", atoi(value) );
else if( !strcmp( attr, "label" ) )
input_item_SetName( p_current_input, value );
else
msg_Warn( p_demux, "stray attribute %s with value %s in element"
" <playlist>", attr, value );
}
p_subitems = input_item_node_Create( p_current_input );
while( (i_ret = xml_ReaderNextNode( p_xml_reader, &node )) > 0 )
{
// Get the node type
switch( i_ret )
{
case XML_READER_STARTELEM:
{
// Read the element name
free( psz_elname );
psz_elname = strdup( node );
if( unlikely(!psz_elname) )
goto end;
// Read the attributes
while( (attr = xml_ReaderNextAttr( p_xml_reader, &value )) )
{
if( !strcmp( psz_elname, "entry" ) &&
!strcmp( attr, "Playstring" ) )
{
free( psz_mrl );
psz_mrl = strdup( value );
}
else
{
msg_Warn( p_demux, "unexpected attribute %s in <%s>",
attr, psz_elname );
}
}
break;
}
case XML_READER_TEXT:
{
char **p;
if( psz_elname == NULL )
break;
if( IsWhitespace( node ) )
break;
if( !strcmp( psz_elname, "Name" ) )
p = &psz_title;
else if( !strcmp( psz_elname, "Genre" ) )
p = &psz_genre;
else if( !strcmp( psz_elname, "Nowplaying" ) )
p = &psz_now;
else if( !strcmp( psz_elname, "Listeners" ) )
p = &psz_listeners;
else if( !strcmp( psz_elname, "Bitrate" ) )
p = &psz_bitrate;
else
{
msg_Warn( p_demux, "unexpected text in element <%s>",
psz_elname );
break;
}
free( *p );
*p = strdup( node );
break;
}
// End element
case XML_READER_ENDELEM:
{
// Read the element name
if( !strcmp( node, "entry" ) )
{
p_input = input_item_New( psz_mrl, psz_title );
if( psz_now )
input_item_SetNowPlaying( p_input, psz_now );
if( psz_genre )
input_item_SetGenre( p_input, psz_genre );
if( psz_listeners )
msg_Err( p_demux, "Unsupported meta listeners" );
if( psz_bitrate )
msg_Err( p_demux, "Unsupported meta bitrate" );
input_item_node_AppendItem( p_subitems, p_input );
vlc_gc_decref( p_input );
FREENULL( psz_title );
FREENULL( psz_mrl );
FREENULL( psz_genre );
FREENULL( psz_bitrate );
FREENULL( psz_listeners );
FREENULL( psz_now );
}
FREENULL( psz_elname );
break;
}
}
}
if( i_ret < 0 )
{
msg_Warn( p_demux, "error while parsing data" );
i_ret = 0; /* Needed for correct operation of go back */
}
end:
free( psz_elname );
if( p_subitems )
input_item_node_PostAndDelete( p_subitems );
vlc_gc_decref( p_current_input );
if( p_xml_reader )
xml_ReaderDelete( p_xml_reader );
return i_ret;
}
/**
* Lexical analyzing. Break up the input stream into elements.
*
* @param in The stream to parse from.
* @param tokenStream The token stream that gets filled up.
*
* @return the lex result from the action.
*/
LexResult Parser::Lex(std::istream& in, std::vector<Element>& tokenStream)
{
size_t const NUMBER_OF_LEXABLES = this->lexables.size();
// Create the character stream.
CharacterStream cs(in);
// Clear out the existing contents first.
tokenStream.clear();
// Store a local result to add errors to.
LexResult result;
// Loop over all characters.
while(cs.HasCharacters())
{
// Consume White Space.
char c = cs.NextCharacter();
if (IsWhitespace(c))
{
cs.Consume();
continue;
}
cs.Rollback1();
c = cs.NextCharacter();
if ( c == '/')
{
c = cs.NextCharacter();
if (c == '/')
{
cs.Consume();
while(cs.HasCharacters())
{
c = cs.NextCharacter();
cs.Consume();
if (c == '\n' || c == '\r')
{
break;
}
}
continue;
}
else
{
cs.Rollback1();
cs.Rollback1();
}
}
else
{
cs.Rollback1();
}
bool got_one = false;
int row = cs.Row();
int column = cs.Column();
// Iterate through each lexable, figure out which one is current.
for(size_t i=0; i<NUMBER_OF_LEXABLES; ++i)
{
std::shared_ptr<LexableElement_> lexable = this->lexables[i];
// Element to read in.
strine::Element element;
bool success = lexable->Process(cs, element);
if (success)
{
element.SetSourceLocation(SourceLocation(row, column));
tokenStream.push_back(element);
got_one = true;
break;
}
}
if (false == got_one)
{
result.AddError("Unrecognized sequence of lexable characters.");
return result;
}
}
result.SetSuccess(true);
return result;
}
// Tokenise a user's typed expression string
void CExpression::ParseUserString(const char* exp, int* pchpos, bool editorMode)
{
toks.clear();
whitespaceCount = 0;
// Loop every char
for (int i = 0; exp[i] != NULL; i++) {
if (pchpos) *pchpos = i;
char NextCh = exp[i+1];
char CurCh = exp[i];
char PrevCh = exp[(i == 0 ? 0 : i-1)];
char strbuf[128];
// Check this char
switch (exp[i]) {
case '+':
AppendToken(T_ADD, "+");
break;
case '-':
// If previous token is not operand, use as negative sign
// Fix 25/4/07: random(4) - random(5) interprets '-' as integer: explicit check for right bracket
if (toks.size() == 0 || ((!TokenFitsRule(toks.back().t, T_ANY_VALUE) || toks.back().t == T_LEFTBRACKET) && toks.back().t != T_RIGHTBRACKET)) {
i += ConsumeDigits(strbuf, &(exp[i]));
// Contains a dot: float
if (StrContains(strbuf, '.'))
AppendToken(T_FLOAT, strbuf);
else // Else integer
AppendToken(T_INTEGER, strbuf);
}
else
AppendToken(T_SUBTRACT, "-");
break;
case '*':
AppendToken(T_MULTIPLY, "*");
break;
case '/':
AppendToken(T_DIVIDE, "/");
break;
case '^':
AppendToken(T_POWER, "^");
break;
case '%':
AppendToken(T_MOD, "%");
break;
case '(':
AppendToken(T_LEFTBRACKET, "(");
break;
case ')':
AppendToken(T_RIGHTBRACKET, ")");
break;
case '{':
AppendToken(T_LEFTCURLY, "{");
break;
case '}':
AppendToken(T_RIGHTCURLY, "}");
break;
case '@':
AppendToken(T_AT, "@");
break;
case ',':
AppendToken(T_COMMA, ",");
break;
case '.':
AppendToken(T_DOT, ".");
break;
case '"':
i += AppendString(&(exp[i]), editorMode);
break;
case '\'':
i += AppendString(&(exp[i]), editorMode, true);
break;
case '=':
AppendToken(T_EQUAL, "=");
break;
case '<':
if (NextCh == '=') {
AppendToken(T_LESSEQUAL, "<=");
i++;
}
else if (NextCh == '>') {
AppendToken(T_NOTEQUAL, "<>");
i++;
}
else
AppendToken(T_LESS, "<");
break;
// Alternative not equal operator != <>
case '!':
if (NextCh == '=') {
AppendToken(T_NOTEQUAL, "!=");
i++;
}
else
NotifyError("Syntax error: '!'");
break;
case '&':
AppendToken(T_AND, "&");
break;
case '|':
AppendToken(T_OR, "|");
break;
case '>':
if (NextCh == '=') {
AppendToken(T_GREATEREQUAL, ">=");
i++;
}
else
AppendToken(T_GREATER, ">");
break;
case '?':
AppendToken(T_CONDITIONAL, "?");
break;
case ':':
AppendToken(T_COLON, ":");
break;
default:
// Parse numbers and idents
if (IsChAlphaNumeric(CurCh))
i += ConsumeAppendIdentifier(strbuf, &(exp[i]));
// Skip over whitespace
else if (IsWhitespace(CurCh)) {
// In editor mode add a T_WHITESPACE token
if (editorMode) {
Token t;
t.length = 0;
t.t = T_WHITESPACE;
while (IsWhitespace(exp[i])) {
t.str += exp[i];
i++;
}
// We want the next for loop iteration to see the next char
i--;
toks.push_back(t);
}
// Add to last token length
if (toks.size() > 0)
toks[toks.size() - 1].length++;
}
// Else unknown character
else {
string error = "Unknown character '";
error += CurCh;
error += "'";
throw runtime_error(error.c_str());
}
}//switch
}//for
// Only perform preprocessing when not in edittime mode
if (!editorMode)
PreProcessTokStream();
///////////////////////////
// Final step: parse to tree
// Determine if the overall expression is constant, collapsing constant parts along
// the way.
// Runtime only
#ifdef RUNTIME
// Make the tree
int step = 0;
ExpTree = DoTree(step);
isConstant = ExpTree->Constant();
// If constant, get the constant value.
if (isConstant) {
CollapseConstantTree(ExpTree);
ExpTree->Evaluate(&constantValue);
}
#endif
}
vector<Token> LexFileContents(char* fileContents, int fileLength){
Token currToken;
currToken.start = fileContents;
currToken.length = 0;
currToken.type = SourceTokenType::EMPTY;
vector<Token> tokens;
for(int i = 0; i < fileLength; i++){
char currChar = fileContents[i];
bool isWhitespace = IsWhitespace(currChar);
bool isQuote = (currChar == '"');
bool isOperator = IsOperator(currChar);
bool isNumber = IsNumber(currChar);
switch (currToken.type){
case SourceTokenType::EMPTY:{
if(!isWhitespace){
currToken.start = &fileContents[i];
currToken.length = 1;
if(isQuote){
currToken.start++;
currToken.length = 0;
currToken.type = SourceTokenType::STRING;
}
else if(isOperator){
currToken.type = SourceTokenType::OPERATOR;
}
else if(isNumber){
currToken.type = SourceTokenType::NUMBER;
}
else{
currToken.type = SourceTokenType::IDENTIFIER;
}
}
}break;
case SourceTokenType::NUMBER:
case SourceTokenType::IDENTIFIER:{
if(isWhitespace){
tokens.push_back(currToken);
currToken.length = 0;
currToken.type = SourceTokenType::EMPTY;
}
else if(isOperator){
tokens.push_back(currToken);
currToken.start = &fileContents[i];
currToken.length = 1;
currToken.type = SourceTokenType::OPERATOR;
}
else{
currToken.length++;
}
}break;
case SourceTokenType::STRING:{
if(isQuote){
tokens.push_back(currToken);
currToken.length = 0;
currToken.type = SourceTokenType::EMPTY;
}
else{
currToken.length++;
}
}break;
case SourceTokenType::OPERATOR:{
tokens.push_back(currToken);
currToken.length = 1;
currToken.start = &fileContents[i];
if(isWhitespace){
currToken.length = 0;
currToken.type = SourceTokenType::EMPTY;
}
else if(isNumber){
currToken.type = SourceTokenType::NUMBER;
}
else if(isOperator){
currToken.type = SourceTokenType::OPERATOR;
}
else if(isQuote){
currToken.start++;
currToken.length = 0;
currToken.type = SourceTokenType::STRING;
}
else{
currToken.type = SourceTokenType::IDENTIFIER;
}
}break;
default:
break;
}
}
return tokens;
}
void CppScanner::SkipWhitespace()
{
while (IsWhitespace(f.Get())) f.GetNext();
}
//
// Skip past all spaces and tabs in the input stream.
//
TCHAR FBaseParser::GetLeadingChar()
{
TCHAR TrailingCommentNewline = 0;
for (;;)
{
bool MultipleNewlines = false;
TCHAR c;
// Skip blanks.
do
{
c = GetChar();
// Check if we've encountered another newline since the last one
if (c == TrailingCommentNewline)
{
MultipleNewlines = true;
}
} while (IsWhitespace(c));
if (c != TEXT('/') || PeekChar() != TEXT('/'))
{
return c;
}
// Clear the comment if we've encountered newlines since the last comment
if (MultipleNewlines)
{
ClearComment();
}
// Record the first slash. The first iteration of the loop will get the second slash.
PrevComment += c;
do
{
c = GetChar(true);
if (c == 0)
return c;
PrevComment += c;
} while (!IsEOL(c));
TrailingCommentNewline = c;
for (;;)
{
c = GetChar();
if (c == 0)
return c;
if (c == TrailingCommentNewline || !IsEOL(c))
{
UngetChar();
break;
}
PrevComment += c;
}
}
}
/**
* Construct a new line with a maximum width.
* @param max_width The maximum width of the string.
* @return A Line, or NULL when at the end of the paragraph.
*/
const ParagraphLayouter::Line *FallbackParagraphLayout::NextLine(int max_width)
{
/* Simple idea:
* - split a line at a newline character, or at a space where we can break a line.
* - split for a visual run whenever a new line happens, or the font changes.
*/
if (this->buffer == NULL) return NULL;
FallbackLine *l = new FallbackLine();
if (*this->buffer == '\0') {
/* Only a newline. */
this->buffer = NULL;
*l->Append() = new FallbackVisualRun(this->runs.Begin()->second, this->buffer, 0, 0);
return l;
}
const WChar *begin = this->buffer;
const WChar *last_space = NULL;
const WChar *last_char = begin;
int width = 0;
int offset = this->buffer - this->buffer_begin;
FontMap::iterator iter = this->runs.Begin();
while (iter->first <= offset) {
iter++;
assert(iter != this->runs.End());
}
const FontCache *fc = iter->second->fc;
const WChar *next_run = this->buffer_begin + iter->first;
for (;;) {
WChar c = *this->buffer;
last_char = this->buffer;
if (c == '\0') {
this->buffer = NULL;
break;
}
if (this->buffer == next_run) {
int w = l->GetWidth();
*l->Append() = new FallbackVisualRun(iter->second, begin, this->buffer - begin, w);
iter++;
assert(iter != this->runs.End());
next_run = this->buffer_begin + iter->first;
begin = this->buffer;
last_space = NULL;
}
if (IsWhitespace(c)) last_space = this->buffer;
if (IsPrintable(c) && !IsTextDirectionChar(c)) {
int char_width = GetCharacterWidth(fc->GetSize(), c);
width += char_width;
if (width > max_width) {
/* The string is longer than maximum width so we need to decide
* what to do with it. */
if (width == char_width) {
/* The character is wider than allowed width; don't know
* what to do with this case... bail out! */
this->buffer = NULL;
return l;
}
if (last_space == NULL) {
/* No space has been found. Just terminate at our current
* location. This usually happens for languages that do not
* require spaces in strings, like Chinese, Japanese and
* Korean. For other languages terminating mid-word might
* not be the best, but terminating the whole string instead
* of continuing the word at the next line is worse. */
last_char = this->buffer;
} else {
/* A space is found; perfect place to terminate */
this->buffer = last_space + 1;
last_char = last_space;
}
break;
}
}
this->buffer++;
}
if (l->Length() == 0 || last_char - begin != 0) {
int w = l->GetWidth();
*l->Append() = new FallbackVisualRun(iter->second, begin, last_char - begin, w);
}
return l;
}
