bool Parser::hasOpt(string const l) { // An optional argument can occur in any of the following forms: // - \foo[bar] // - \foo [bar] // - \foo // [bar] // - \foo %comment // [bar] // remember current position unsigned int oldpos = pos_; // skip spaces and comments while (good()) { get_token(); if (isParagraph()) { putback(); break; } if (curr_token().cat() == catSpace || curr_token().cat() == catNewline || curr_token().cat() == catComment) continue; putback(); break; } bool const retval = (next_token().asInput() == l); pos_ = oldpos; return retval; }
bool Parser::skip_spaces(bool skip_comments) { // We just silently return if we have no more tokens. // skip_spaces() should be callable at any time, // the caller must check p::good() anyway. bool skipped = false; while (good()) { get_token(); if (isParagraph()) { putback(); break; } if (curr_token().cat() == catSpace || curr_token().cat() == catNewline) { skipped = true; continue; } if ((curr_token().cat() == catComment && curr_token().cs().empty())) continue; if (skip_comments && curr_token().cat() == catComment) { // If positions_ is not empty we are doing some kind // of look ahead if (!positions_.empty()) cerr << " Ignoring comment: " << curr_token().asInput(); } else { putback(); break; } } return skipped; }
S32 LLSDNotationParser::parseMap(std::istream& istr, LLSD& map) const { // map: { string:object, string:object } map = LLSD::emptyMap(); S32 parse_count = 0; char c = get(istr); if(c == '{') { // eat commas, white bool found_name = false; std::string name; c = get(istr); while(c != '}' && istr.good()) { if(!found_name) { if((c == '\"') || (c == '\'') || (c == 's')) { putback(istr, c); found_name = true; int count = deserialize_string(istr, name, mMaxBytesLeft); if(PARSE_FAILURE == count) return PARSE_FAILURE; account(count); } c = get(istr); } else { if(isspace(c) || (c == ':')) { c = get(istr); continue; } putback(istr, c); LLSD child; S32 count = doParse(istr, child); if(count > 0) { // There must be a value for every key, thus // child_count must be greater than 0. parse_count += count; map.insert(name, child); } else { return PARSE_FAILURE; } found_name = false; c = get(istr); } } if(c != '}') { map.clear(); return PARSE_FAILURE; } } return parse_count; }
bool Scanner::scanInt (char c, Parser& parser, bool& cont) { assert(c != '\0'); std::string value; value += c; bool error = false; while (get (c)) { if (std::isdigit (c)) { value += c; } else if (c!='-' && isStringCharacter (c)) { error = true; value += c; } else if (c=='.') { if (error) { putback (c); break; } return scanFloat (value, parser, cont); } else { putback (c); break; } } if (error) { /// workaround that allows names to begin with digits /// \todo disable TokenLoc loc (mLoc); mLoc.mLiteral.clear(); cont = parser.parseName (value, loc, *this); return true; // return false; } TokenLoc loc (mLoc); mLoc.mLiteral.clear(); std::istringstream stream (value); int intValue = 0; stream >> intValue; cont = parser.parseInt (intValue, loc, *this); return true; }
dofile() { char word[WORDSIZE], *wp; char cmd[WORDSIZE] , sub[WORDSIZE] ; int n ; log_lev++; /* one file deeper */ pen = G_MOVE ; /* pick up the pen for each new file. */ while ( (n = getword(word, W_MUST)) > 0) { *cmd = 0; *sub = 0; /* see if it is a system call out */ if (n == SYS_W || n == SIN_W) { /* so shell gets replaced */ char *ex = "exec " ; /* char *ex = "" ; */ strcpy(word, ex); wp = (word + strlen(ex)) ; (void)getword(wp, W_LITR) ; /*collect litteral string*/ docmd("sys" , (n==SIN_W ? "in" : "") , word); } /* see if it is a command */ else if (*word == cmd_char) { wp = word + 1; if (!mkcmd(wp, cmd, sub)) continue; /* get the (possible) argument */ (void)getword(word, W_ANY ) ; /* got a command, so no arg */ if (*word == cmd_char || *word == sys_char) { putback(word); *word = 0; /* null argument */ } /* interpret the command, if word not used, putback */ if (docmd(cmd, sub, word) == DOBACK) putback(word); } /* a number or some word that we don't care about */ else { docmd("" , "" , word); } }/* of loop through file */ log_lev--; /* back from a file depth */ }/* of dofile */
/* Interpret a single statement or block of code. When interp_block() returns from its initial call, the final brace (or a return) in main() has been encountered. */ void interp_block(void) { int value; char block = 0; do { token_type = get_token(); /* If interpreting single statement, return on first semicolon. */ /* see what kind of token is up */ if(token_type == IDENTIFIER) { /* Not a keyword, so process expression. */ putback(); /* restore token to input stream for further processing by eval_exp() */ eval_exp(&value); /* process the expression */ if(*token!=';') sntx_err(SEMI_EXPECTED); } else if(token_type==BLOCK) { /* if block delimiter */ if(*token == '{') /* is a block */ block = 1; /* interpreting block, not statement */ else return; /* is a }, so return */ } else /* is keyword */ switch(tok) { case CHAR: case INT: /* declare local variables */ putback(); decl_local(); break; case RETURN: /* return from function call */ func_ret(); return; case IF: /* process an if statement */ exec_if(); break; case ELSE: /* process an else statement */ find_eob(); /* find end of else block and continue execution */ break; case WHILE: /* process a while loop */ exec_while(); break; case DO: /* process a do-while loop */ exec_do(); break; case FOR: /* process a for loop */ exec_for(); break; case END: exit(0); } } while (tok != FINISHED && block); }
bool Scanner::scanName (char c, std::string& name) { bool first = false; bool error = false; name.clear(); putback (c); while (get (c)) { if (!name.empty() && name[0]=='"') { if (c=='"') { name += c; break; } // ignoring escape sequences for now, because they are messing up stupid Windows path names. // else if (c=='\\') // { // if (!get (c)) // { // mErrorHandler.error ("incomplete escape sequence", mLoc); // break; // } // } else if (c=='\n') { mErrorHandler.error ("incomplete string or name", mLoc); break; } } else if (!(c=='"' && name.empty())) { if (!(std::isalpha (c) || std::isdigit (c) || c=='_' || c=='`' || /// \todo add an option to disable the following hack. Also, find out who is /// responsible for allowing it in the first place and meet up with that person in /// a dark alley. (c=='-' && !name.empty() && std::isalpha (mStream.peek())))) { putback (c); break; } if (first && std::isdigit (c)) error = true; } name += c; first = false; } return !error; }
TokenTypes Scanner::handleSlash( Token * tok ) { TokenTypes result; int current = nextch(); if( current == S_ENDC ) { HCWarning( RTF_BADEOF, _source->name() ); result = TOK_END; } else if( current == '*' ) { // Certain RTF commands begin with "\*\", not "\". current = nextch(); if( current != '\\' ) { HCWarning( RTF_BADCOMMAND, _lineNum, _source->name() ); if( current != S_ENDC ) { putback( current ); } result = TOK_NONE; } else { result = handleSlash( tok ); } } else if( current == '\n' ) { // A "\" just before a new-line is the same as "\par". memcpy( tok->_text, "par", 4 ); result = TOK_COMMAND; ++_lineNum; } else if( isSpecial( current ) ) { // Some characters are escaped, like "\{". result = TOK_SPEC_CHAR; tok->_value = current; } else if( current == '\'' ) { // "\'nnn" signifies the byte with value nnn. result = TOK_SPEC_CHAR; pullHex( tok ); } else if( islower( current ) ) { // All RTF commands are in lower case. putback( current ); result = TOK_COMMAND; pullCommand( tok ); } else { HCWarning( RTF_BADCOMMAND, _lineNum, _source->name() ); result = TOK_NONE; } return( result ); }
bool Scanner::scanName (char c, std::string& name) { bool first = false; bool error = false; name.clear(); putback (c); while (get (c)) { if (!name.empty() && name[0]=='"') { if (c=='"') { name += c; break; } // ignoring escape sequences for now, because they are messing up stupid Windows path names. // else if (c=='\\') // { // if (!get (c)) // { // mErrorHandler.error ("incomplete escape sequence", mLoc); // break; // } // } else if (c=='\n') { mErrorHandler.error ("incomplete string or name", mLoc); break; } } else if (!(c=='"' && name.empty())) { if (!(std::isalpha (c) || std::isdigit (c) || c=='_')) { putback (c); break; } if (first && std::isdigit (c)) error = true; } name += c; first = false; } return !error; }
/* * pbnum - convert number to string, push back on input. */ void pbnum(int n) { int num; num = (n < 0) ? -n : n; do { putback(num % 10 + '0'); } while ((num /= 10) > 0); if (n < 0) putback('-'); }
void Scanner::getToken( Token * tok ) { int current; while( (current = nextch()) == '\n' ) { ++_lineNum; } tok->_lineNum = _lineNum; switch( current ) { case S_ENDC: tok->_type = TOK_END; break; case '{': tok->_type = TOK_PUSH_STATE; break; case '}': tok->_type = TOK_POP_STATE; break; case '\\': current = nextch(); if( current == '~' ) { tok->_type = TOK_TEXT; putback( HARD_SPACE ); pullText( tok ); } else { putback( current ); tok->_type = handleSlash( tok ); } break; case '\t': tok->_type = TOK_COMMAND; memcpy( tok->_text, "tab", 4 ); break; default: if( isFootnoteChar( current ) ) { tok->_type = TOK_SPEC_CHAR; tok->_value = current; } else { tok->_type = TOK_TEXT; putback( current ); pullText( tok ); } } }
/* Обработка выражения в присваивании */ void eval_exp0(int *value) { char temp[ID_LEN]; /* содержит имя переменной, которой присваивается значение */ register int temp_tok; if(token_type == IDENTIFIER) { if(is_var(token)) { /* если эта переменная, посмотреть, присваивается ли ей значение */ if(is_arr(token)) { sntx_err(SYNTAX); // todo: make message: redefinition of array is illegal } strcpy(temp, token); temp_tok = token_type; get_token(); if(*token == '=') { /* это присваивание */ get_token(); eval_exp0(value); /* вычислить присваемое значение */ assign_var(temp, *value); /* присвоить значение */ return; } else { /* не присваивание */ putback(); /* востановление лексемы */ strcpy(token, temp); token_type = temp_tok; } } else if(is_arr(token)) { strcpy(temp, token); temp_tok = token_type; get_token(); if(*token == '[') { get_token(); eval_exp0(value); // вычисление выражения в [] скобках int arr_index = *value; get_token(); // '=' or ';' if(*token == '=') { /* это присваивание */ get_token(); eval_exp0(value); /* вычислить присваемое значение */ assign_arr_element(temp, arr_index, *value); /* присвоить значение */ return; } else { /* не присваивание */ putback(); /* востановление лексемы */ strcpy(token, temp); token_type = temp_tok; } } else { sntx_err(ASSIGN_ARRAY_ILLEGAL); } } } eval_exp1(value); }
void Scanner::pullCommand( Token * tok ) { int current = 0; char num_string[7]; size_t i = 0; tok->_text[i] = static_cast<char>(nextch()); for( i = 1; i < BUF_SIZE - 1; i++ ) { current = nextch(); if( !islower( current ) ) break; tok->_text[i] = static_cast<char>(current); } tok->_text[i] = '\0'; if( current == S_ENDC || (!isdigit(current) && current != '-') ) { tok->_hasValue = false; } else { tok->_hasValue = true; for( i=0; i<6; i++ ) { num_string[i] = static_cast<char>(current); current = nextch(); if( !isdigit(current) ) { break; } } num_string[i + 1] = '\0'; tok->_value = atoi( num_string ); } if( current != S_ENDC && current != ' ' ) { putback( current ); } }
/* Process an assignment expression */ void eval_exp0(int *value) { char temp[ID_LEN]; /* holds name of var receiving the assignment */ register int temp_tok; if(token_type==IDENTIFIER) { if(is_var(token)) { /* if a var, see if assignment */ strcpy(temp, token); temp_tok = token_type; get_token(); if(*token=='=') { /* is an assignment */ get_token(); eval_exp0(value); /* get value to assign */ assign_var(temp, *value); /* assign the value */ return; } else { /* not an assignment */ putback(); /* restore original token */ strcpy(token, temp); token_type = temp_tok; } } } eval_exp1(value); }
void eval_exp0(int &value) { char temp[MAX_ID_LEN + 1]; tok_types temp_tok; if (token_type == IDENTIFIER) { if (is_var(token)) { strcpy(temp, token); temp_tok = token_type; get_token(); if (*token == '=') { get_token(); eval_exp0(value); assign_var(temp, value); return; } else { putback(); strcpy(token, temp); token_type = temp_tok; } } } eval_exp1(value); }
/* Processa uma atribuicao */ void Parser::eval_exp1(double *answer) { int slot; char ttok_type; char temp_token[80]; if (tok_type == VARIAVEL) { /* salva token antigo */ strcpy(temp_token, token); ttok_type = tok_type; /* calcula o indice da variavel */ slot = toupper(*token)-'A'; get_token(); if (*token != '=') { putback(); /* devolve token atual */ /* restaura token antigo - nenhuma atribuicao */ strcpy(token, temp_token); tok_type = ttok_type; } else { get_token(); /* pega a proxima parte da expressao */ eval_exp2(answer); vars[slot] = *answer; return; } } eval_exp2(answer); }
TObject* Scanner::extractComparator(char c){ char n = getChar(); TOKEN_TYPE type; string* str = new string(1, c); if(n=='='){ str->append(1, n); switch(c){ case '<':type = LESS_THAN_OR_EQUALS; break; case '>':type = GREATER_THAN_OR_EQUALS; break; case '=':type = EQUALITY; break; } }else{ //put it back otherwise putback(); switch(c){ case '<':type = LESS_THAN; break; case '>':type = GREATER_THAN; break; case '=':type = EQUAL_SIGN; break; } } return new TObject(str, type, line, char_line); }
static int scanch(void) { int c, c2; c = next(); if ('\\' == c) { switch (c = next()) { case 'a': return '\a'; case 'b': return '\b'; case 'f': return '\f'; case 'n': return '\n'; case 'r': return '\r'; case 't': return '\t'; case 'v': return '\v'; case '\\': return '\\'; case '"': return '"' | 256; case '\'': return '\''; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': for (c2 = 0; isdigit(c) && c < '8'; c = next()) c2 = c2 * 8 + (c - '0'); putback(c); return c2; case 'x': return hexchar(); default: cerror("unknown escape sequence: %s", c); return ' '; } } else { return c; } }
/************************************************ * this must not be used inside the basic - this is * only interpolation of the variable * OR * solving a formula * * Look up a a bas_token's internal representation in the bas_token table. * * HO HO HO. What if s is formula? with *+-/ * * */ double lookup_table2(char* s){ // register int i,j; int i; /* * solve a formula or just call lookup_table( s ) */ // printf("...........%s..............\n", s ); if ( (strpbrk( s , "+" )!=NULL)|| (strpbrk( s , "-" )!=NULL)|| (strpbrk( s , "*" )!=NULL)|| (strpbrk( s , "/" )!=NULL) ){ // printf("............. found a formula\n%s",""); double value; prog=s; bas_token_type = get_token(); putback(); get_exp(&value); // printf("toktyp=%d, value = %lf\n",bas_token_type, value ); return value; }// expr else{ // standard lookup_table( s ) int io=lookup_table_tk(s); // printf("............. NOT a formula, the var sits at pos %d\n", io ); return variables[io]; } return 0.0; /* unknown command */ }//------------------------lookup_table 2
string const Parser::ertEnvironment(string const & name) { if (!good()) return string(); ostringstream os; for (Token t = get_token(); good(); t = get_token()) { if (t.cat() == catBegin) { putback(); os << '{' << verbatim_item() << '}'; } else if (t.asInput() == "\\begin") { string const env = getArg('{', '}'); os << "\\begin{" << env << '}' << ertEnvironment(env) << "\\end{" << env << '}'; } else if (t.asInput() == "\\end") { string const end = getArg('{', '}'); if (end != name) cerr << "\\end{" << end << "} does not match \\begin{" << name << "}." << endl; return os.str(); } else os << t.asInput(); } cerr << "unexpected end of input" << endl; return os.str(); }
const char* AsxTokenizer::read_data (char start, bool &entirely_whitespace) { clear_buffer_str (); entirely_whitespace = true; char c = start; while (c != -1) { if (c == OP_OPEN_ELEMENT) { in_data = false; putback (c); break; } if (!g_ascii_isspace (c)) entirely_whitespace = false; g_string_append_c (buffer, c); c = read_char (); } return buffer->str; }
bool expression_parser::is_keyword(name_t keyword_name) { const stream_lex_token_t& result(get_token()); if (result.first == keyword_k && result.second.cast<name_t>() == keyword_name) return true; putback(); return false; }
// sheet . bool adam_test_parser::is_sheet() { if (!is_keyword(name_t("sheet"))) return false; putback(); sheets_m.push_back(new queryable_sheet_t(*this)); return true; }
static int scanint(int c) { int val, radix, k, i = 0; val = 0; radix = 10; if ('0' == c) { Text[i++] = '0'; if ((c = next()) == 'x') { radix = 16; Text[i++] = c; c = next(); } else { radix = 8; } } while ((k = chrpos("0123456789abcdef", tolower(c))) >= 0) { Text[i++] = c; if (k >= radix) cerror("invalid digit in integer literal: %s", c); val = val * radix + k; c = next(); } putback(c); Text[i] = 0; return val; }
bool expression_parser::is_token(name_t tokenName) { const stream_lex_token_t& result(get_token()); if (result.first == tokenName) { return true; } putback(); return false; }
/* * pbunsigned - convert unsigned long to string, push back on input. */ void pbunsigned(unsigned long n) { do { putback(n % 10 + '0'); } while ((n /= 10) > 0); }
void interp_block() { int value; char block = 0; do { token_type = get_token(); if(token_type==IDENTIFIER) { putback(); eval_exp(&value); if(*token!=';') sntx_err(SEMI_EXPECTED); } else if(token_type==BLOCK) { if(*token=='{') block = 1; else return; } else switch(tok) { case CHAR: case INT: putback(); decl_local(); break; case RETURN: func_ret(); return; case IF: exec_if(); break; case ELSE: find_eob(); break; case WHILE: exec_while(); break; case DO: exec_do(); break; case FOR: exec_for(); break; case END: return; } } while (tok != FINISHED && block); }
void atom(int &value) { int i; char temp[MAX_ID_LEN + 1]; switch (token_type) { case IDENTIFIER: i = internal_func(token); if (i != -1) { // Call "standard library" function. value = (*intern_func[i].p)(); } else if (find_func(token)) { // Call programmer-created function. call(); value = ret_value; } else { value = find_var(token); // get var's value strcpy(temp, token); // save variable name // Check for ++ or --. get_token(); if (*token == INC || *token == DEC) { if (*token == INC) assign_var(temp, find_var(temp) + 1); else assign_var(temp, find_var(temp) - 1); } else putback(); } get_token(); return; case NUMBER: // is numeric constant value = atoi(token); get_token(); return; case DELIMITER: // see if character constant if (*token == '\'') { value = *prog; prog++; if (*prog != '\'') throw InterpExc(QUOTE_EXPECTED); prog++; get_token(); return; } if (*token == ')') return; // process empty expression else throw InterpExc(SYNTAX); // otherwise, syntax error default: throw InterpExc(SYNTAX); // syntax error } }
bool expression_parser::is_token(name_t tokenName, any_regular_t& tokenValue) { const stream_lex_token_t& result(get_token()); if (result.first == tokenName) { tokenValue = std::move(result.second); return true; } putback(); return false; }
/* * Look ahead for `token'. * (on input `t == token[0]') * Used for comment and quoting delimiters. * Returns 1 if `token' present; copied to output. * 0 if `token' not found; all characters pushed back */ static int do_look_ahead(int t, const char *token) { int i; assert((unsigned char)t == (unsigned char)token[0]); for (i = 1; *++token; i++) { t = gpbc(); if (t == EOF || (unsigned char)t != (unsigned char)*token) { putback(t); while (--i) putback(*--token); return 0; } } return 1; }