static sl_node_base_t* statement(sl_parse_state_t* ps) { sl_node_base_t* node; switch(peek_token(ps)->type) { case SL_TOK_CLOSE_TAG: next_token(ps); node = inline_raw(ps); if(peek_token(ps)->type != SL_TOK_END) { expect_token(ps, SL_TOK_OPEN_TAG); } return node; case SL_TOK_SEMICOLON: next_token(ps); return NULL; case SL_TOK_IF: case SL_TOK_UNLESS: return if_expression(ps); case SL_TOK_FOR: return for_expression(ps); case SL_TOK_WHILE: case SL_TOK_UNTIL: return while_expression(ps); default: node = expression(ps); if(peek_token(ps)->type != SL_TOK_CLOSE_TAG && peek_token(ps)->type != SL_TOK_CLOSE_BRACE && token(ps)->type != SL_TOK_CLOSE_BRACE && peek_token(ps)->type != SL_TOK_END) { expect_token(ps, SL_TOK_SEMICOLON); } return node; } }
static sl_node_base_t* send_with_args_expression(sl_parse_state_t* ps, sl_node_base_t* recv, SLID id) { size_t argc = 0, cap = 2; sl_node_base_t** argv = sl_alloc(ps->vm->arena, sizeof(sl_node_base_t*) * cap); bool splat_last = false; expect_token(ps, SL_TOK_OPEN_PAREN); while(peek_token(ps)->type != SL_TOK_CLOSE_PAREN) { if(argc >= cap) { cap *= 2; argv = sl_realloc(ps->vm->arena, argv, sizeof(sl_node_base_t*) * cap); } if(peek_token(ps)->type == SL_TOK_TIMES) { next_token(ps); splat_last = true; argv[argc++] = expression(ps); break; } argv[argc++] = expression(ps); if(peek_token(ps)->type != SL_TOK_CLOSE_PAREN) { expect_token(ps, SL_TOK_COMMA); } } expect_token(ps, SL_TOK_CLOSE_PAREN); return sl_make_send_node(ps, recv, id, argc, argv, splat_last); }
static sl_node_base_t* if_expression(sl_parse_state_t* ps) { sl_node_base_t *condition, *if_true, *if_false = NULL; int negate_condition = 0; if(peek_token(ps)->type == SL_TOK_ELSIF) { expect_token(ps, SL_TOK_ELSIF); } else if(peek_token(ps)->type == SL_TOK_UNLESS) { expect_token(ps, SL_TOK_UNLESS); negate_condition = 1; } else { expect_token(ps, SL_TOK_IF); } condition = expression(ps); if(negate_condition) { condition = sl_make_unary_node(ps, condition, SL_NODE_NOT); } if_true = body_expression(ps); if(peek_token(ps)->type == SL_TOK_ELSIF) { if_false = if_expression(ps); } if(peek_token(ps)->type == SL_TOK_ELSE) { next_token(ps); if_false = body_expression(ps); } return sl_make_if_node(ps, condition, if_true, if_false); }
static sl_node_base_t* switch_expression(sl_parse_state_t* ps) { expect_token(ps, SL_TOK_SWITCH); sl_node_base_t* value = expression(ps); expect_token(ps, SL_TOK_OPEN_BRACE); size_t case_count = 0, case_cap = 2; sl_node_switch_case_t* cases = sl_alloc(ps->vm->arena, sizeof(sl_node_switch_case_t) * case_cap); sl_node_base_t* else_body = NULL; while(peek_token(ps)->type != SL_TOK_CLOSE_BRACE) { if(peek_token(ps)->type == SL_TOK_ELSE) { next_token(ps); else_body = body_expression(ps); break; } if(case_count + 1 >= case_cap) { case_cap *= 2; cases = sl_realloc(ps->vm->arena, cases, sizeof(sl_node_switch_case_t) * case_cap); } cases[case_count].value = expression(ps); cases[case_count].body = body_expression(ps); case_count++; } expect_token(ps, SL_TOK_CLOSE_BRACE); return sl_make_switch_node(ps, value, case_count, cases, else_body); }
/*----------------------------------------------------------------------*/ static ast_expression_t *parse_function_call(parse_state_t *p) { token_t *token; ast_expression_t *expression; ast_expression_list_t *param, *new_param; expression = ALLOC(sizeof(ast_expression_t)); BZERO(expression); expression->tag = AST_EXPRESSION_FUNCTION_CALL; /* function identifier */ token = next_token(p); EXPECT(token, TK_IDENTIFIER, "function name expected"); expression->u.function_call.identifier = token; /* opening bracket */ token = next_token(p); EXPECT(token, TK_LBRACKET, "'(' expected"); /* parameter list */ param = NULL; expression->u.function_call.parameter_count = 0; for (;;) { token = peek_token(p); if (test_token(token, TK_RBRACKET)) break; /* parameter */ new_param = ALLOC(sizeof(ast_expression_list_t)); new_param->next = NULL; new_param->expr = parse_logical_expression(p); if (new_param->expr == NULL) error(p, "expression expected"); if (param == NULL) { expression->u.function_call.parameter_expr_list = new_param; } else { param->next = new_param; } param = new_param; expression->u.function_call.parameter_count += 1; /* comma? */ token = peek_token(p); if (test_token(token, TK_RBRACKET)) break; if (test_token(token, TK_COMMA)) { next_token(p); continue; } error(p, "',' or ')' expected"); } /* closing bracket */ token = next_token(p); EXPECT(token, TK_RBRACKET, "')' expected"); return expression; }
/* Consumes tokens from source until the next character is neither incrementer nor decrementer. Returns the number of incrementer occurrences minus the number of decrementer occurrences. */ static int collapse(FILE* source, char decrementer, char incrementer) { int net_change = 0; while (peek_token(source) == decrementer || peek_token(source) == incrementer) { if (get_token(source) == decrementer) net_change--; else net_change++; } return net_change; }
Token Lexer::get_expected_token(TokenType tt, const string& raw) { TokenType p = peek_token().type; string s = peek_token().as_string(); if (p != tt && s != raw) { string msg = S("expected ") + tokentype2str(tt) + " or `" + raw + "'"; throw_syntax_error(p == kTokenNop ? msg : msg + " instead of `" + s + "'"); } return get_token(); }
Token Lexer::get_expected_token(const string& raw1, const string& raw2) { TokenType p = peek_token().type; string s = peek_token().as_string(); if (s != raw1 && s != raw2) { string msg = "expected `" + raw1 + "' or `" + raw2 + "'"; throw_syntax_error(p == kTokenNop ? msg : msg + " instead of `" + s + "'"); } return get_token(); }
static sl_node_base_t* call_expression(sl_parse_state_t* ps) { sl_node_base_t* left = primary_expression(ps); sl_node_base_t** nodes; size_t node_len; size_t node_cap; sl_token_t* tok; if(left->type == SL_NODE_VAR && peek_token(ps)->type == SL_TOK_OPEN_PAREN) { left = send_with_args_expression(ps, sl_make_self_node(ps), sl_intern2(ps->vm, sl_make_ptr((sl_object_t*)((sl_node_var_t*)left)->name))); } while(1) { tok = peek_token(ps); switch(tok->type) { case SL_TOK_DOT: next_token(ps); left = send_expression(ps, left); break; case SL_TOK_COLON: next_token(ps); left = sl_make_bind_method_node(ps, left, def_expression_method_name(ps)); break; case SL_TOK_PAAMAYIM_NEKUDOTAYIM: next_token(ps); tok = expect_token(ps, SL_TOK_CONSTANT); left = sl_make_const_node(ps, left, sl_intern2(ps->vm, sl_make_string(ps->vm, tok->as.str.buff, tok->as.str.len))); break; case SL_TOK_OPEN_BRACKET: next_token(ps); node_cap = 1; node_len = 0; nodes = sl_alloc(ps->vm->arena, sizeof(SLVAL) * node_cap); while(peek_token(ps)->type != SL_TOK_CLOSE_BRACKET) { if(node_len >= node_cap) { node_cap *= 2; nodes = sl_realloc(ps->vm->arena, nodes, sizeof(SLVAL) * node_cap); } nodes[node_len++] = expression(ps); if(peek_token(ps)->type != SL_TOK_CLOSE_BRACKET) { expect_token(ps, SL_TOK_COMMA); } } expect_token(ps, SL_TOK_CLOSE_BRACKET); left = sl_make_send_node(ps, left, sl_intern(ps->vm, "[]"), node_len, nodes); break; default: return left; } } }
static sl_node_base_t* add_expression(sl_parse_state_t* ps) { sl_node_base_t* left = mul_expression(ps); sl_node_base_t* right; sl_token_t* tok; while(peek_token(ps)->type == SL_TOK_PLUS || peek_token(ps)->type == SL_TOK_MINUS) { tok = next_token(ps); right = mul_expression(ps); left = sl_make_send_node(ps, left, sl_intern2(ps->vm, tok->str), 1, &right); } return left; }
static sl_node_base_t* shift_expression(sl_parse_state_t* ps) { sl_node_base_t* left = add_expression(ps); sl_node_base_t* right; sl_token_t* tok; while(peek_token(ps)->type == SL_TOK_SHIFT_LEFT || peek_token(ps)->type == SL_TOK_SHIFT_RIGHT) { tok = next_token(ps); right = add_expression(ps); left = sl_make_send_node(ps, left, sl_intern2(ps->vm, tok->str), 1, &right); } return left; }
static SLID def_expression_method_name(sl_parse_state_t* ps) { switch(peek_token(ps)->type) { case SL_TOK_IDENTIFIER: return sl_intern2(ps->vm, next_token(ps)->str); /* operators: */ case SL_TOK_SHIFT_LEFT: case SL_TOK_SHIFT_RIGHT: case SL_TOK_DBL_EQUALS: case SL_TOK_NOT_EQUALS: case SL_TOK_SPACESHIP: case SL_TOK_LTE: case SL_TOK_LT: case SL_TOK_GTE: case SL_TOK_GT: case SL_TOK_PLUS: case SL_TOK_POW: case SL_TOK_TIMES: case SL_TOK_DIVIDE: case SL_TOK_MOD: case SL_TOK_CARET: case SL_TOK_AMP: case SL_TOK_PIPE: return sl_intern2(ps->vm, next_token(ps)->str); /* operators that can also be unary: */ case SL_TOK_MINUS: case SL_TOK_TILDE: { sl_token_t* tok = next_token(ps); if(peek_token(ps)->type == SL_TOK_SELF) { return sl_intern2(ps->vm, sl_string_concat(ps->vm, tok->str, next_token(ps)->str)); } else { return sl_intern2(ps->vm, tok->str); } break; } /* keywords: */ case SL_TOK_LAST: case SL_TOK_NEXT: return sl_intern2(ps->vm, next_token(ps)->str); case SL_TOK_OPEN_BRACKET: next_token(ps); expect_token(ps, SL_TOK_CLOSE_BRACKET); return sl_intern(ps->vm, "[]"); default: unexpected(ps, next_token(ps)); SLID dummy; return dummy; /* never reached */ } }
static sl_node_base_t* low_precedence_logical_expression(sl_parse_state_t* ps) { sl_node_base_t* left = low_precedence_not_expression(ps); if(peek_token(ps)->type == SL_TOK_LP_AND) { next_token(ps); left = sl_make_binary_node(ps, left, low_precedence_logical_expression(ps), SL_NODE_AND); } if(peek_token(ps)->type == SL_TOK_LP_OR) { next_token(ps); left = sl_make_binary_node(ps, left, low_precedence_logical_expression(ps), SL_NODE_OR); } return left; }
static sl_node_base_t* mul_expression(sl_parse_state_t* ps) { sl_node_base_t* left = unary_expression(ps); sl_node_base_t* right; sl_token_t* tok; while(peek_token(ps)->type == SL_TOK_TIMES || peek_token(ps)->type == SL_TOK_DIVIDE || peek_token(ps)->type == SL_TOK_MOD) { tok = next_token(ps); right = unary_expression(ps); left = sl_make_send_node(ps, left, sl_intern2(ps->vm, tok->str), 1, &right, false); } return left; }
static sl_node_base_t* while_expression(sl_parse_state_t* ps) { sl_node_base_t *condition, *body; sl_parse_scope_t scope; int until = 0; if(peek_token(ps)->type == SL_TOK_UNTIL) { next_token(ps); until = 1; } else { expect_token(ps, SL_TOK_WHILE); } condition = expression(ps); if(until) { condition = sl_make_unary_node(ps, condition, SL_NODE_NOT); } scope.prev = ps->scope; scope.flags = scope.prev->flags | SL_PF_CAN_NEXT_LAST; ps->scope = &scope; body = body_expression(ps); ps->scope = scope.prev; if(scope.flags & SL_PF_SCOPE_CLOSURE) { ps->scope->flags |= SL_PF_SCOPE_CLOSURE; } return sl_make_while_node(ps, condition, body); }
int main (int argc, char **argv) { program_name = basename (argv[0]); scan_options (argc, argv); stack *stack = new_stack (); token *scanner = new_token (stdin); for (;;) { int token = scan_token (scanner); if (token == EOF) break; switch (token) { case NUMBER: do_push (stack, peek_token (scanner)); break; case '+': do_binop (stack, add_bigint); break; case '-': do_binop (stack, sub_bigint); break; case '*': do_binop (stack, mul_bigint); break; case 'c': do_clear (stack); break; case 'f': do_print_all (stack); break; case 'p': do_print (stack); break; default: unimplemented (token); break; } } do_clear(stack); free_stack(stack); free_token(scanner); DEBUGF ('m', "EXIT %d\n", exit_status); return EXIT_SUCCESS; }
/* * Parse a sequence of numbers separated by the token specified in separator. * Exactly max numbers are expected. */ int parse_numeric_aggregate(FILE *cfile, unsigned char *buf, int max, int separator, int base) { char *val; int token, count; if (buf == NULL || max == 0) error("no space for numeric aggregate"); for (count = 0; count < max; count++, buf++) { if (count && (peek_token(&val, cfile) == separator)) token = next_token(&val, cfile); token = next_token(&val, cfile); if (token == TOK_NUMBER || (base == 16 && token == TOK_NUMBER_OR_NAME)) /* XXX Need to check if conversion was successful. */ convert_num(buf, val, base, 8); else break; } if (count < max) { parse_warn("numeric aggregate too short."); return (0); } return (1); }
bool TokenStream::is_at_end_of_command() { Token t; if (peek_token(t)) { return (t.is_equal("\n") || t.is_equal(";")); } return true; }
TOKEN type_specifier(void) { TOKEN tok = peek_token(); if(tok == NULL) { fprintf(stderr, "no type specifier found...\n"); return NULL; } TOKEN type_spec; if( true == token_matches_keyword(tok, VOID) || true == token_matches_keyword(tok, CHAR) || true == token_matches_keyword(tok, SHORT) || true == token_matches_keyword(tok, INT) || true == token_matches_keyword(tok, LONG) || //true == token_matches_keyword(tok, FLOAT) || //true == token_matches_keyword(tok, DOUBLE) || true == token_matches_keyword(tok, SIGNED) || true == token_matches_keyword(tok, UNSIGNED) ) { type_spec = get_token(); } else { type_spec = NULL; } return type_spec; }
TOKEN declaration(SYMBOL s) { TOKEN t = NULL; TOKEN dec = NULL; s->kind = VARSYM; //if we have made it this far, i think it's safe to assume we are declaring a variable declaration_specifiers(s); dec = init_declarator_list(s); t = peek_token(); if(false == token_matches_delimiter(t, SEMICOLON)) { //it turns out that we haven't found a variable declaration, so exit //and allow the function definition recognizer take a stab at parsing dec = NULL; } else { t = get_token(); //consume the SEMICOLON SYMBOL entry = searchins(s->namestring); copy_symbol(s, entry); } return dec; }
/*----------------------------------------------------------------------*/ static ast_expression_t *parse_sum_op(parse_state_t *p, ast_expression_t *left) { token_t* token; token = peek_token(p); if (token == NULL) { return NULL; } else if (token->type == TK_PLUS || token->type == TK_MINUS) { ast_expression_t *right; ast_expression_t *bin_op; ast_expression_t *sum_op; next_token(p); right = parse_term(p); bin_op = ALLOC(sizeof(ast_expression_t)); BZERO(bin_op); bin_op->tag = AST_EXPRESSION_BIN_OP; bin_op->u.bin_op.left = left; bin_op->u.bin_op.right = right; bin_op->u.bin_op.operation = token->type; sum_op = parse_sum_op(p, bin_op); return sum_op != NULL ? sum_op : bin_op; } else { return NULL; } }
/* Skip to the semicolon ending the current statement. If we encounter * braces, the matching closing brace terminates the statement. If we * encounter a right brace but haven't encountered a left brace, return * leaving the brace in the token buffer for the caller. If we see a * semicolon and haven't seen a left brace, return. This lets us skip * over: * * statement; * statement foo bar { } * statement foo bar { statement { } } * statement} * * ...et cetera. */ void skip_to_semi(FILE *cfile) { int brace_count = 0, token; char *val; do { token = peek_token(&val, cfile); if (token == RBRACE) { if (brace_count) { token = next_token(&val, cfile); if (!--brace_count) return; } else return; } else if (token == LBRACE) { brace_count++; } else if (token == SEMI && !brace_count) { token = next_token(&val, cfile); return; } else if (token == '\n') { /* * EOL only happens when parsing * /etc/resolv.conf, and we treat it like a * semicolon because the resolv.conf file is * line-oriented. */ token = next_token(&val, cfile); return; } token = next_token(&val, cfile); } while (token != EOF); }
/*----------------------------------------------------------------------*/ static ast_statement_t *parse_statement(parse_state_t *p) { token_t *token; token = peek_token(p); if (token == NULL) { return NULL; } else if (test_token(token, TK_ELSE) || test_token(token, TK_END)) { return NULL; } else if (test_token(token, TK_VAR)) { return parse_declare_var(p); } else if (test_token(token, TK_FUNCTION)) { return parse_define_function(p); } else if (test_token(token, TK_RETURN)) { return parse_return(p); } else if (test_token(token, TK_IF)) { return parse_if(p); } else if (test_token(token, TK_FOR)) { return parse_for(p); } else if (test_token(token, TK_IDENTIFIER)) { return parse_assignment(p); } next_token(p); error(p, "statement expected"); return NULL; /* unreachable */ }
/*----------------------------------------------------------------------*/ static ast_statement_t *parse_if(parse_state_t *p) { token_t *token; ast_statement_t *statement; /* if */ token = next_token(p); EXPECT(token, TK_IF, "'if' expected"); statement = ALLOC(sizeof(ast_statement_t)); BZERO(statement); statement->tag = AST_STATEMENT_IF; /* if predicate / block */ statement->u.if_.condition = parse_logical_expression(p); statement->u.if_.if_block = parse_statement_list(p); /* else? */ token = peek_token(p); if (test_token(token, TK_ELSE)) { next_token(p); /* else block */ statement->u.if_.else_block = parse_statement_list(p); } else { statement->u.if_.else_block = NULL; } /* end */ token = next_token(p); EXPECT(token, TK_END, "'end' expected"); return statement; }
static void collect_arguments (symbol *sym, struct obstack *argptr, struct obstack *arguments) { token_data td; token_data *tdp; bool more_args; bool groks_macro_args = SYMBOL_MACRO_ARGS (sym); TOKEN_DATA_TYPE (&td) = TOKEN_TEXT; TOKEN_DATA_TEXT (&td) = SYMBOL_NAME (sym); tdp = (token_data *) obstack_copy (arguments, &td, sizeof td); obstack_ptr_grow (argptr, tdp); if (peek_token () == TOKEN_OPEN) { next_token (&td, NULL); /* gobble parenthesis */ do { more_args = expand_argument (arguments, &td); if (!groks_macro_args && TOKEN_DATA_TYPE (&td) == TOKEN_FUNC) { TOKEN_DATA_TYPE (&td) = TOKEN_TEXT; TOKEN_DATA_TEXT (&td) = (char *) ""; } tdp = (token_data *) obstack_copy (arguments, &td, sizeof td); obstack_ptr_grow (argptr, tdp); } while (more_args); } }
int read_client_conf_file (const char *name, struct interface_info *ip, struct client_config *client) { int file; struct parse *cfile; const char *val; int token; isc_result_t status; if ((file = open (name, O_RDONLY)) < 0) return uerr2isc (errno); cfile = (struct parse *)0; new_parse (&cfile, file, (char *)0, 0, path_dhclient_conf, 0); do { token = peek_token (&val, (unsigned *)0, cfile); if (token == END_OF_FILE) break; parse_client_statement (cfile, ip, client); } while (1); token = next_token (&val, (unsigned *)0, cfile); status = (cfile -> warnings_occurred ? ISC_R_BADPARSE : ISC_R_SUCCESS); end_parse (&cfile); return status; }
/*----------------------------------------------------------------------*/ static ast_expression_t *parse_compare_op(parse_state_t *p, ast_expression_t *left) { token_t* token; token = peek_token(p); if (token == NULL) { return NULL; } else if (token->type == TK_LESS || token->type == TK_LESS_EQUAL || token->type == TK_EQUALS_EQUALS || token->type == TK_GREATER_EQUAL || token->type == TK_GREATER) { ast_expression_t *right; ast_expression_t *bin_op; ast_expression_t *compare_op; next_token(p); right = parse_expression(p); bin_op = ALLOC(sizeof(ast_expression_t)); BZERO(bin_op); bin_op->tag = AST_EXPRESSION_BIN_OP; bin_op->u.bin_op.left = left; bin_op->u.bin_op.right = right; bin_op->u.bin_op.operation = token->type; compare_op = parse_compare_op(p, bin_op); return compare_op != NULL ? compare_op : bin_op; } else { return NULL; } }
/*----------------------------------------------------------------------*/ static ast_expression_t *parse_term_op(parse_state_t *p, ast_expression_t *left) { token_t* token; token = peek_token(p); if (token == NULL) { return NULL; } else if (token->type == TK_STAR || token->type == TK_SLASH) { ast_expression_t *right; ast_expression_t *bin_op; ast_expression_t *term_op; next_token(p); right = parse_signed_factor(p); bin_op = ALLOC(sizeof(ast_expression_t)); bin_op->tag = AST_EXPRESSION_BIN_OP; bin_op->u.bin_op.left = left; bin_op->u.bin_op.right = right; bin_op->u.bin_op.operation = token->type; term_op = parse_term_op(p, bin_op); return term_op != NULL ? term_op : bin_op; } else { return NULL; } }
void generate_code(java_file file, size_t method_index, FILE* source) { struct java_method* method = java_get_class(file)->methods + method_index; buffer bytecode_buffer = buffer_create(1024); bool success; write_method_prologue(bytecode_buffer); write_section(bytecode_buffer, source, file); write_method_epilogue(bytecode_buffer); success = (peek_token(source) == EOF); /* Otherwise it's an unmatched ']'. */ if (success) { method->max_stack = 6; method->max_locals = 1; method->bytecode_length = buffer_length(bytecode_buffer); method->bytecode = buffer_publish(bytecode_buffer); } buffer_free(bytecode_buffer); if (!success) { printf("Error: Unmatched \"]\".\n"); abort(); } }
static sl_node_base_t* low_precedence_not_expression(sl_parse_state_t* ps) { if(peek_token(ps)->type == SL_TOK_LP_NOT) { return sl_make_unary_node(ps, assignment_expression(ps), SL_NODE_NOT); } else { return assignment_expression(ps); } }