void mp_parse_node_free(mp_parse_node_t pn) { if (MP_PARSE_NODE_IS_STRUCT(pn)) { mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn; mp_uint_t n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); mp_uint_t rule_id = MP_PARSE_NODE_STRUCT_KIND(pns); if (rule_id == RULE_string) { m_del(char, (char*)pns->nodes[0], (mp_uint_t)pns->nodes[1]); return; } bool adjust = ADD_BLANK_NODE(rule_id); if (adjust) { n--; } for (mp_uint_t i = 0; i < n; i++) { mp_parse_node_free(pns->nodes[i]); } if (adjust) { n++; } m_del_var(mp_parse_node_struct_t, mp_parse_node_t, n, pns); }
return (mp_parse_node_t)(kind | (arg << 1)); } return (mp_parse_node_t)(kind | (arg << 4)); } void mp_parse_node_free(mp_parse_node_t pn) { if (MP_PARSE_NODE_IS_STRUCT(pn)) { mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn; mp_uint_t n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); mp_uint_t rule_id = MP_PARSE_NODE_STRUCT_KIND(pns); if (rule_id == RULE_string || rule_id == RULE_bytes) { m_del(char, (char*)pns->nodes[0], (mp_uint_t)pns->nodes[1]); } else if (rule_id == RULE_const_object) { // don't free the const object since it's probably used by the compiled code } else { bool adjust = ADD_BLANK_NODE(rules[rule_id]); if (adjust) { n--; } for (mp_uint_t i = 0; i < n; i++) { mp_parse_node_free(pns->nodes[i]); } if (adjust) { n++; } } m_del_var(mp_parse_node_struct_t, mp_parse_node_t, n, pns); } } int mp_parse_node_extract_list(mp_parse_node_t *pn, mp_uint_t pn_kind, mp_parse_node_t **nodes) {
mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // initialise parser and allocate memory for its stacks parser_t parser; parser.parse_error = PARSE_ERROR_NONE; parser.rule_stack_alloc = MICROPY_ALLOC_PARSE_RULE_INIT; parser.rule_stack_top = 0; parser.rule_stack = m_new_maybe(rule_stack_t, parser.rule_stack_alloc); parser.result_stack_alloc = MICROPY_ALLOC_PARSE_RESULT_INIT; parser.result_stack_top = 0; parser.result_stack = m_new_maybe(mp_parse_node_t, parser.result_stack_alloc); parser.lexer = lex; parser.tree.chunk = NULL; parser.cur_chunk = NULL; #if MICROPY_COMP_CONST mp_map_init(&parser.consts, 0); #endif // check if we could allocate the stacks if (parser.rule_stack == NULL || parser.result_stack == NULL) { goto memory_error; } // work out the top-level rule to use, and push it on the stack size_t top_level_rule; switch (input_kind) { case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break; case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break; default: top_level_rule = RULE_file_input; } push_rule(&parser, lex->tok_line, rules[top_level_rule], 0); // parse! size_t n, i; // state for the current rule size_t rule_src_line; // source line for the first token matched by the current rule bool backtrack = false; const rule_t *rule = NULL; for (;;) { next_rule: if (parser.rule_stack_top == 0 || parser.parse_error) { break; } pop_rule(&parser, &rule, &i, &rule_src_line); n = rule->act & RULE_ACT_ARG_MASK; /* // debugging printf("depth=%d ", parser.rule_stack_top); for (int j = 0; j < parser.rule_stack_top; ++j) { printf(" "); } printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack); */ switch (rule->act & RULE_ACT_KIND_MASK) { case RULE_ACT_OR: if (i > 0 && !backtrack) { goto next_rule; } else { backtrack = false; } for (; i < n; ++i) { uint16_t kind = rule->arg[i] & RULE_ARG_KIND_MASK; if (kind == RULE_ARG_TOK) { if (lex->tok_kind == (rule->arg[i] & RULE_ARG_ARG_MASK)) { push_result_token(&parser); mp_lexer_to_next(lex); goto next_rule; } } else { assert(kind == RULE_ARG_RULE); if (i + 1 < n) { push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule } push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule goto next_rule; } } backtrack = true; break; case RULE_ACT_AND: { // failed, backtrack if we can, else syntax error if (backtrack) { assert(i > 0); if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) { // an optional rule that failed, so continue with next arg push_result_node(&parser, MP_PARSE_NODE_NULL); backtrack = false; } else { // a mandatory rule that failed, so propagate backtrack if (i > 1) { // already eaten tokens so can't backtrack goto syntax_error; } else { goto next_rule; } } } // progress through the rule for (; i < n; ++i) { switch (rule->arg[i] & RULE_ARG_KIND_MASK) { case RULE_ARG_TOK: { // need to match a token mp_token_kind_t tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK; if (lex->tok_kind == tok_kind) { // matched token if (tok_kind == MP_TOKEN_NAME) { push_result_token(&parser); } mp_lexer_to_next(lex); } else { // failed to match token if (i > 0) { // already eaten tokens so can't backtrack goto syntax_error; } else { // this rule failed, so backtrack backtrack = true; goto next_rule; } } break; } case RULE_ARG_RULE: case RULE_ARG_OPT_RULE: rule_and_no_other_choice: push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule goto next_rule; default: assert(0); goto rule_and_no_other_choice; // to help flow control analysis } } assert(i == n); // matched the rule, so now build the corresponding parse_node // count number of arguments for the parse_node i = 0; bool emit_rule = false; for (size_t x = 0; x < n; ++x) { if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK; if (tok_kind >= MP_TOKEN_NAME) { emit_rule = true; } if (tok_kind == MP_TOKEN_NAME) { // only tokens which were names are pushed to stack i += 1; } } else { // rules are always pushed i += 1; } } #if !MICROPY_ENABLE_DOC_STRING // this code discards lonely statements, such as doc strings if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) { mp_parse_node_t p = peek_result(&parser, 1); if ((MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p)) || MP_PARSE_NODE_IS_STRUCT_KIND(p, RULE_string)) { pop_result(&parser); // MP_PARSE_NODE_NULL mp_parse_node_t pn = pop_result(&parser); // possibly RULE_string if (MP_PARSE_NODE_IS_STRUCT(pn)) { mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn; if (MP_PARSE_NODE_STRUCT_KIND(pns) == RULE_string) { m_del(char, (char*)pns->nodes[0], (size_t)pns->nodes[1]); } } push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0); break; } } #endif // always emit these rules, even if they have only 1 argument if (rule->rule_id == RULE_expr_stmt || rule->rule_id == RULE_yield_stmt) { emit_rule = true; } // if a rule has the RULE_ACT_ALLOW_IDENT bit set then this // rule should not be emitted if it has only 1 argument if (rule->act & RULE_ACT_ALLOW_IDENT) { emit_rule = false; } // always emit these rules, and add an extra blank node at the end (to be used by the compiler to store data) if (ADD_BLANK_NODE(rule)) { emit_rule = true; push_result_node(&parser, MP_PARSE_NODE_NULL); i += 1; } size_t num_not_nil = 0; for (size_t x = 0; x < i; ++x) { if (peek_result(&parser, x) != MP_PARSE_NODE_NULL) { num_not_nil += 1; } } if (emit_rule || num_not_nil != 1) { // need to add rule when num_not_nil==0 for, eg, atom_paren, testlist_comp_3b push_result_rule(&parser, rule_src_line, rule, i); } else { // single result, leave it on stack mp_parse_node_t pn = MP_PARSE_NODE_NULL; for (size_t x = 0; x < i; ++x) { mp_parse_node_t pn2 = pop_result(&parser); if (pn2 != MP_PARSE_NODE_NULL) { pn = pn2; } } push_result_node(&parser, pn); } break; }
mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_parse_error_kind_t *parse_error_kind_out) { // initialise parser and allocate memory for its stacks parser_t parser; parser.had_memory_error = false; parser.rule_stack_alloc = MICROPY_ALLOC_PARSE_RULE_INIT; parser.rule_stack_top = 0; parser.rule_stack = m_new_maybe(rule_stack_t, parser.rule_stack_alloc); parser.result_stack_alloc = MICROPY_ALLOC_PARSE_RESULT_INIT; parser.result_stack_top = 0; parser.result_stack = m_new_maybe(mp_parse_node_t, parser.result_stack_alloc); parser.lexer = lex; // check if we could allocate the stacks if (parser.rule_stack == NULL || parser.result_stack == NULL) { goto memory_error; } // work out the top-level rule to use, and push it on the stack int top_level_rule; switch (input_kind) { case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break; case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break; default: top_level_rule = RULE_file_input; } push_rule(&parser, mp_lexer_cur(lex)->src_line, rules[top_level_rule], 0); // parse! uint n, i; // state for the current rule uint rule_src_line; // source line for the first token matched by the current rule bool backtrack = false; const rule_t *rule = NULL; mp_token_kind_t tok_kind; bool emit_rule; bool had_trailing_sep; for (;;) { next_rule: if (parser.rule_stack_top == 0 || parser.had_memory_error) { break; } pop_rule(&parser, &rule, &i, &rule_src_line); n = rule->act & RULE_ACT_ARG_MASK; /* // debugging printf("depth=%d ", parser.rule_stack_top); for (int j = 0; j < parser.rule_stack_top; ++j) { printf(" "); } printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack); */ switch (rule->act & RULE_ACT_KIND_MASK) { case RULE_ACT_OR: if (i > 0 && !backtrack) { goto next_rule; } else { backtrack = false; } for (; i < n - 1; ++i) { switch (rule->arg[i] & RULE_ARG_KIND_MASK) { case RULE_ARG_TOK: if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) { push_result_token(&parser, lex); mp_lexer_to_next(lex); goto next_rule; } break; case RULE_ARG_RULE: push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule goto next_rule; default: assert(0); } } if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) { push_result_token(&parser, lex); mp_lexer_to_next(lex); } else { backtrack = true; goto next_rule; } } else { push_rule_from_arg(&parser, rule->arg[i]); } break; case RULE_ACT_AND: // failed, backtrack if we can, else syntax error if (backtrack) { assert(i > 0); if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) { // an optional rule that failed, so continue with next arg push_result_node(&parser, MP_PARSE_NODE_NULL); backtrack = false; } else { // a mandatory rule that failed, so propagate backtrack if (i > 1) { // already eaten tokens so can't backtrack goto syntax_error; } else { goto next_rule; } } } // progress through the rule for (; i < n; ++i) { switch (rule->arg[i] & RULE_ARG_KIND_MASK) { case RULE_ARG_TOK: // need to match a token tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK; if (mp_lexer_is_kind(lex, tok_kind)) { // matched token if (tok_kind == MP_TOKEN_NAME) { push_result_token(&parser, lex); } mp_lexer_to_next(lex); } else { // failed to match token if (i > 0) { // already eaten tokens so can't backtrack goto syntax_error; } else { // this rule failed, so backtrack backtrack = true; goto next_rule; } } break; case RULE_ARG_RULE: case RULE_ARG_OPT_RULE: push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule goto next_rule; default: assert(0); } } assert(i == n); // matched the rule, so now build the corresponding parse_node // count number of arguments for the parse_node i = 0; emit_rule = false; for (int x = 0; x < n; ++x) { if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK; if (tok_kind >= MP_TOKEN_NAME) { emit_rule = true; } if (tok_kind == MP_TOKEN_NAME) { // only tokens which were names are pushed to stack i += 1; } } else { // rules are always pushed i += 1; } } #if !MICROPY_EMIT_CPYTHON && !MICROPY_ENABLE_DOC_STRING // this code discards lonely statements, such as doc strings if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) { mp_parse_node_t p = peek_result(&parser, 1); if ((MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p)) || MP_PARSE_NODE_IS_STRUCT_KIND(p, RULE_string)) { pop_result(&parser); pop_result(&parser); push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0); break; } } #endif // always emit these rules, even if they have only 1 argument if (rule->rule_id == RULE_expr_stmt || rule->rule_id == RULE_yield_stmt) { emit_rule = true; } // never emit these rules if they have only 1 argument // NOTE: can't put atom_paren here because we need it to distinguisg, for example, [a,b] from [(a,b)] // TODO possibly put varargslist_name, varargslist_equal here as well if (rule->rule_id == RULE_else_stmt || rule->rule_id == RULE_testlist_comp_3b || rule->rule_id == RULE_import_as_names_paren || rule->rule_id == RULE_typedargslist_name || rule->rule_id == RULE_typedargslist_colon || rule->rule_id == RULE_typedargslist_equal || rule->rule_id == RULE_dictorsetmaker_colon || rule->rule_id == RULE_classdef_2 || rule->rule_id == RULE_with_item_as || rule->rule_id == RULE_assert_stmt_extra || rule->rule_id == RULE_as_name || rule->rule_id == RULE_raise_stmt_from || rule->rule_id == RULE_vfpdef) { emit_rule = false; } // always emit these rules, and add an extra blank node at the end (to be used by the compiler to store data) if (ADD_BLANK_NODE(rule->rule_id)) { emit_rule = true; push_result_node(&parser, MP_PARSE_NODE_NULL); i += 1; } int num_not_nil = 0; for (int x = 0; x < i; ++x) { if (peek_result(&parser, x) != MP_PARSE_NODE_NULL) { num_not_nil += 1; } } //printf("done and %s n=%d i=%d notnil=%d\n", rule->rule_name, n, i, num_not_nil); if (emit_rule) { push_result_rule(&parser, rule_src_line, rule, i); } else if (num_not_nil == 0) { push_result_rule(&parser, rule_src_line, rule, i); // needed for, eg, atom_paren, testlist_comp_3b //result_stack_show(parser); //assert(0); } else if (num_not_nil == 1) { // single result, leave it on stack mp_parse_node_t pn = MP_PARSE_NODE_NULL; for (int x = 0; x < i; ++x) { mp_parse_node_t pn2 = pop_result(&parser); if (pn2 != MP_PARSE_NODE_NULL) { pn = pn2; } } push_result_node(&parser, pn); } else { push_result_rule(&parser, rule_src_line, rule, i); } break; case RULE_ACT_LIST: // n=2 is: item item* // n=1 is: item (sep item)* // n=3 is: item (sep item)* [sep] if (backtrack) { list_backtrack: had_trailing_sep = false; if (n == 2) { if (i == 1) { // fail on item, first time round; propagate backtrack goto next_rule; } else { // fail on item, in later rounds; finish with this rule backtrack = false; } } else { if (i == 1) { // fail on item, first time round; propagate backtrack goto next_rule; } else if ((i & 1) == 1) { // fail on item, in later rounds; have eaten tokens so can't backtrack if (n == 3) { // list allows trailing separator; finish parsing list had_trailing_sep = true; backtrack = false; } else { // list doesn't allowing trailing separator; fail goto syntax_error; } } else { // fail on separator; finish parsing list backtrack = false; } } } else { for (;;) { uint arg = rule->arg[i & 1 & n]; switch (arg & RULE_ARG_KIND_MASK) { case RULE_ARG_TOK: if (mp_lexer_is_kind(lex, arg & RULE_ARG_ARG_MASK)) { if (i & 1 & n) { // separators which are tokens are not pushed to result stack } else { push_result_token(&parser, lex); } mp_lexer_to_next(lex); // got element of list, so continue parsing list i += 1; } else { // couldn't get element of list i += 1; backtrack = true; goto list_backtrack; } break; case RULE_ARG_RULE: push_rule(&parser, rule_src_line, rule, i + 1); // save this list-rule push_rule_from_arg(&parser, arg); // push child of list-rule goto next_rule; default: assert(0); } } } assert(i >= 1); // compute number of elements in list, result in i i -= 1; if ((n & 1) && (rule->arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { // don't count separators when they are tokens i = (i + 1) / 2; } if (i == 1) { // list matched single item if (had_trailing_sep) { // if there was a trailing separator, make a list of a single item push_result_rule(&parser, rule_src_line, rule, i); } else { // just leave single item on stack (ie don't wrap in a list) } } else { //printf("done list %s %d %d\n", rule->rule_name, n, i); push_result_rule(&parser, rule_src_line, rule, i); } break; default: assert(0); } } mp_parse_node_t result; // check if we had a memory error if (parser.had_memory_error) { memory_error: *parse_error_kind_out = MP_PARSE_ERROR_MEMORY; result = MP_PARSE_NODE_NULL; goto finished; } // check we are at the end of the token stream if (!mp_lexer_is_kind(lex, MP_TOKEN_END)) { goto syntax_error; } //printf("--------------\n"); //result_stack_show(parser); //printf("rule stack alloc: %d\n", parser.rule_stack_alloc); //printf("result stack alloc: %d\n", parser.result_stack_alloc); //printf("number of parse nodes allocated: %d\n", num_parse_nodes_allocated); // get the root parse node that we created assert(parser.result_stack_top == 1); result = parser.result_stack[0]; finished: // free the memory that we don't need anymore m_del(rule_stack_t, parser.rule_stack, parser.rule_stack_alloc); m_del(mp_parse_node_t, parser.result_stack, parser.result_stack_alloc); // return the result return result; syntax_error: if (mp_lexer_is_kind(lex, MP_TOKEN_INDENT)) { *parse_error_kind_out = MP_PARSE_ERROR_UNEXPECTED_INDENT; } else if (mp_lexer_is_kind(lex, MP_TOKEN_DEDENT_MISMATCH)) { *parse_error_kind_out = MP_PARSE_ERROR_UNMATCHED_UNINDENT; } else { *parse_error_kind_out = MP_PARSE_ERROR_INVALID_SYNTAX; #ifdef USE_RULE_NAME // debugging: print the rule name that failed and the token printf("rule: %s\n", rule->rule_name); #if MICROPY_DEBUG_PRINTERS mp_token_show(mp_lexer_cur(lex)); #endif #endif } result = MP_PARSE_NODE_NULL; goto finished; }