// return empty string in case of error, so we can attempt to parse the string
// without a special check if it was in fact a string
STATIC const char *get_arg_str(mp_parse_node_t pn) {
if (MP_PARSE_NODE_IS_ID(pn)) {
qstr qst = MP_PARSE_NODE_LEAF_ARG(pn);
return qstr_str(qst);
} else {
return "";
}
}
STATIC uint get_arg_rlo(qstr op, mp_parse_node_t *pn_args, int wanted_arg_num) {
if (!MP_PARSE_NODE_IS_ID(pn_args[wanted_arg_num])) {
printf("SyntaxError: '%s' expects a register in position %d\n", qstr_str(op), wanted_arg_num);
return 0;
}
qstr reg_qstr = MP_PARSE_NODE_LEAF_ARG(pn_args[wanted_arg_num]);
const char *reg_str = qstr_str(reg_qstr);
if (!(strlen(reg_str) == 2 && reg_str[0] == 'r' && ('0' <= reg_str[1] && reg_str[1] <= '7'))) {
printf("SyntaxError: '%s' expects a register in position %d\n", qstr_str(op), wanted_arg_num);
return 0;
}
return reg_str[1] - '0';
}
STATIC mp_uint_t get_arg_reglist(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) {
// a register list looks like {r0, r1, r2} and is parsed as a Python set
if (!MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_atom_brace)) {
goto bad_arg;
}
mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn;
assert(MP_PARSE_NODE_STRUCT_NUM_NODES(pns) == 1); // should always be
pn = pns->nodes[0];
mp_uint_t reglist = 0;
if (MP_PARSE_NODE_IS_ID(pn)) {
// set with one element
reglist |= 1 << get_arg_reg(emit, op, pn, 15);
} else if (MP_PARSE_NODE_IS_STRUCT(pn)) {
pns = (mp_parse_node_struct_t*)pn;
if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_dictorsetmaker) {
assert(MP_PARSE_NODE_IS_STRUCT(pns->nodes[1])); // should succeed
mp_parse_node_struct_t *pns1 = (mp_parse_node_struct_t*)pns->nodes[1];
if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_dictorsetmaker_list) {
// set with multiple elements
// get first element of set (we rely on get_arg_reg to catch syntax errors)
reglist |= 1 << get_arg_reg(emit, op, pns->nodes[0], 15);
// get tail elements (2nd, 3rd, ...)
mp_parse_node_t *nodes;
int n = mp_parse_node_extract_list(&pns1->nodes[0], PN_dictorsetmaker_list2, &nodes);
// process rest of elements
for (int i = 0; i < n; i++) {
reglist |= 1 << get_arg_reg(emit, op, nodes[i], 15);
}
} else {
goto bad_arg;
}
} else {
goto bad_arg;
}
} else {
goto bad_arg;
}
return reglist;
bad_arg:
emit_inline_thumb_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, "'%s' expects {r0, r1, ...}", op));
return 0;
}
STATIC int get_arg_label(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) {
if (!MP_PARSE_NODE_IS_ID(pn)) {
emit_inline_thumb_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, "'%s' expects a label", op));
return 0;
}
qstr label_qstr = MP_PARSE_NODE_LEAF_ARG(pn);
for (uint i = 0; i < emit->max_num_labels; i++) {
if (emit->label_lookup[i] == label_qstr) {
return i;
}
}
// only need to have the labels on the last pass
if (emit->pass == MP_PASS_EMIT) {
emit_inline_thumb_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, "label '%q' not defined", label_qstr));
}
return 0;
}
STATIC int get_arg_label(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) {
if (!MP_PARSE_NODE_IS_ID(pn)) {
emit_inline_thumb_error(emit, "'%s' expects a label\n", op);
return 0;
}
qstr label_qstr = MP_PARSE_NODE_LEAF_ARG(pn);
for (int i = 0; i < emit->max_num_labels; i++) {
if (emit->label_lookup[i] == label_qstr) {
return i;
}
}
// only need to have the labels on the last pass
if (emit->pass == PASS_3) {
emit_inline_thumb_error(emit, "label '%s' not defined\n", qstr_str(label_qstr));
}
return 0;
}
STATIC int get_arg_label(emit_inline_asm_t *emit, qstr op, mp_parse_node_t *pn_args, int wanted_arg_num) {
if (!MP_PARSE_NODE_IS_ID(pn_args[wanted_arg_num])) {
printf("SyntaxError: '%s' expects a label in position %d\n", qstr_str(op), wanted_arg_num);
return 0;
}
qstr label_qstr = MP_PARSE_NODE_LEAF_ARG(pn_args[wanted_arg_num]);
for (int i = 0; i < emit->max_num_labels; i++) {
if (emit->label_lookup[i] == label_qstr) {
return i;
}
}
// only need to have the labels on the last pass
if (emit->pass == PASS_3) {
printf("SyntaxError: label '%s' not defined\n", qstr_str(label_qstr));
}
return 0;
}
STATIC mp_uint_t emit_inline_thumb_count_params(emit_inline_asm_t *emit, mp_uint_t n_params, mp_parse_node_t *pn_params) {
if (n_params > 4) {
emit_inline_thumb_error_msg(emit, "can only have up to 4 parameters to Thumb assembly");
return 0;
}
for (mp_uint_t i = 0; i < n_params; i++) {
if (!MP_PARSE_NODE_IS_ID(pn_params[i])) {
emit_inline_thumb_error_msg(emit, "parameters must be registers in sequence r0 to r3");
return 0;
}
const char *p = qstr_str(MP_PARSE_NODE_LEAF_ARG(pn_params[i]));
if (!(strlen(p) == 2 && p[0] == 'r' && p[1] == '0' + i)) {
emit_inline_thumb_error_msg(emit, "parameters must be registers in sequence r0 to r3");
return 0;
}
}
return n_params;
}
STATIC int emit_inline_thumb_count_params(emit_inline_asm_t *emit, int n_params, mp_parse_node_t *pn_params) {
if (n_params > 4) {
emit_inline_thumb_error(emit, "can only have up to 4 parameters to inline thumb assembly\n");
return 0;
}
for (int i = 0; i < n_params; i++) {
if (!MP_PARSE_NODE_IS_ID(pn_params[i])) {
emit_inline_thumb_error(emit, "parameter to inline assembler must be an identifier\n");
return 0;
}
const char *p = qstr_str(MP_PARSE_NODE_LEAF_ARG(pn_params[i]));
if (!(strlen(p) == 2 && p[0] == 'r' && p[1] == '0' + i)) {
emit_inline_thumb_error(emit, "parameter %d to inline assembler must be r%d\n", i + 1, i);
return 0;
}
}
return n_params;
}
STATIC uint get_arg_reg(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn, uint max_reg) {
if (MP_PARSE_NODE_IS_ID(pn)) {
qstr reg_qstr = MP_PARSE_NODE_LEAF_ARG(pn);
const char *reg_str = qstr_str(reg_qstr);
for (uint i = 0; i < sizeof(reg_name_table) / sizeof(reg_name_table[0]); i++) {
const reg_name_t *r = ®_name_table[i];
if (reg_str[0] == r->name[0] && reg_str[1] == r->name[1] && reg_str[2] == r->name[2] && (reg_str[2] == '\0' || reg_str[3] == '\0')) {
if (r->reg > max_reg) {
emit_inline_thumb_error(emit, "'%s' expects at most r%d\n", op, max_reg);
return 0;
} else {
return r->reg;
}
}
}
}
emit_inline_thumb_error(emit, "'%s' expects a register\n", op);
return 0;
}
mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
// initialise parser and allocate memory for its stacks
parser_t parser;
parser.rule_stack_alloc = MICROPY_ALLOC_PARSE_RULE_INIT;
parser.rule_stack_top = 0;
parser.rule_stack = m_new(rule_stack_t, parser.rule_stack_alloc);
parser.result_stack_alloc = MICROPY_ALLOC_PARSE_RESULT_INIT;
parser.result_stack_top = 0;
parser.result_stack = m_new(mp_parse_node_t, parser.result_stack_alloc);
parser.lexer = lex;
parser.tree.chunk = NULL;
parser.cur_chunk = NULL;
#if MICROPY_COMP_CONST
mp_map_init(&parser.consts, 0);
#endif
// work out the top-level rule to use, and push it on the stack
size_t top_level_rule;
switch (input_kind) {
case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break;
case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break;
default: top_level_rule = RULE_file_input;
}
push_rule(&parser, lex->tok_line, rules[top_level_rule], 0);
// parse!
size_t n, i; // state for the current rule
size_t rule_src_line; // source line for the first token matched by the current rule
bool backtrack = false;
const rule_t *rule = NULL;
for (;;) {
next_rule:
if (parser.rule_stack_top == 0) {
break;
}
pop_rule(&parser, &rule, &i, &rule_src_line);
n = rule->act & RULE_ACT_ARG_MASK;
/*
// debugging
printf("depth=%d ", parser.rule_stack_top);
for (int j = 0; j < parser.rule_stack_top; ++j) {
printf(" ");
}
printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack);
*/
switch (rule->act & RULE_ACT_KIND_MASK) {
case RULE_ACT_OR:
if (i > 0 && !backtrack) {
goto next_rule;
} else {
backtrack = false;
}
for (; i < n; ++i) {
uint16_t kind = rule->arg[i] & RULE_ARG_KIND_MASK;
if (kind == RULE_ARG_TOK) {
if (lex->tok_kind == (rule->arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(&parser, rule);
mp_lexer_to_next(lex);
goto next_rule;
}
} else {
assert(kind == RULE_ARG_RULE);
if (i + 1 < n) {
push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule
}
push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule
goto next_rule;
}
}
backtrack = true;
break;
case RULE_ACT_AND: {
// failed, backtrack if we can, else syntax error
if (backtrack) {
assert(i > 0);
if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) {
// an optional rule that failed, so continue with next arg
push_result_node(&parser, MP_PARSE_NODE_NULL);
backtrack = false;
} else {
// a mandatory rule that failed, so propagate backtrack
if (i > 1) {
// already eaten tokens so can't backtrack
goto syntax_error;
} else {
goto next_rule;
}
}
}
// progress through the rule
for (; i < n; ++i) {
if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
// need to match a token
mp_token_kind_t tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK;
if (lex->tok_kind == tok_kind) {
// matched token
if (tok_kind == MP_TOKEN_NAME) {
push_result_token(&parser, rule);
}
mp_lexer_to_next(lex);
} else {
// failed to match token
if (i > 0) {
// already eaten tokens so can't backtrack
goto syntax_error;
} else {
// this rule failed, so backtrack
backtrack = true;
goto next_rule;
}
}
} else {
push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule
push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule
goto next_rule;
}
}
assert(i == n);
// matched the rule, so now build the corresponding parse_node
#if !MICROPY_ENABLE_DOC_STRING
// this code discards lonely statements, such as doc strings
if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) {
mp_parse_node_t p = peek_result(&parser, 1);
if ((MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p))
|| MP_PARSE_NODE_IS_STRUCT_KIND(p, RULE_const_object)) {
pop_result(&parser); // MP_PARSE_NODE_NULL
pop_result(&parser); // const expression (leaf or RULE_const_object)
// Pushing the "pass" rule here will overwrite any RULE_const_object
// entry that was on the result stack, allowing the GC to reclaim
// the memory from the const object when needed.
push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0);
break;
}
}
#endif
// count number of arguments for the parse node
i = 0;
size_t num_not_nil = 0;
for (size_t x = n; x > 0;) {
--x;
if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK;
if (tok_kind == MP_TOKEN_NAME) {
// only tokens which were names are pushed to stack
i += 1;
num_not_nil += 1;
}
} else {
// rules are always pushed
if (peek_result(&parser, i) != MP_PARSE_NODE_NULL) {
num_not_nil += 1;
}
i += 1;
}
}
if (num_not_nil == 1 && (rule->act & RULE_ACT_ALLOW_IDENT)) {
// this rule has only 1 argument and should not be emitted
mp_parse_node_t pn = MP_PARSE_NODE_NULL;
for (size_t x = 0; x < i; ++x) {
mp_parse_node_t pn2 = pop_result(&parser);
if (pn2 != MP_PARSE_NODE_NULL) {
pn = pn2;
}
}
push_result_node(&parser, pn);
} else {
// this rule must be emitted
if (rule->act & RULE_ACT_ADD_BLANK) {
// and add an extra blank node at the end (used by the compiler to store data)
push_result_node(&parser, MP_PARSE_NODE_NULL);
i += 1;
}
push_result_rule(&parser, rule_src_line, rule, i);
}
break;
}
default: {
assert((rule->act & RULE_ACT_KIND_MASK) == RULE_ACT_LIST);
// n=2 is: item item*
// n=1 is: item (sep item)*
// n=3 is: item (sep item)* [sep]
bool had_trailing_sep;
if (backtrack) {
list_backtrack:
had_trailing_sep = false;
if (n == 2) {
if (i == 1) {
// fail on item, first time round; propagate backtrack
goto next_rule;
} else {
// fail on item, in later rounds; finish with this rule
backtrack = false;
}
} else {
if (i == 1) {
// fail on item, first time round; propagate backtrack
goto next_rule;
} else if ((i & 1) == 1) {
// fail on item, in later rounds; have eaten tokens so can't backtrack
if (n == 3) {
// list allows trailing separator; finish parsing list
had_trailing_sep = true;
backtrack = false;
} else {
// list doesn't allowing trailing separator; fail
goto syntax_error;
}
} else {
// fail on separator; finish parsing list
backtrack = false;
}
}
} else {
for (;;) {
size_t arg = rule->arg[i & 1 & n];
if ((arg & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
if (lex->tok_kind == (arg & RULE_ARG_ARG_MASK)) {
if (i & 1 & n) {
// separators which are tokens are not pushed to result stack
} else {
push_result_token(&parser, rule);
}
mp_lexer_to_next(lex);
// got element of list, so continue parsing list
i += 1;
} else {
// couldn't get element of list
i += 1;
backtrack = true;
goto list_backtrack;
}
} else {
assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE);
push_rule(&parser, rule_src_line, rule, i + 1); // save this list-rule
push_rule_from_arg(&parser, arg); // push child of list-rule
goto next_rule;
}
}
}
assert(i >= 1);
// compute number of elements in list, result in i
i -= 1;
if ((n & 1) && (rule->arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
// don't count separators when they are tokens
i = (i + 1) / 2;
}
if (i == 1) {
// list matched single item
if (had_trailing_sep) {
// if there was a trailing separator, make a list of a single item
push_result_rule(&parser, rule_src_line, rule, i);
} else {
// just leave single item on stack (ie don't wrap in a list)
}
} else {
push_result_rule(&parser, rule_src_line, rule, i);
}
break;
}
}
}
#if MICROPY_COMP_CONST
mp_map_deinit(&parser.consts);
#endif
// truncate final chunk and link into chain of chunks
if (parser.cur_chunk != NULL) {
(void)m_renew_maybe(byte, parser.cur_chunk,
sizeof(mp_parse_chunk_t) + parser.cur_chunk->alloc,
sizeof(mp_parse_chunk_t) + parser.cur_chunk->union_.used,
false);
parser.cur_chunk->alloc = parser.cur_chunk->union_.used;
parser.cur_chunk->union_.next = parser.tree.chunk;
parser.tree.chunk = parser.cur_chunk;
}
if (
lex->tok_kind != MP_TOKEN_END // check we are at the end of the token stream
|| parser.result_stack_top == 0 // check that we got a node (can fail on empty input)
) {
syntax_error:;
mp_obj_t exc;
if (lex->tok_kind == MP_TOKEN_INDENT) {
exc = mp_obj_new_exception_msg(&mp_type_IndentationError,
"unexpected indent");
} else if (lex->tok_kind == MP_TOKEN_DEDENT_MISMATCH) {
exc = mp_obj_new_exception_msg(&mp_type_IndentationError,
"unindent does not match any outer indentation level");
} else {
exc = mp_obj_new_exception_msg(&mp_type_SyntaxError,
"invalid syntax");
}
// add traceback to give info about file name and location
// we don't have a 'block' name, so just pass the NULL qstr to indicate this
mp_obj_exception_add_traceback(exc, lex->source_name, lex->tok_line, MP_QSTR_NULL);
nlr_raise(exc);
}
// get the root parse node that we created
assert(parser.result_stack_top == 1);
parser.tree.root = parser.result_stack[0];
// free the memory that we don't need anymore
m_del(rule_stack_t, parser.rule_stack, parser.rule_stack_alloc);
m_del(mp_parse_node_t, parser.result_stack, parser.result_stack_alloc);
// we also free the lexer on behalf of the caller
mp_lexer_free(lex);
return parser.tree;
}
STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args) {
// this code does folding of arbitrary integer expressions, eg 1 + 2 * 3 + 4
// it does not do partial folding, eg 1 + 2 + x -> 3 + x
mp_obj_t arg0;
if (rule->rule_id == RULE_expr
|| rule->rule_id == RULE_xor_expr
|| rule->rule_id == RULE_and_expr) {
// folding for binary ops: | ^ &
mp_parse_node_t pn = peek_result(parser, num_args - 1);
if (!mp_parse_node_get_int_maybe(pn, &arg0)) {
return false;
}
mp_binary_op_t op;
if (rule->rule_id == RULE_expr) {
op = MP_BINARY_OP_OR;
} else if (rule->rule_id == RULE_xor_expr) {
op = MP_BINARY_OP_XOR;
} else {
op = MP_BINARY_OP_AND;
}
for (ssize_t i = num_args - 2; i >= 0; --i) {
pn = peek_result(parser, i);
mp_obj_t arg1;
if (!mp_parse_node_get_int_maybe(pn, &arg1)) {
return false;
}
arg0 = mp_binary_op(op, arg0, arg1);
}
} else if (rule->rule_id == RULE_shift_expr
|| rule->rule_id == RULE_arith_expr
|| rule->rule_id == RULE_term) {
// folding for binary ops: << >> + - * / % //
mp_parse_node_t pn = peek_result(parser, num_args - 1);
if (!mp_parse_node_get_int_maybe(pn, &arg0)) {
return false;
}
for (ssize_t i = num_args - 2; i >= 1; i -= 2) {
pn = peek_result(parser, i - 1);
mp_obj_t arg1;
if (!mp_parse_node_get_int_maybe(pn, &arg1)) {
return false;
}
mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(peek_result(parser, i));
static const uint8_t token_to_op[] = {
MP_BINARY_OP_ADD,
MP_BINARY_OP_SUBTRACT,
MP_BINARY_OP_MULTIPLY,
255,//MP_BINARY_OP_POWER,
255,//MP_BINARY_OP_TRUE_DIVIDE,
MP_BINARY_OP_FLOOR_DIVIDE,
MP_BINARY_OP_MODULO,
255,//MP_BINARY_OP_LESS
MP_BINARY_OP_LSHIFT,
255,//MP_BINARY_OP_MORE
MP_BINARY_OP_RSHIFT,
};
mp_binary_op_t op = token_to_op[tok - MP_TOKEN_OP_PLUS];
if (op == (mp_binary_op_t)255) {
return false;
}
int rhs_sign = mp_obj_int_sign(arg1);
if (op <= MP_BINARY_OP_RSHIFT) {
// << and >> can't have negative rhs
if (rhs_sign < 0) {
return false;
}
} else if (op >= MP_BINARY_OP_FLOOR_DIVIDE) {
// % and // can't have zero rhs
if (rhs_sign == 0) {
return false;
}
}
arg0 = mp_binary_op(op, arg0, arg1);
}
} else if (rule->rule_id == RULE_factor_2) {
// folding for unary ops: + - ~
mp_parse_node_t pn = peek_result(parser, 0);
if (!mp_parse_node_get_int_maybe(pn, &arg0)) {
return false;
}
mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(peek_result(parser, 1));
mp_unary_op_t op;
if (tok == MP_TOKEN_OP_PLUS) {
op = MP_UNARY_OP_POSITIVE;
} else if (tok == MP_TOKEN_OP_MINUS) {
op = MP_UNARY_OP_NEGATIVE;
} else {
assert(tok == MP_TOKEN_OP_TILDE); // should be
op = MP_UNARY_OP_INVERT;
}
arg0 = mp_unary_op(op, arg0);
#if MICROPY_COMP_CONST
} else if (rule->rule_id == RULE_expr_stmt) {
mp_parse_node_t pn1 = peek_result(parser, 0);
if (!MP_PARSE_NODE_IS_NULL(pn1)
&& !(MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_expr_stmt_augassign)
|| MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_expr_stmt_assign_list))) {
// this node is of the form <x> = <y>
mp_parse_node_t pn0 = peek_result(parser, 1);
if (MP_PARSE_NODE_IS_ID(pn0)
&& MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_atom_expr_normal)
&& MP_PARSE_NODE_IS_ID(((mp_parse_node_struct_t*)pn1)->nodes[0])
&& MP_PARSE_NODE_LEAF_ARG(((mp_parse_node_struct_t*)pn1)->nodes[0]) == MP_QSTR_const
&& MP_PARSE_NODE_IS_STRUCT_KIND(((mp_parse_node_struct_t*)pn1)->nodes[1], RULE_trailer_paren)
) {
// code to assign dynamic constants: id = const(value)
// get the id
qstr id = MP_PARSE_NODE_LEAF_ARG(pn0);
// get the value
mp_parse_node_t pn_value = ((mp_parse_node_struct_t*)((mp_parse_node_struct_t*)pn1)->nodes[1])->nodes[0];
mp_obj_t value;
if (!mp_parse_node_get_int_maybe(pn_value, &value)) {
mp_obj_t exc = mp_obj_new_exception_msg(&mp_type_SyntaxError,
"constant must be an integer");
mp_obj_exception_add_traceback(exc, parser->lexer->source_name,
((mp_parse_node_struct_t*)pn1)->source_line, MP_QSTR_NULL);
nlr_raise(exc);
}
// store the value in the table of dynamic constants
mp_map_elem_t *elem = mp_map_lookup(&parser->consts, MP_OBJ_NEW_QSTR(id), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND);
assert(elem->value == MP_OBJ_NULL);
elem->value = value;
// If the constant starts with an underscore then treat it as a private
// variable and don't emit any code to store the value to the id.
if (qstr_str(id)[0] == '_') {
pop_result(parser); // pop const(value)
pop_result(parser); // pop id
push_result_rule(parser, 0, rules[RULE_pass_stmt], 0); // replace with "pass"
return true;
}
// replace const(value) with value
pop_result(parser);
push_result_node(parser, pn_value);
// finished folding this assignment, but we still want it to be part of the tree
return false;
}
}
return false;
#endif
#if MICROPY_COMP_MODULE_CONST
} else if (rule->rule_id == RULE_atom_expr_normal) {
mp_parse_node_t pn0 = peek_result(parser, 1);
mp_parse_node_t pn1 = peek_result(parser, 0);
if (!(MP_PARSE_NODE_IS_ID(pn0)
&& MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_trailer_period))) {
return false;
}
// id1.id2
// look it up in constant table, see if it can be replaced with an integer
mp_parse_node_struct_t *pns1 = (mp_parse_node_struct_t*)pn1;
assert(MP_PARSE_NODE_IS_ID(pns1->nodes[0]));
qstr q_base = MP_PARSE_NODE_LEAF_ARG(pn0);
qstr q_attr = MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0]);
mp_map_elem_t *elem = mp_map_lookup((mp_map_t*)&mp_constants_map, MP_OBJ_NEW_QSTR(q_base), MP_MAP_LOOKUP);
if (elem == NULL) {
return false;
}
mp_obj_t dest[2];
mp_load_method_maybe(elem->value, q_attr, dest);
if (!(dest[0] != MP_OBJ_NULL && MP_OBJ_IS_INT(dest[0]) && dest[1] == MP_OBJ_NULL)) {
return false;
}
arg0 = dest[0];
#endif
} else {
return false;
}
// success folding this rule
for (size_t i = num_args; i > 0; i--) {
pop_result(parser);
}
if (MP_OBJ_IS_SMALL_INT(arg0)) {
push_result_node(parser, mp_parse_node_new_small_int(MP_OBJ_SMALL_INT_VALUE(arg0)));
} else {
// TODO reuse memory for parse node struct?
push_result_node(parser, make_node_const_object(parser, 0, arg0));
}
return true;
}
mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
// initialise parser and allocate memory for its stacks
parser_t parser;
parser.parse_error = PARSE_ERROR_NONE;
parser.rule_stack_alloc = MICROPY_ALLOC_PARSE_RULE_INIT;
parser.rule_stack_top = 0;
parser.rule_stack = m_new_maybe(rule_stack_t, parser.rule_stack_alloc);
parser.result_stack_alloc = MICROPY_ALLOC_PARSE_RESULT_INIT;
parser.result_stack_top = 0;
parser.result_stack = m_new_maybe(mp_parse_node_t, parser.result_stack_alloc);
parser.lexer = lex;
parser.tree.chunk = NULL;
parser.cur_chunk = NULL;
#if MICROPY_COMP_CONST
mp_map_init(&parser.consts, 0);
#endif
// check if we could allocate the stacks
if (parser.rule_stack == NULL || parser.result_stack == NULL) {
goto memory_error;
}
// work out the top-level rule to use, and push it on the stack
size_t top_level_rule;
switch (input_kind) {
case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break;
case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break;
default: top_level_rule = RULE_file_input;
}
push_rule(&parser, lex->tok_line, rules[top_level_rule], 0);
// parse!
size_t n, i; // state for the current rule
size_t rule_src_line; // source line for the first token matched by the current rule
bool backtrack = false;
const rule_t *rule = NULL;
for (;;) {
next_rule:
if (parser.rule_stack_top == 0 || parser.parse_error) {
break;
}
pop_rule(&parser, &rule, &i, &rule_src_line);
n = rule->act & RULE_ACT_ARG_MASK;
/*
// debugging
printf("depth=%d ", parser.rule_stack_top);
for (int j = 0; j < parser.rule_stack_top; ++j) {
printf(" ");
}
printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack);
*/
switch (rule->act & RULE_ACT_KIND_MASK) {
case RULE_ACT_OR:
if (i > 0 && !backtrack) {
goto next_rule;
} else {
backtrack = false;
}
for (; i < n; ++i) {
uint16_t kind = rule->arg[i] & RULE_ARG_KIND_MASK;
if (kind == RULE_ARG_TOK) {
if (lex->tok_kind == (rule->arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(&parser);
mp_lexer_to_next(lex);
goto next_rule;
}
} else {
assert(kind == RULE_ARG_RULE);
if (i + 1 < n) {
push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule
}
push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule
goto next_rule;
}
}
backtrack = true;
break;
case RULE_ACT_AND: {
// failed, backtrack if we can, else syntax error
if (backtrack) {
assert(i > 0);
if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) {
// an optional rule that failed, so continue with next arg
push_result_node(&parser, MP_PARSE_NODE_NULL);
backtrack = false;
} else {
// a mandatory rule that failed, so propagate backtrack
if (i > 1) {
// already eaten tokens so can't backtrack
goto syntax_error;
} else {
goto next_rule;
}
}
}
// progress through the rule
for (; i < n; ++i) {
switch (rule->arg[i] & RULE_ARG_KIND_MASK) {
case RULE_ARG_TOK: {
// need to match a token
mp_token_kind_t tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK;
if (lex->tok_kind == tok_kind) {
// matched token
if (tok_kind == MP_TOKEN_NAME) {
push_result_token(&parser);
}
mp_lexer_to_next(lex);
} else {
// failed to match token
if (i > 0) {
// already eaten tokens so can't backtrack
goto syntax_error;
} else {
// this rule failed, so backtrack
backtrack = true;
goto next_rule;
}
}
break;
}
case RULE_ARG_RULE:
case RULE_ARG_OPT_RULE:
rule_and_no_other_choice:
push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule
push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule
goto next_rule;
default:
assert(0);
goto rule_and_no_other_choice; // to help flow control analysis
}
}
assert(i == n);
// matched the rule, so now build the corresponding parse_node
#if !MICROPY_ENABLE_DOC_STRING
// this code discards lonely statements, such as doc strings
if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) {
mp_parse_node_t p = peek_result(&parser, 1);
if ((MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p)) || MP_PARSE_NODE_IS_STRUCT_KIND(p, RULE_string)) {
pop_result(&parser); // MP_PARSE_NODE_NULL
mp_parse_node_t pn = pop_result(&parser); // possibly RULE_string
if (MP_PARSE_NODE_IS_STRUCT(pn)) {
mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn;
if (MP_PARSE_NODE_STRUCT_KIND(pns) == RULE_string) {
m_del(char, (char*)pns->nodes[0], (size_t)pns->nodes[1]);
}
}
push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0);
break;
}
}
#endif
// count number of arguments for the parse node
i = 0;
size_t num_not_nil = 0;
for (size_t x = n; x > 0;) {
--x;
if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK;
if (tok_kind == MP_TOKEN_NAME) {
// only tokens which were names are pushed to stack
i += 1;
num_not_nil += 1;
}
} else {
// rules are always pushed
if (peek_result(&parser, i) != MP_PARSE_NODE_NULL) {
num_not_nil += 1;
}
i += 1;
}
}
if (num_not_nil == 1 && (rule->act & RULE_ACT_ALLOW_IDENT)) {
// this rule has only 1 argument and should not be emitted
mp_parse_node_t pn = MP_PARSE_NODE_NULL;
for (size_t x = 0; x < i; ++x) {
mp_parse_node_t pn2 = pop_result(&parser);
if (pn2 != MP_PARSE_NODE_NULL) {
pn = pn2;
}
}
push_result_node(&parser, pn);
} else {
// this rule must be emitted
if (rule->act & RULE_ACT_ADD_BLANK) {
// and add an extra blank node at the end (used by the compiler to store data)
push_result_node(&parser, MP_PARSE_NODE_NULL);
i += 1;
}
push_result_rule(&parser, rule_src_line, rule, i);
}
break;
}
mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_parse_error_kind_t *parse_error_kind_out) {
// initialise parser and allocate memory for its stacks
parser_t parser;
parser.had_memory_error = false;
parser.rule_stack_alloc = MICROPY_ALLOC_PARSE_RULE_INIT;
parser.rule_stack_top = 0;
parser.rule_stack = m_new_maybe(rule_stack_t, parser.rule_stack_alloc);
parser.result_stack_alloc = MICROPY_ALLOC_PARSE_RESULT_INIT;
parser.result_stack_top = 0;
parser.result_stack = m_new_maybe(mp_parse_node_t, parser.result_stack_alloc);
parser.lexer = lex;
// check if we could allocate the stacks
if (parser.rule_stack == NULL || parser.result_stack == NULL) {
goto memory_error;
}
// work out the top-level rule to use, and push it on the stack
int top_level_rule;
switch (input_kind) {
case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break;
case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break;
default: top_level_rule = RULE_file_input;
}
push_rule(&parser, mp_lexer_cur(lex)->src_line, rules[top_level_rule], 0);
// parse!
uint n, i; // state for the current rule
uint rule_src_line; // source line for the first token matched by the current rule
bool backtrack = false;
const rule_t *rule = NULL;
mp_token_kind_t tok_kind;
bool emit_rule;
bool had_trailing_sep;
for (;;) {
next_rule:
if (parser.rule_stack_top == 0 || parser.had_memory_error) {
break;
}
pop_rule(&parser, &rule, &i, &rule_src_line);
n = rule->act & RULE_ACT_ARG_MASK;
/*
// debugging
printf("depth=%d ", parser.rule_stack_top);
for (int j = 0; j < parser.rule_stack_top; ++j) {
printf(" ");
}
printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack);
*/
switch (rule->act & RULE_ACT_KIND_MASK) {
case RULE_ACT_OR:
if (i > 0 && !backtrack) {
goto next_rule;
} else {
backtrack = false;
}
for (; i < n - 1; ++i) {
switch (rule->arg[i] & RULE_ARG_KIND_MASK) {
case RULE_ARG_TOK:
if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(&parser, lex);
mp_lexer_to_next(lex);
goto next_rule;
}
break;
case RULE_ARG_RULE:
push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule
push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule
goto next_rule;
default:
assert(0);
}
}
if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(&parser, lex);
mp_lexer_to_next(lex);
} else {
backtrack = true;
goto next_rule;
}
} else {
push_rule_from_arg(&parser, rule->arg[i]);
}
break;
case RULE_ACT_AND:
// failed, backtrack if we can, else syntax error
if (backtrack) {
assert(i > 0);
if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) {
// an optional rule that failed, so continue with next arg
push_result_node(&parser, MP_PARSE_NODE_NULL);
backtrack = false;
} else {
// a mandatory rule that failed, so propagate backtrack
if (i > 1) {
// already eaten tokens so can't backtrack
goto syntax_error;
} else {
goto next_rule;
}
}
}
// progress through the rule
for (; i < n; ++i) {
switch (rule->arg[i] & RULE_ARG_KIND_MASK) {
case RULE_ARG_TOK:
// need to match a token
tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK;
if (mp_lexer_is_kind(lex, tok_kind)) {
// matched token
if (tok_kind == MP_TOKEN_NAME) {
push_result_token(&parser, lex);
}
mp_lexer_to_next(lex);
} else {
// failed to match token
if (i > 0) {
// already eaten tokens so can't backtrack
goto syntax_error;
} else {
// this rule failed, so backtrack
backtrack = true;
goto next_rule;
}
}
break;
case RULE_ARG_RULE:
case RULE_ARG_OPT_RULE:
push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule
push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule
goto next_rule;
default:
assert(0);
}
}
assert(i == n);
// matched the rule, so now build the corresponding parse_node
// count number of arguments for the parse_node
i = 0;
emit_rule = false;
for (int x = 0; x < n; ++x) {
if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK;
if (tok_kind >= MP_TOKEN_NAME) {
emit_rule = true;
}
if (tok_kind == MP_TOKEN_NAME) {
// only tokens which were names are pushed to stack
i += 1;
}
} else {
// rules are always pushed
i += 1;
}
}
#if !MICROPY_EMIT_CPYTHON && !MICROPY_ENABLE_DOC_STRING
// this code discards lonely statements, such as doc strings
if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) {
mp_parse_node_t p = peek_result(&parser, 1);
if ((MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p)) || MP_PARSE_NODE_IS_STRUCT_KIND(p, RULE_string)) {
pop_result(&parser);
pop_result(&parser);
push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0);
break;
}
}
#endif
// always emit these rules, even if they have only 1 argument
if (rule->rule_id == RULE_expr_stmt || rule->rule_id == RULE_yield_stmt) {
emit_rule = true;
}
// never emit these rules if they have only 1 argument
// NOTE: can't put atom_paren here because we need it to distinguisg, for example, [a,b] from [(a,b)]
// TODO possibly put varargslist_name, varargslist_equal here as well
if (rule->rule_id == RULE_else_stmt || rule->rule_id == RULE_testlist_comp_3b || rule->rule_id == RULE_import_as_names_paren || rule->rule_id == RULE_typedargslist_name || rule->rule_id == RULE_typedargslist_colon || rule->rule_id == RULE_typedargslist_equal || rule->rule_id == RULE_dictorsetmaker_colon || rule->rule_id == RULE_classdef_2 || rule->rule_id == RULE_with_item_as || rule->rule_id == RULE_assert_stmt_extra || rule->rule_id == RULE_as_name || rule->rule_id == RULE_raise_stmt_from || rule->rule_id == RULE_vfpdef) {
emit_rule = false;
}
// always emit these rules, and add an extra blank node at the end (to be used by the compiler to store data)
if (ADD_BLANK_NODE(rule->rule_id)) {
emit_rule = true;
push_result_node(&parser, MP_PARSE_NODE_NULL);
i += 1;
}
int num_not_nil = 0;
for (int x = 0; x < i; ++x) {
if (peek_result(&parser, x) != MP_PARSE_NODE_NULL) {
num_not_nil += 1;
}
}
//printf("done and %s n=%d i=%d notnil=%d\n", rule->rule_name, n, i, num_not_nil);
if (emit_rule) {
push_result_rule(&parser, rule_src_line, rule, i);
} else if (num_not_nil == 0) {
push_result_rule(&parser, rule_src_line, rule, i); // needed for, eg, atom_paren, testlist_comp_3b
//result_stack_show(parser);
//assert(0);
} else if (num_not_nil == 1) {
// single result, leave it on stack
mp_parse_node_t pn = MP_PARSE_NODE_NULL;
for (int x = 0; x < i; ++x) {
mp_parse_node_t pn2 = pop_result(&parser);
if (pn2 != MP_PARSE_NODE_NULL) {
pn = pn2;
}
}
push_result_node(&parser, pn);
} else {
push_result_rule(&parser, rule_src_line, rule, i);
}
break;
case RULE_ACT_LIST:
// n=2 is: item item*
// n=1 is: item (sep item)*
// n=3 is: item (sep item)* [sep]
if (backtrack) {
list_backtrack:
had_trailing_sep = false;
if (n == 2) {
if (i == 1) {
// fail on item, first time round; propagate backtrack
goto next_rule;
} else {
// fail on item, in later rounds; finish with this rule
backtrack = false;
}
} else {
if (i == 1) {
// fail on item, first time round; propagate backtrack
goto next_rule;
} else if ((i & 1) == 1) {
// fail on item, in later rounds; have eaten tokens so can't backtrack
if (n == 3) {
// list allows trailing separator; finish parsing list
had_trailing_sep = true;
backtrack = false;
} else {
// list doesn't allowing trailing separator; fail
goto syntax_error;
}
} else {
// fail on separator; finish parsing list
backtrack = false;
}
}
} else {
for (;;) {
uint arg = rule->arg[i & 1 & n];
switch (arg & RULE_ARG_KIND_MASK) {
case RULE_ARG_TOK:
if (mp_lexer_is_kind(lex, arg & RULE_ARG_ARG_MASK)) {
if (i & 1 & n) {
// separators which are tokens are not pushed to result stack
} else {
push_result_token(&parser, lex);
}
mp_lexer_to_next(lex);
// got element of list, so continue parsing list
i += 1;
} else {
// couldn't get element of list
i += 1;
backtrack = true;
goto list_backtrack;
}
break;
case RULE_ARG_RULE:
push_rule(&parser, rule_src_line, rule, i + 1); // save this list-rule
push_rule_from_arg(&parser, arg); // push child of list-rule
goto next_rule;
default:
assert(0);
}
}
}
assert(i >= 1);
// compute number of elements in list, result in i
i -= 1;
if ((n & 1) && (rule->arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
// don't count separators when they are tokens
i = (i + 1) / 2;
}
if (i == 1) {
// list matched single item
if (had_trailing_sep) {
// if there was a trailing separator, make a list of a single item
push_result_rule(&parser, rule_src_line, rule, i);
} else {
// just leave single item on stack (ie don't wrap in a list)
}
} else {
//printf("done list %s %d %d\n", rule->rule_name, n, i);
push_result_rule(&parser, rule_src_line, rule, i);
}
break;
default:
assert(0);
}
}
mp_parse_node_t result;
// check if we had a memory error
if (parser.had_memory_error) {
memory_error:
*parse_error_kind_out = MP_PARSE_ERROR_MEMORY;
result = MP_PARSE_NODE_NULL;
goto finished;
}
// check we are at the end of the token stream
if (!mp_lexer_is_kind(lex, MP_TOKEN_END)) {
goto syntax_error;
}
//printf("--------------\n");
//result_stack_show(parser);
//printf("rule stack alloc: %d\n", parser.rule_stack_alloc);
//printf("result stack alloc: %d\n", parser.result_stack_alloc);
//printf("number of parse nodes allocated: %d\n", num_parse_nodes_allocated);
// get the root parse node that we created
assert(parser.result_stack_top == 1);
result = parser.result_stack[0];
finished:
// free the memory that we don't need anymore
m_del(rule_stack_t, parser.rule_stack, parser.rule_stack_alloc);
m_del(mp_parse_node_t, parser.result_stack, parser.result_stack_alloc);
// return the result
return result;
syntax_error:
if (mp_lexer_is_kind(lex, MP_TOKEN_INDENT)) {
*parse_error_kind_out = MP_PARSE_ERROR_UNEXPECTED_INDENT;
} else if (mp_lexer_is_kind(lex, MP_TOKEN_DEDENT_MISMATCH)) {
*parse_error_kind_out = MP_PARSE_ERROR_UNMATCHED_UNINDENT;
} else {
*parse_error_kind_out = MP_PARSE_ERROR_INVALID_SYNTAX;
#ifdef USE_RULE_NAME
// debugging: print the rule name that failed and the token
printf("rule: %s\n", rule->rule_name);
#if MICROPY_DEBUG_PRINTERS
mp_token_show(mp_lexer_cur(lex));
#endif
#endif
}
result = MP_PARSE_NODE_NULL;
goto finished;
}