STATIC void execute_from_lexer(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, bool is_repl) {
if (lex == NULL) {
return;
}
if (0) {
// just tokenise
while (!mp_lexer_is_kind(lex, MP_TOKEN_END)) {
mp_token_show(mp_lexer_cur(lex));
mp_lexer_to_next(lex);
}
mp_lexer_free(lex);
return;
}
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, input_kind, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return;
}
qstr source_name = mp_lexer_source_name(lex);
mp_lexer_free(lex);
/*
printf("----------------\n");
mp_parse_node_print(pn, 0);
printf("----------------\n");
*/
mp_obj_t module_fun = mp_compile(pn, source_name, emit_opt, is_repl);
if (module_fun == mp_const_none) {
// compile error
return;
}
if (compile_only) {
return;
}
// execute it
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
mp_call_function_0(module_fun);
nlr_pop();
} else {
// uncaught exception
mp_obj_print_exception((mp_obj_t)nlr.ret_val);
}
}
void do_file(const char *file) {
mp_lexer_t *lex = mp_lexer_new_from_file(file);
if (lex == NULL) {
return;
}
if (0) {
// just tokenise
while (!mp_lexer_is_kind(lex, MP_TOKEN_END)) {
mp_token_show(mp_lexer_cur(lex));
mp_lexer_to_next(lex);
}
mp_lexer_free(lex);
} else {
// parse
qstr parse_exc_id;
const char *parse_exc_msg;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT, &parse_exc_id, &parse_exc_msg);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_lexer_show_error_pythonic_prefix(lex);
printf("%s: %s\n", qstr_str(parse_exc_id), parse_exc_msg);
mp_lexer_free(lex);
return;
}
mp_lexer_free(lex);
if (pn != MP_PARSE_NODE_NULL) {
//printf("----------------\n");
//mp_parse_node_print(pn, 0);
//printf("----------------\n");
// compile
mp_obj_t module_fun = mp_compile(pn, 0, false);
//printf("----------------\n");
if (module_fun == mp_const_none) {
printf("compile error\n");
}
}
}
}
void do_file(const char *file) {
mp_lexer_t *lex = mp_lexer_new_from_file(file);
if (lex == NULL) {
return;
}
if (0) {
// just tokenise
while (!mp_lexer_is_kind(lex, MP_TOKEN_END)) {
mp_token_show(mp_lexer_cur(lex));
mp_lexer_to_next(lex);
}
mp_lexer_free(lex);
} else {
// parse
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return;
}
mp_lexer_free(lex);
if (pn != MP_PARSE_NODE_NULL) {
//printf("----------------\n");
//mp_parse_node_print(pn, 0);
//printf("----------------\n");
// compile
mp_obj_t module_fun = mp_compile(pn, 0, false);
//printf("----------------\n");
if (module_fun == mp_const_none) {
printf("compile error\n");
}
}
}
}
mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
// allocate memory for the parser and its stacks
parser_t *parser = m_new_obj(parser_t);
parser->rule_stack_alloc = 64;
parser->rule_stack_top = 0;
parser->rule_stack = m_new(rule_stack_t, parser->rule_stack_alloc);
parser->result_stack_alloc = 64;
parser->result_stack_top = 0;
parser->result_stack = m_new(mp_parse_node_t, parser->result_stack_alloc);
// work out the top-level rule to use, and push it on the stack
int top_level_rule;
switch (input_kind) {
case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break;
//case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break;
default: top_level_rule = RULE_file_input;
}
push_rule(parser, rules[top_level_rule], 0);
// parse!
uint n, i;
bool backtrack = false;
const rule_t *rule;
mp_token_kind_t tok_kind;
bool emit_rule;
bool had_trailing_sep;
for (;;) {
next_rule:
if (parser->rule_stack_top == 0) {
break;
}
pop_rule(parser, &rule, &i);
n = rule->act & RULE_ACT_ARG_MASK;
/*
// debugging
printf("depth=%d ", parser->rule_stack_top);
for (int j = 0; j < parser->rule_stack_top; ++j) {
printf(" ");
}
printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack);
*/
switch (rule->act & RULE_ACT_KIND_MASK) {
case RULE_ACT_OR:
if (i > 0 && !backtrack) {
goto next_rule;
} else {
backtrack = false;
}
for (; i < n - 1; ++i) {
switch (rule->arg[i] & RULE_ARG_KIND_MASK) {
case RULE_ARG_TOK:
if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(parser, lex);
mp_lexer_to_next(lex);
goto next_rule;
}
break;
case RULE_ARG_RULE:
push_rule(parser, rule, i + 1);
push_rule_from_arg(parser, rule->arg[i]);
goto next_rule;
default:
assert(0);
}
}
if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(parser, lex);
mp_lexer_to_next(lex);
} else {
backtrack = true;
goto next_rule;
}
} else {
push_rule_from_arg(parser, rule->arg[i]);
}
break;
case RULE_ACT_AND:
// failed, backtrack if we can, else syntax error
if (backtrack) {
assert(i > 0);
if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) {
// an optional rule that failed, so continue with next arg
push_result_node(parser, MP_PARSE_NODE_NULL);
backtrack = false;
} else {
// a mandatory rule that failed, so propagate backtrack
if (i > 1) {
// already eaten tokens so can't backtrack
goto syntax_error;
} else {
goto next_rule;
}
}
}
// progress through the rule
for (; i < n; ++i) {
switch (rule->arg[i] & RULE_ARG_KIND_MASK) {
case RULE_ARG_TOK:
// need to match a token
tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK;
if (mp_lexer_is_kind(lex, tok_kind)) {
// matched token
if (tok_kind == MP_TOKEN_NAME) {
push_result_token(parser, lex);
}
mp_lexer_to_next(lex);
} else {
// failed to match token
if (i > 0) {
// already eaten tokens so can't backtrack
goto syntax_error;
} else {
// this rule failed, so backtrack
backtrack = true;
goto next_rule;
}
}
break;
case RULE_ARG_RULE:
//if (i + 1 < n) {
push_rule(parser, rule, i + 1);
//}
push_rule_from_arg(parser, rule->arg[i]);
goto next_rule;
case RULE_ARG_OPT_RULE:
push_rule(parser, rule, i + 1);
push_rule_from_arg(parser, rule->arg[i]);
goto next_rule;
default:
assert(0);
}
}
assert(i == n);
// matched the rule, so now build the corresponding parse_node
// count number of arguments for the parse_node
i = 0;
emit_rule = false;
for (int x = 0; x < n; ++x) {
if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK;
if (tok_kind >= MP_TOKEN_NAME) {
emit_rule = true;
}
if (tok_kind == MP_TOKEN_NAME) {
// only tokens which were names are pushed to stack
i += 1;
}
} else {
// rules are always pushed
i += 1;
}
}
// always emit these rules, even if they have only 1 argument
if (rule->rule_id == RULE_expr_stmt || rule->rule_id == RULE_yield_stmt) {
emit_rule = true;
}
// never emit these rules if they have only 1 argument
// NOTE: can't put atom_paren here because we need it to distinguisg, for example, [a,b] from [(a,b)]
// TODO possibly put varargslist_name, varargslist_equal here as well
if (rule->rule_id == RULE_else_stmt || rule->rule_id == RULE_testlist_comp_3b || rule->rule_id == RULE_import_as_names_paren || rule->rule_id == RULE_typedargslist_name || rule->rule_id == RULE_typedargslist_colon || rule->rule_id == RULE_typedargslist_equal || rule->rule_id == RULE_dictorsetmaker_colon || rule->rule_id == RULE_classdef_2 || rule->rule_id == RULE_with_item_as || rule->rule_id == RULE_assert_stmt_extra || rule->rule_id == RULE_as_name || rule->rule_id == RULE_raise_stmt_from || rule->rule_id == RULE_vfpdef) {
emit_rule = false;
}
// always emit these rules, and add an extra blank node at the end (to be used by the compiler to store data)
if (rule->rule_id == RULE_funcdef || rule->rule_id == RULE_classdef || rule->rule_id == RULE_comp_for || rule->rule_id == RULE_lambdef || rule->rule_id == RULE_lambdef_nocond) {
emit_rule = true;
push_result_node(parser, MP_PARSE_NODE_NULL);
i += 1;
}
int num_not_nil = 0;
for (int x = 0; x < i; ++x) {
if (peek_result(parser, x) != MP_PARSE_NODE_NULL) {
num_not_nil += 1;
}
}
//printf("done and %s n=%d i=%d notnil=%d\n", rule->rule_name, n, i, num_not_nil);
if (emit_rule) {
push_result_rule(parser, rule, i);
} else if (num_not_nil == 0) {
push_result_rule(parser, rule, i); // needed for, eg, atom_paren, testlist_comp_3b
//result_stack_show(parser);
//assert(0);
} else if (num_not_nil == 1) {
// single result, leave it on stack
mp_parse_node_t pn = MP_PARSE_NODE_NULL;
for (int x = 0; x < i; ++x) {
mp_parse_node_t pn2 = pop_result(parser);
if (pn2 != MP_PARSE_NODE_NULL) {
pn = pn2;
}
}
push_result_node(parser, pn);
} else {
push_result_rule(parser, rule, i);
}
break;
case RULE_ACT_LIST:
// n=2 is: item item*
// n=1 is: item (sep item)*
// n=3 is: item (sep item)* [sep]
if (backtrack) {
list_backtrack:
had_trailing_sep = false;
if (n == 2) {
if (i == 1) {
// fail on item, first time round; propagate backtrack
goto next_rule;
} else {
// fail on item, in later rounds; finish with this rule
backtrack = false;
}
} else {
if (i == 1) {
// fail on item, first time round; propagate backtrack
goto next_rule;
} else if ((i & 1) == 1) {
// fail on item, in later rounds; have eaten tokens so can't backtrack
if (n == 3) {
// list allows trailing separator; finish parsing list
had_trailing_sep = true;
backtrack = false;
} else {
// list doesn't allowing trailing separator; fail
goto syntax_error;
}
} else {
// fail on separator; finish parsing list
backtrack = false;
}
}
} else {
for (;;) {
uint arg = rule->arg[i & 1 & n];
switch (arg & RULE_ARG_KIND_MASK) {
case RULE_ARG_TOK:
if (mp_lexer_is_kind(lex, arg & RULE_ARG_ARG_MASK)) {
if (i & 1 & n) {
// separators which are tokens are not pushed to result stack
} else {
push_result_token(parser, lex);
}
mp_lexer_to_next(lex);
// got element of list, so continue parsing list
i += 1;
} else {
// couldn't get element of list
i += 1;
backtrack = true;
goto list_backtrack;
}
break;
case RULE_ARG_RULE:
push_rule(parser, rule, i + 1);
push_rule_from_arg(parser, arg);
goto next_rule;
default:
assert(0);
}
}
}
assert(i >= 1);
// compute number of elements in list, result in i
i -= 1;
if ((n & 1) && (rule->arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
// don't count separators when they are tokens
i = (i + 1) / 2;
}
if (i == 1) {
// list matched single item
if (had_trailing_sep) {
// if there was a trailing separator, make a list of a single item
push_result_rule(parser, rule, i);
} else {
// just leave single item on stack (ie don't wrap in a list)
}
} else {
//printf("done list %s %d %d\n", rule->rule_name, n, i);
push_result_rule(parser, rule, i);
}
break;
default:
assert(0);
}
}
// check we are at the end of the token stream
if (!mp_lexer_is_kind(lex, MP_TOKEN_END)) {
goto syntax_error;
}
//printf("--------------\n");
//result_stack_show(parser);
//printf("rule stack alloc: %d\n", parser->rule_stack_alloc);
//printf("result stack alloc: %d\n", parser->result_stack_alloc);
//printf("number of parse nodes allocated: %d\n", num_parse_nodes_allocated);
// get the root parse node that we created
assert(parser->result_stack_top == 1);
mp_parse_node_t result = parser->result_stack[0];
finished:
// free the memory that we don't need anymore
m_del(rule_stack_t, parser->rule_stack, parser->rule_stack_alloc);
m_del(mp_parse_node_t, parser->result_stack, parser->result_stack_alloc);
m_del_obj(parser_t, parser);
// return the result
return result;
syntax_error:
// TODO these should raise a proper exception
if (mp_lexer_is_kind(lex, MP_TOKEN_INDENT)) {
mp_lexer_show_error_pythonic(lex, "IndentationError: unexpected indent");
} else if (mp_lexer_is_kind(lex, MP_TOKEN_DEDENT_MISMATCH)) {
mp_lexer_show_error_pythonic(lex, "IndentationError: unindent does not match any outer indentation level");
} else {
mp_lexer_show_error_pythonic(lex, "syntax error:");
#ifdef USE_RULE_NAME
mp_lexer_show_error(lex, rule->rule_name);
#endif
mp_token_show(mp_lexer_cur(lex));
}
result = MP_PARSE_NODE_NULL;
goto finished;
}
static void push_result_token(parser_t *parser, const mp_lexer_t *lex) {
const mp_token_t *tok = mp_lexer_cur(lex);
mp_parse_node_t pn;
if (tok->kind == MP_TOKEN_NAME) {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_ID, qstr_from_strn_copy(tok->str, tok->len));
} else if (tok->kind == MP_TOKEN_NUMBER) {
bool dec = false;
bool small_int = true;
machine_int_t int_val = 0;
int len = tok->len;
const char *str = tok->str;
int base = 10;
int i = 0;
if (len >= 3 && str[0] == '0') {
if (str[1] == 'o' || str[1] == 'O') {
// octal
base = 8;
i = 2;
} else if (str[1] == 'x' || str[1] == 'X') {
// hexadecimal
base = 16;
i = 2;
} else if (str[1] == 'b' || str[1] == 'B') {
// binary
base = 2;
i = 2;
}
}
bool overflow = false;
for (; i < len; i++) {
machine_int_t old_val = int_val;
if (unichar_isdigit(str[i]) && str[i] - '0' < base) {
int_val = base * int_val + str[i] - '0';
} else if (base == 16 && 'a' <= str[i] && str[i] <= 'f') {
int_val = base * int_val + str[i] - 'a' + 10;
} else if (base == 16 && 'A' <= str[i] && str[i] <= 'F') {
int_val = base * int_val + str[i] - 'A' + 10;
} else if (str[i] == '.' || str[i] == 'e' || str[i] == 'E' || str[i] == 'j' || str[i] == 'J') {
dec = true;
break;
} else {
small_int = false;
break;
}
if (int_val < old_val) {
// If new value became less than previous, it's overflow
overflow = true;
} else if ((old_val ^ int_val) & WORD_MSBIT_HIGH) {
// If signed number changed sign - it's overflow
overflow = true;
}
}
if (dec) {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_DECIMAL, qstr_from_strn_copy(str, len));
} else if (small_int && !overflow && MP_FIT_SMALL_INT(int_val)) {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_SMALL_INT, int_val);
} else {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_INTEGER, qstr_from_strn_copy(str, len));
}
} else if (tok->kind == MP_TOKEN_STRING) {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_STRING, qstr_from_strn_copy(tok->str, tok->len));
} else if (tok->kind == MP_TOKEN_BYTES) {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_BYTES, qstr_from_strn_copy(tok->str, tok->len));
} else {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_TOKEN, tok->kind);
}
push_result_node(parser, pn);
}
// returns standard error codes: 0 for success, 1 for all other errors
STATIC int execute_from_lexer(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, bool is_repl) {
if (lex == NULL) {
return 1;
}
if (0) {
// just tokenise
while (!mp_lexer_is_kind(lex, MP_TOKEN_END)) {
mp_token_show(mp_lexer_cur(lex));
mp_lexer_to_next(lex);
}
mp_lexer_free(lex);
return 0;
}
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, input_kind, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return 1;
}
qstr source_name = mp_lexer_source_name(lex);
#if MICROPY_PY___FILE__
if (input_kind == MP_PARSE_FILE_INPUT) {
mp_store_global(MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name));
}
#endif
mp_lexer_free(lex);
/*
printf("----------------\n");
mp_parse_node_print(pn, 0);
printf("----------------\n");
*/
mp_obj_t module_fun = mp_compile(pn, source_name, emit_opt, is_repl);
if (module_fun == mp_const_none) {
// compile error
return 1;
}
if (compile_only) {
return 0;
}
// execute it
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
mp_call_function_0(module_fun);
nlr_pop();
return 0;
} else {
// uncaught exception
// check for SystemExit
mp_obj_t exc = (mp_obj_t)nlr.ret_val;
if (mp_obj_is_subclass_fast(mp_obj_get_type(exc), &mp_type_SystemExit)) {
mp_obj_t exit_val = mp_obj_exception_get_value(exc);
mp_int_t val;
if (!mp_obj_get_int_maybe(exit_val, &val)) {
val = 0;
}
exit(val);
}
mp_obj_print_exception((mp_obj_t)nlr.ret_val);
return 1;
}
}
STATIC void push_rule_from_arg(parser_t *parser, mp_uint_t arg) {
assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE || (arg & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE);
mp_uint_t rule_id = arg & RULE_ARG_ARG_MASK;
assert(rule_id < RULE_maximum_number_of);
push_rule(parser, mp_lexer_cur(parser->lexer)->src_line, rules[rule_id], 0);
}
mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_parse_error_kind_t *parse_error_kind_out) {
// initialise parser and allocate memory for its stacks
parser_t parser;
parser.had_memory_error = false;
parser.rule_stack_alloc = MICROPY_ALLOC_PARSE_RULE_INIT;
parser.rule_stack_top = 0;
parser.rule_stack = m_new_maybe(rule_stack_t, parser.rule_stack_alloc);
parser.result_stack_alloc = MICROPY_ALLOC_PARSE_RESULT_INIT;
parser.result_stack_top = 0;
parser.result_stack = m_new_maybe(mp_parse_node_t, parser.result_stack_alloc);
parser.lexer = lex;
// check if we could allocate the stacks
if (parser.rule_stack == NULL || parser.result_stack == NULL) {
goto memory_error;
}
// work out the top-level rule to use, and push it on the stack
int top_level_rule;
switch (input_kind) {
case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break;
case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break;
default: top_level_rule = RULE_file_input;
}
push_rule(&parser, mp_lexer_cur(lex)->src_line, rules[top_level_rule], 0);
// parse!
uint n, i; // state for the current rule
uint rule_src_line; // source line for the first token matched by the current rule
bool backtrack = false;
const rule_t *rule = NULL;
mp_token_kind_t tok_kind;
bool emit_rule;
bool had_trailing_sep;
for (;;) {
next_rule:
if (parser.rule_stack_top == 0 || parser.had_memory_error) {
break;
}
pop_rule(&parser, &rule, &i, &rule_src_line);
n = rule->act & RULE_ACT_ARG_MASK;
/*
// debugging
printf("depth=%d ", parser.rule_stack_top);
for (int j = 0; j < parser.rule_stack_top; ++j) {
printf(" ");
}
printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack);
*/
switch (rule->act & RULE_ACT_KIND_MASK) {
case RULE_ACT_OR:
if (i > 0 && !backtrack) {
goto next_rule;
} else {
backtrack = false;
}
for (; i < n - 1; ++i) {
switch (rule->arg[i] & RULE_ARG_KIND_MASK) {
case RULE_ARG_TOK:
if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(&parser, lex);
mp_lexer_to_next(lex);
goto next_rule;
}
break;
case RULE_ARG_RULE:
push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule
push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule
goto next_rule;
default:
assert(0);
}
}
if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(&parser, lex);
mp_lexer_to_next(lex);
} else {
backtrack = true;
goto next_rule;
}
} else {
push_rule_from_arg(&parser, rule->arg[i]);
}
break;
case RULE_ACT_AND:
// failed, backtrack if we can, else syntax error
if (backtrack) {
assert(i > 0);
if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) {
// an optional rule that failed, so continue with next arg
push_result_node(&parser, MP_PARSE_NODE_NULL);
backtrack = false;
} else {
// a mandatory rule that failed, so propagate backtrack
if (i > 1) {
// already eaten tokens so can't backtrack
goto syntax_error;
} else {
goto next_rule;
}
}
}
// progress through the rule
for (; i < n; ++i) {
switch (rule->arg[i] & RULE_ARG_KIND_MASK) {
case RULE_ARG_TOK:
// need to match a token
tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK;
if (mp_lexer_is_kind(lex, tok_kind)) {
// matched token
if (tok_kind == MP_TOKEN_NAME) {
push_result_token(&parser, lex);
}
mp_lexer_to_next(lex);
} else {
// failed to match token
if (i > 0) {
// already eaten tokens so can't backtrack
goto syntax_error;
} else {
// this rule failed, so backtrack
backtrack = true;
goto next_rule;
}
}
break;
case RULE_ARG_RULE:
case RULE_ARG_OPT_RULE:
push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule
push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule
goto next_rule;
default:
assert(0);
}
}
assert(i == n);
// matched the rule, so now build the corresponding parse_node
// count number of arguments for the parse_node
i = 0;
emit_rule = false;
for (int x = 0; x < n; ++x) {
if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK;
if (tok_kind >= MP_TOKEN_NAME) {
emit_rule = true;
}
if (tok_kind == MP_TOKEN_NAME) {
// only tokens which were names are pushed to stack
i += 1;
}
} else {
// rules are always pushed
i += 1;
}
}
#if !MICROPY_EMIT_CPYTHON && !MICROPY_ENABLE_DOC_STRING
// this code discards lonely statements, such as doc strings
if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) {
mp_parse_node_t p = peek_result(&parser, 1);
if ((MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p)) || MP_PARSE_NODE_IS_STRUCT_KIND(p, RULE_string)) {
pop_result(&parser);
pop_result(&parser);
push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0);
break;
}
}
#endif
// always emit these rules, even if they have only 1 argument
if (rule->rule_id == RULE_expr_stmt || rule->rule_id == RULE_yield_stmt) {
emit_rule = true;
}
// never emit these rules if they have only 1 argument
// NOTE: can't put atom_paren here because we need it to distinguisg, for example, [a,b] from [(a,b)]
// TODO possibly put varargslist_name, varargslist_equal here as well
if (rule->rule_id == RULE_else_stmt || rule->rule_id == RULE_testlist_comp_3b || rule->rule_id == RULE_import_as_names_paren || rule->rule_id == RULE_typedargslist_name || rule->rule_id == RULE_typedargslist_colon || rule->rule_id == RULE_typedargslist_equal || rule->rule_id == RULE_dictorsetmaker_colon || rule->rule_id == RULE_classdef_2 || rule->rule_id == RULE_with_item_as || rule->rule_id == RULE_assert_stmt_extra || rule->rule_id == RULE_as_name || rule->rule_id == RULE_raise_stmt_from || rule->rule_id == RULE_vfpdef) {
emit_rule = false;
}
// always emit these rules, and add an extra blank node at the end (to be used by the compiler to store data)
if (ADD_BLANK_NODE(rule->rule_id)) {
emit_rule = true;
push_result_node(&parser, MP_PARSE_NODE_NULL);
i += 1;
}
int num_not_nil = 0;
for (int x = 0; x < i; ++x) {
if (peek_result(&parser, x) != MP_PARSE_NODE_NULL) {
num_not_nil += 1;
}
}
//printf("done and %s n=%d i=%d notnil=%d\n", rule->rule_name, n, i, num_not_nil);
if (emit_rule) {
push_result_rule(&parser, rule_src_line, rule, i);
} else if (num_not_nil == 0) {
push_result_rule(&parser, rule_src_line, rule, i); // needed for, eg, atom_paren, testlist_comp_3b
//result_stack_show(parser);
//assert(0);
} else if (num_not_nil == 1) {
// single result, leave it on stack
mp_parse_node_t pn = MP_PARSE_NODE_NULL;
for (int x = 0; x < i; ++x) {
mp_parse_node_t pn2 = pop_result(&parser);
if (pn2 != MP_PARSE_NODE_NULL) {
pn = pn2;
}
}
push_result_node(&parser, pn);
} else {
push_result_rule(&parser, rule_src_line, rule, i);
}
break;
case RULE_ACT_LIST:
// n=2 is: item item*
// n=1 is: item (sep item)*
// n=3 is: item (sep item)* [sep]
if (backtrack) {
list_backtrack:
had_trailing_sep = false;
if (n == 2) {
if (i == 1) {
// fail on item, first time round; propagate backtrack
goto next_rule;
} else {
// fail on item, in later rounds; finish with this rule
backtrack = false;
}
} else {
if (i == 1) {
// fail on item, first time round; propagate backtrack
goto next_rule;
} else if ((i & 1) == 1) {
// fail on item, in later rounds; have eaten tokens so can't backtrack
if (n == 3) {
// list allows trailing separator; finish parsing list
had_trailing_sep = true;
backtrack = false;
} else {
// list doesn't allowing trailing separator; fail
goto syntax_error;
}
} else {
// fail on separator; finish parsing list
backtrack = false;
}
}
} else {
for (;;) {
uint arg = rule->arg[i & 1 & n];
switch (arg & RULE_ARG_KIND_MASK) {
case RULE_ARG_TOK:
if (mp_lexer_is_kind(lex, arg & RULE_ARG_ARG_MASK)) {
if (i & 1 & n) {
// separators which are tokens are not pushed to result stack
} else {
push_result_token(&parser, lex);
}
mp_lexer_to_next(lex);
// got element of list, so continue parsing list
i += 1;
} else {
// couldn't get element of list
i += 1;
backtrack = true;
goto list_backtrack;
}
break;
case RULE_ARG_RULE:
push_rule(&parser, rule_src_line, rule, i + 1); // save this list-rule
push_rule_from_arg(&parser, arg); // push child of list-rule
goto next_rule;
default:
assert(0);
}
}
}
assert(i >= 1);
// compute number of elements in list, result in i
i -= 1;
if ((n & 1) && (rule->arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
// don't count separators when they are tokens
i = (i + 1) / 2;
}
if (i == 1) {
// list matched single item
if (had_trailing_sep) {
// if there was a trailing separator, make a list of a single item
push_result_rule(&parser, rule_src_line, rule, i);
} else {
// just leave single item on stack (ie don't wrap in a list)
}
} else {
//printf("done list %s %d %d\n", rule->rule_name, n, i);
push_result_rule(&parser, rule_src_line, rule, i);
}
break;
default:
assert(0);
}
}
mp_parse_node_t result;
// check if we had a memory error
if (parser.had_memory_error) {
memory_error:
*parse_error_kind_out = MP_PARSE_ERROR_MEMORY;
result = MP_PARSE_NODE_NULL;
goto finished;
}
// check we are at the end of the token stream
if (!mp_lexer_is_kind(lex, MP_TOKEN_END)) {
goto syntax_error;
}
//printf("--------------\n");
//result_stack_show(parser);
//printf("rule stack alloc: %d\n", parser.rule_stack_alloc);
//printf("result stack alloc: %d\n", parser.result_stack_alloc);
//printf("number of parse nodes allocated: %d\n", num_parse_nodes_allocated);
// get the root parse node that we created
assert(parser.result_stack_top == 1);
result = parser.result_stack[0];
finished:
// free the memory that we don't need anymore
m_del(rule_stack_t, parser.rule_stack, parser.rule_stack_alloc);
m_del(mp_parse_node_t, parser.result_stack, parser.result_stack_alloc);
// return the result
return result;
syntax_error:
if (mp_lexer_is_kind(lex, MP_TOKEN_INDENT)) {
*parse_error_kind_out = MP_PARSE_ERROR_UNEXPECTED_INDENT;
} else if (mp_lexer_is_kind(lex, MP_TOKEN_DEDENT_MISMATCH)) {
*parse_error_kind_out = MP_PARSE_ERROR_UNMATCHED_UNINDENT;
} else {
*parse_error_kind_out = MP_PARSE_ERROR_INVALID_SYNTAX;
#ifdef USE_RULE_NAME
// debugging: print the rule name that failed and the token
printf("rule: %s\n", rule->rule_name);
#if MICROPY_DEBUG_PRINTERS
mp_token_show(mp_lexer_cur(lex));
#endif
#endif
}
result = MP_PARSE_NODE_NULL;
goto finished;
}
STATIC void push_result_token(parser_t *parser, const mp_lexer_t *lex) {
const mp_token_t *tok = mp_lexer_cur(lex);
mp_parse_node_t pn;
if (tok->kind == MP_TOKEN_NAME) {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_ID, qstr_from_strn(tok->str, tok->len));
} else if (tok->kind == MP_TOKEN_NUMBER) {
bool dec = false;
bool small_int = true;
machine_int_t int_val = 0;
int len = tok->len;
const char *str = tok->str;
int base = 0;
int i = mp_parse_num_base(str, len, &base);
bool overflow = false;
for (; i < len; i++) {
int dig;
if (unichar_isdigit(str[i]) && str[i] - '0' < base) {
dig = str[i] - '0';
} else if (base == 16 && 'a' <= str[i] && str[i] <= 'f') {
dig = str[i] - 'a' + 10;
} else if (base == 16 && 'A' <= str[i] && str[i] <= 'F') {
dig = str[i] - 'A' + 10;
} else if (str[i] == '.' || str[i] == 'e' || str[i] == 'E' || str[i] == 'j' || str[i] == 'J') {
dec = true;
break;
} else {
small_int = false;
break;
}
// add next digi and check for overflow
if (mp_small_int_mul_overflow(int_val, base)) {
overflow = true;
}
int_val = int_val * base + dig;
if (!MP_SMALL_INT_FITS(int_val)) {
overflow = true;
}
}
if (dec) {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_DECIMAL, qstr_from_strn(str, len));
} else if (small_int && !overflow && MP_SMALL_INT_FITS(int_val)) {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_SMALL_INT, int_val);
} else {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_INTEGER, qstr_from_strn(str, len));
}
} else if (tok->kind == MP_TOKEN_STRING) {
// Don't automatically intern all strings. doc strings (which are usually large)
// will be discarded by the compiler, and so we shouldn't intern them.
qstr qst = MP_QSTR_NULL;
if (tok->len <= MICROPY_ALLOC_PARSE_INTERN_STRING_LEN) {
// intern short strings
qst = qstr_from_strn(tok->str, tok->len);
} else {
// check if this string is already interned
qst = qstr_find_strn(tok->str, tok->len);
}
if (qst != MP_QSTR_NULL) {
// qstr exists, make a leaf node
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_STRING, qst);
} else {
// not interned, make a node holding a pointer to the string data
push_result_string(parser, mp_lexer_cur(lex)->src_line, tok->str, tok->len);
return;
}
} else if (tok->kind == MP_TOKEN_BYTES) {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_BYTES, qstr_from_strn(tok->str, tok->len));
} else {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_TOKEN, tok->kind);
}
push_result_node(parser, pn);
}
