STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
// create the lexer
mp_lexer_t *lex = mp_lexer_new_from_file(vstr_str(file));
if (lex == NULL) {
// we verified the file exists using stat, but lexer could still fail
nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ImportError, "No module named '%s'", vstr_str(file)));
}
qstr source_name = mp_lexer_source_name(lex);
// save the old context
mp_obj_dict_t *old_locals = mp_locals_get();
mp_obj_dict_t *old_globals = mp_globals_get();
// set the new context
mp_locals_set(mp_obj_module_get_globals(module_obj));
mp_globals_set(mp_obj_module_get_globals(module_obj));
// parse the imported script
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error; clean up and raise exception
mp_obj_t exc = mp_parse_make_exception(lex, parse_error_kind);
mp_lexer_free(lex);
mp_locals_set(old_locals);
mp_globals_set(old_globals);
nlr_raise(exc);
}
mp_lexer_free(lex);
// compile the imported script
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, false);
mp_parse_node_free(pn);
if (module_fun == mp_const_none) {
// TODO handle compile error correctly
mp_locals_set(old_locals);
mp_globals_set(old_globals);
nlr_raise(mp_obj_new_exception_msg(&mp_type_SyntaxError, "Syntax error in imported module"));
}
// complied successfully, execute it
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
mp_call_function_0(module_fun);
nlr_pop();
} else {
// exception; restore context and re-raise same exception
mp_locals_set(old_locals);
mp_globals_set(old_globals);
nlr_raise(nlr.ret_val);
}
mp_locals_set(old_locals);
mp_globals_set(old_globals);
}
STATIC void execute_from_lexer(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, bool is_repl) {
if (lex == NULL) {
return;
}
if (0) {
// just tokenise
while (!mp_lexer_is_kind(lex, MP_TOKEN_END)) {
mp_token_show(mp_lexer_cur(lex));
mp_lexer_to_next(lex);
}
mp_lexer_free(lex);
return;
}
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, input_kind, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return;
}
qstr source_name = mp_lexer_source_name(lex);
mp_lexer_free(lex);
/*
printf("----------------\n");
mp_parse_node_print(pn, 0);
printf("----------------\n");
*/
mp_obj_t module_fun = mp_compile(pn, source_name, emit_opt, is_repl);
if (module_fun == mp_const_none) {
// compile error
return;
}
if (compile_only) {
return;
}
// execute it
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
mp_call_function_0(module_fun);
nlr_pop();
} else {
// uncaught exception
mp_obj_print_exception((mp_obj_t)nlr.ret_val);
}
}
mp_lexer_t *mp_lexer_new(qstr src_name, void *stream_data, mp_lexer_stream_next_byte_t stream_next_byte, mp_lexer_stream_close_t stream_close) {
mp_lexer_t *lex = m_new_obj_maybe(mp_lexer_t);
// check for memory allocation error
if (lex == NULL) {
if (stream_close) {
stream_close(stream_data);
}
return NULL;
}
lex->source_name = src_name;
lex->stream_data = stream_data;
lex->stream_next_byte = stream_next_byte;
lex->stream_close = stream_close;
lex->line = 1;
lex->column = 1;
lex->emit_dent = 0;
lex->nested_bracket_level = 0;
lex->alloc_indent_level = MICROPY_ALLOC_LEXER_INDENT_INIT;
lex->num_indent_level = 1;
lex->indent_level = m_new_maybe(uint16_t, lex->alloc_indent_level);
vstr_init(&lex->vstr, 32);
// check for memory allocation error
if (lex->indent_level == NULL || vstr_had_error(&lex->vstr)) {
mp_lexer_free(lex);
return NULL;
}
// store sentinel for first indentation level
lex->indent_level[0] = 0;
// preload characters
lex->chr0 = stream_next_byte(stream_data);
lex->chr1 = stream_next_byte(stream_data);
lex->chr2 = stream_next_byte(stream_data);
// if input stream is 0, 1 or 2 characters long and doesn't end in a newline, then insert a newline at the end
if (lex->chr0 == MP_LEXER_EOF) {
lex->chr0 = '\n';
} else if (lex->chr1 == MP_LEXER_EOF) {
if (lex->chr0 == '\r') {
lex->chr0 = '\n';
} else if (lex->chr0 != '\n') {
lex->chr1 = '\n';
}
} else if (lex->chr2 == MP_LEXER_EOF) {
if (lex->chr1 == '\r') {
lex->chr1 = '\n';
} else if (lex->chr1 != '\n') {
lex->chr2 = '\n';
}
}
// preload first token
mp_lexer_next_token_into(lex, true);
return lex;
}
static mp_obj_t mp_builtin_eval(mp_obj_t o_in) {
uint str_len;
const byte *str = mp_obj_str_get_data(o_in, &str_len);
// create the lexer
mp_lexer_t *lex = mp_lexer_new_from_str_len(MP_QSTR__lt_string_gt_, (const char*)str, str_len, 0);
qstr source_name = mp_lexer_source_name(lex);
// parse the string
qstr parse_exc_id;
const char *parse_exc_msg;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_EVAL_INPUT, &parse_exc_id, &parse_exc_msg);
mp_lexer_free(lex);
if (pn == MP_PARSE_NODE_NULL) {
// parse error; raise exception
nlr_jump(mp_obj_new_exception_msg(parse_exc_id, parse_exc_msg));
}
// compile the string
mp_obj_t module_fun = mp_compile(pn, source_name, false);
mp_parse_node_free(pn);
if (module_fun == mp_const_none) {
// TODO handle compile error correctly
return mp_const_none;
}
// complied successfully, execute it
return rt_call_function_0(module_fun);
}
STATIC mp_obj_t parse_compile_execute(mp_obj_t o_in, mp_parse_input_kind_t parse_input_kind) {
uint str_len;
const char *str = mp_obj_str_get_data(o_in, &str_len);
// create the lexer
mp_lexer_t *lex = mp_lexer_new_from_str_len(MP_QSTR__lt_string_gt_, str, str_len, 0);
qstr source_name = mp_lexer_source_name(lex);
// parse the string
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, parse_input_kind, &parse_error_kind);
mp_lexer_free(lex);
if (pn == MP_PARSE_NODE_NULL) {
// parse error; raise exception
nlr_jump(mp_parse_make_exception(parse_error_kind));
}
// compile the string
mp_obj_t module_fun = mp_compile(pn, source_name, false);
mp_parse_node_free(pn);
if (module_fun == mp_const_none) {
// TODO handle compile error correctly
return mp_const_none;
}
// complied successfully, execute it
return rt_call_function_0(module_fun);
}
void do_file(const char *file) {
mp_lexer_t *lex = mp_lexer_new_from_file(file);
if (lex == NULL) {
return;
}
if (0) {
// just tokenise
while (!mp_lexer_is_kind(lex, MP_TOKEN_END)) {
mp_token_show(mp_lexer_cur(lex));
mp_lexer_to_next(lex);
}
mp_lexer_free(lex);
} else {
// parse
qstr parse_exc_id;
const char *parse_exc_msg;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT, &parse_exc_id, &parse_exc_msg);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_lexer_show_error_pythonic_prefix(lex);
printf("%s: %s\n", qstr_str(parse_exc_id), parse_exc_msg);
mp_lexer_free(lex);
return;
}
mp_lexer_free(lex);
if (pn != MP_PARSE_NODE_NULL) {
//printf("----------------\n");
//mp_parse_node_print(pn, 0);
//printf("----------------\n");
// compile
mp_obj_t module_fun = mp_compile(pn, 0, false);
//printf("----------------\n");
if (module_fun == mp_const_none) {
printf("compile error\n");
}
}
}
}
void do_file(const char *file) {
mp_lexer_t *lex = mp_lexer_new_from_file(file);
if (lex == NULL) {
return;
}
if (0) {
// just tokenise
while (lex->tok_kind != MP_TOKEN_END) {
mp_lexer_show_token(lex);
mp_lexer_to_next(lex);
}
mp_lexer_free(lex);
} else {
// parse
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return;
}
mp_lexer_free(lex);
if (pn != MP_PARSE_NODE_NULL) {
//printf("----------------\n");
//mp_parse_node_print(pn, 0);
//printf("----------------\n");
// compile
mp_obj_t module_fun = mp_compile(pn, 0, MP_EMIT_OPT_NONE, false);
//printf("----------------\n");
if (mp_obj_is_exception_instance(module_fun)) {
mp_obj_print_exception(module_fun);
}
}
}
}
void do_file(const char *file) {
mp_lexer_t *lex = mp_lexer_new_from_file(file);
if (lex == NULL) {
return;
}
if (0) {
// just tokenise
while (!mp_lexer_is_kind(lex, MP_TOKEN_END)) {
mp_token_show(mp_lexer_cur(lex));
mp_lexer_to_next(lex);
}
mp_lexer_free(lex);
} else {
// parse
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return;
}
mp_lexer_free(lex);
if (pn != MP_PARSE_NODE_NULL) {
//printf("----------------\n");
//mp_parse_node_print(pn, 0);
//printf("----------------\n");
// compile
mp_obj_t module_fun = mp_compile(pn, 0, false);
//printf("----------------\n");
if (module_fun == mp_const_none) {
printf("compile error\n");
}
}
}
}
inline void do_str(const char *src) {
mp_lexer_t *lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, src, strlen(src), 0);
if (lex == NULL) {
tt_abort_msg("Lexer initialization error");
}
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
tt_abort_msg("Parser error");
}
// parse okay
qstr source_name = mp_lexer_source_name(lex);
mp_lexer_free(lex);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, true);
mp_parse_node_free(pn);
if (module_fun == mp_const_none) {
tt_abort_msg("Computer error");
}
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
mp_call_function_0(module_fun);
nlr_pop();
} else {
mp_obj_print_exception((mp_obj_t)nlr.ret_val);
tt_abort_msg("Uncaught exception");
}
end:
;
}
void do_str(const char *src) {
mp_lexer_t *lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, src, strlen(src), 0);
if (lex == NULL) {
return;
}
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_SINGLE_INPUT, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return;
}
// parse okay
qstr source_name = mp_lexer_source_name(lex);
mp_lexer_free(lex);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, true);
if (mp_obj_is_exception_instance(module_fun)) {
// compile error
mp_obj_print_exception(module_fun);
return;
}
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
mp_call_function_0(module_fun);
nlr_pop();
} else {
// uncaught exception
mp_obj_print_exception((mp_obj_t)nlr.ret_val);
}
}
// parses, compiles and executes the code in the lexer
// frees the lexer before returning
// EXEC_FLAG_PRINT_EOF prints 2 EOF chars: 1 after normal output, 1 after exception output
// EXEC_FLAG_ALLOW_DEBUGGING allows debugging info to be printed after executing the code
// EXEC_FLAG_IS_REPL is used for REPL inputs (flag passed on to mp_compile)
STATIC int parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, int exec_flags) {
int ret = 0;
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, input_kind, &parse_error_kind);
qstr source_name = mp_lexer_source_name(lex);
// check for parse error
if (pn == MP_PARSE_NODE_NULL) {
if (exec_flags & EXEC_FLAG_PRINT_EOF) {
stdout_tx_strn("\x04", 1);
}
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
goto finish;
}
mp_lexer_free(lex);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, exec_flags & EXEC_FLAG_IS_REPL);
// check for compile error
if (mp_obj_is_exception_instance(module_fun)) {
if (exec_flags & EXEC_FLAG_PRINT_EOF) {
stdout_tx_strn("\x04", 1);
}
mp_obj_print_exception(module_fun);
goto finish;
}
// execute code
nlr_buf_t nlr;
uint32_t start = HAL_GetTick();
if (nlr_push(&nlr) == 0) {
mp_hal_set_interrupt_char(CHAR_CTRL_C); // allow ctrl-C to interrupt us
mp_call_function_0(module_fun);
mp_hal_set_interrupt_char(-1); // disable interrupt
nlr_pop();
ret = 1;
if (exec_flags & EXEC_FLAG_PRINT_EOF) {
stdout_tx_strn("\x04", 1);
}
} else {
// uncaught exception
// FIXME it could be that an interrupt happens just before we disable it here
mp_hal_set_interrupt_char(-1); // disable interrupt
// print EOF after normal output
if (exec_flags & EXEC_FLAG_PRINT_EOF) {
stdout_tx_strn("\x04", 1);
}
// check for SystemExit
if (mp_obj_is_subclass_fast(mp_obj_get_type((mp_obj_t)nlr.ret_val), &mp_type_SystemExit)) {
// at the moment, the value of SystemExit is unused
ret = PYEXEC_FORCED_EXIT;
} else {
mp_obj_print_exception((mp_obj_t)nlr.ret_val);
ret = 0;
}
}
// display debugging info if wanted
if ((exec_flags & EXEC_FLAG_ALLOW_DEBUGGING) && repl_display_debugging_info) {
mp_uint_t ticks = HAL_GetTick() - start; // TODO implement a function that does this properly
printf("took " UINT_FMT " ms\n", ticks);
gc_collect();
// qstr info
{
mp_uint_t n_pool, n_qstr, n_str_data_bytes, n_total_bytes;
qstr_pool_info(&n_pool, &n_qstr, &n_str_data_bytes, &n_total_bytes);
printf("qstr:\n n_pool=" UINT_FMT "\n n_qstr=" UINT_FMT "\n n_str_data_bytes=" UINT_FMT "\n n_total_bytes=" UINT_FMT "\n", n_pool, n_qstr, n_str_data_bytes, n_total_bytes);
}
// GC info
{
gc_info_t info;
gc_info(&info);
printf("GC:\n");
printf(" " UINT_FMT " total\n", info.total);
printf(" " UINT_FMT " : " UINT_FMT "\n", info.used, info.free);
printf(" 1=" UINT_FMT " 2=" UINT_FMT " m=" UINT_FMT "\n", info.num_1block, info.num_2block, info.max_block);
}
}
finish:
if (exec_flags & EXEC_FLAG_PRINT_EOF) {
stdout_tx_strn("\x04", 1);
}
return ret;
}
mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
// initialise parser and allocate memory for its stacks
parser_t parser;
parser.rule_stack_alloc = MICROPY_ALLOC_PARSE_RULE_INIT;
parser.rule_stack_top = 0;
parser.rule_stack = m_new(rule_stack_t, parser.rule_stack_alloc);
parser.result_stack_alloc = MICROPY_ALLOC_PARSE_RESULT_INIT;
parser.result_stack_top = 0;
parser.result_stack = m_new(mp_parse_node_t, parser.result_stack_alloc);
parser.lexer = lex;
parser.tree.chunk = NULL;
parser.cur_chunk = NULL;
#if MICROPY_COMP_CONST
mp_map_init(&parser.consts, 0);
#endif
// work out the top-level rule to use, and push it on the stack
size_t top_level_rule;
switch (input_kind) {
case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break;
case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break;
default: top_level_rule = RULE_file_input;
}
push_rule(&parser, lex->tok_line, rules[top_level_rule], 0);
// parse!
size_t n, i; // state for the current rule
size_t rule_src_line; // source line for the first token matched by the current rule
bool backtrack = false;
const rule_t *rule = NULL;
for (;;) {
next_rule:
if (parser.rule_stack_top == 0) {
break;
}
pop_rule(&parser, &rule, &i, &rule_src_line);
n = rule->act & RULE_ACT_ARG_MASK;
/*
// debugging
printf("depth=%d ", parser.rule_stack_top);
for (int j = 0; j < parser.rule_stack_top; ++j) {
printf(" ");
}
printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack);
*/
switch (rule->act & RULE_ACT_KIND_MASK) {
case RULE_ACT_OR:
if (i > 0 && !backtrack) {
goto next_rule;
} else {
backtrack = false;
}
for (; i < n; ++i) {
uint16_t kind = rule->arg[i] & RULE_ARG_KIND_MASK;
if (kind == RULE_ARG_TOK) {
if (lex->tok_kind == (rule->arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(&parser, rule);
mp_lexer_to_next(lex);
goto next_rule;
}
} else {
assert(kind == RULE_ARG_RULE);
if (i + 1 < n) {
push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule
}
push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule
goto next_rule;
}
}
backtrack = true;
break;
case RULE_ACT_AND: {
// failed, backtrack if we can, else syntax error
if (backtrack) {
assert(i > 0);
if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) {
// an optional rule that failed, so continue with next arg
push_result_node(&parser, MP_PARSE_NODE_NULL);
backtrack = false;
} else {
// a mandatory rule that failed, so propagate backtrack
if (i > 1) {
// already eaten tokens so can't backtrack
goto syntax_error;
} else {
goto next_rule;
}
}
}
// progress through the rule
for (; i < n; ++i) {
if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
// need to match a token
mp_token_kind_t tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK;
if (lex->tok_kind == tok_kind) {
// matched token
if (tok_kind == MP_TOKEN_NAME) {
push_result_token(&parser, rule);
}
mp_lexer_to_next(lex);
} else {
// failed to match token
if (i > 0) {
// already eaten tokens so can't backtrack
goto syntax_error;
} else {
// this rule failed, so backtrack
backtrack = true;
goto next_rule;
}
}
} else {
push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule
push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule
goto next_rule;
}
}
assert(i == n);
// matched the rule, so now build the corresponding parse_node
#if !MICROPY_ENABLE_DOC_STRING
// this code discards lonely statements, such as doc strings
if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) {
mp_parse_node_t p = peek_result(&parser, 1);
if ((MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p))
|| MP_PARSE_NODE_IS_STRUCT_KIND(p, RULE_const_object)) {
pop_result(&parser); // MP_PARSE_NODE_NULL
pop_result(&parser); // const expression (leaf or RULE_const_object)
// Pushing the "pass" rule here will overwrite any RULE_const_object
// entry that was on the result stack, allowing the GC to reclaim
// the memory from the const object when needed.
push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0);
break;
}
}
#endif
// count number of arguments for the parse node
i = 0;
size_t num_not_nil = 0;
for (size_t x = n; x > 0;) {
--x;
if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK;
if (tok_kind == MP_TOKEN_NAME) {
// only tokens which were names are pushed to stack
i += 1;
num_not_nil += 1;
}
} else {
// rules are always pushed
if (peek_result(&parser, i) != MP_PARSE_NODE_NULL) {
num_not_nil += 1;
}
i += 1;
}
}
if (num_not_nil == 1 && (rule->act & RULE_ACT_ALLOW_IDENT)) {
// this rule has only 1 argument and should not be emitted
mp_parse_node_t pn = MP_PARSE_NODE_NULL;
for (size_t x = 0; x < i; ++x) {
mp_parse_node_t pn2 = pop_result(&parser);
if (pn2 != MP_PARSE_NODE_NULL) {
pn = pn2;
}
}
push_result_node(&parser, pn);
} else {
// this rule must be emitted
if (rule->act & RULE_ACT_ADD_BLANK) {
// and add an extra blank node at the end (used by the compiler to store data)
push_result_node(&parser, MP_PARSE_NODE_NULL);
i += 1;
}
push_result_rule(&parser, rule_src_line, rule, i);
}
break;
}
default: {
assert((rule->act & RULE_ACT_KIND_MASK) == RULE_ACT_LIST);
// n=2 is: item item*
// n=1 is: item (sep item)*
// n=3 is: item (sep item)* [sep]
bool had_trailing_sep;
if (backtrack) {
list_backtrack:
had_trailing_sep = false;
if (n == 2) {
if (i == 1) {
// fail on item, first time round; propagate backtrack
goto next_rule;
} else {
// fail on item, in later rounds; finish with this rule
backtrack = false;
}
} else {
if (i == 1) {
// fail on item, first time round; propagate backtrack
goto next_rule;
} else if ((i & 1) == 1) {
// fail on item, in later rounds; have eaten tokens so can't backtrack
if (n == 3) {
// list allows trailing separator; finish parsing list
had_trailing_sep = true;
backtrack = false;
} else {
// list doesn't allowing trailing separator; fail
goto syntax_error;
}
} else {
// fail on separator; finish parsing list
backtrack = false;
}
}
} else {
for (;;) {
size_t arg = rule->arg[i & 1 & n];
if ((arg & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
if (lex->tok_kind == (arg & RULE_ARG_ARG_MASK)) {
if (i & 1 & n) {
// separators which are tokens are not pushed to result stack
} else {
push_result_token(&parser, rule);
}
mp_lexer_to_next(lex);
// got element of list, so continue parsing list
i += 1;
} else {
// couldn't get element of list
i += 1;
backtrack = true;
goto list_backtrack;
}
} else {
assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE);
push_rule(&parser, rule_src_line, rule, i + 1); // save this list-rule
push_rule_from_arg(&parser, arg); // push child of list-rule
goto next_rule;
}
}
}
assert(i >= 1);
// compute number of elements in list, result in i
i -= 1;
if ((n & 1) && (rule->arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
// don't count separators when they are tokens
i = (i + 1) / 2;
}
if (i == 1) {
// list matched single item
if (had_trailing_sep) {
// if there was a trailing separator, make a list of a single item
push_result_rule(&parser, rule_src_line, rule, i);
} else {
// just leave single item on stack (ie don't wrap in a list)
}
} else {
push_result_rule(&parser, rule_src_line, rule, i);
}
break;
}
}
}
#if MICROPY_COMP_CONST
mp_map_deinit(&parser.consts);
#endif
// truncate final chunk and link into chain of chunks
if (parser.cur_chunk != NULL) {
(void)m_renew_maybe(byte, parser.cur_chunk,
sizeof(mp_parse_chunk_t) + parser.cur_chunk->alloc,
sizeof(mp_parse_chunk_t) + parser.cur_chunk->union_.used,
false);
parser.cur_chunk->alloc = parser.cur_chunk->union_.used;
parser.cur_chunk->union_.next = parser.tree.chunk;
parser.tree.chunk = parser.cur_chunk;
}
if (
lex->tok_kind != MP_TOKEN_END // check we are at the end of the token stream
|| parser.result_stack_top == 0 // check that we got a node (can fail on empty input)
) {
syntax_error:;
mp_obj_t exc;
if (lex->tok_kind == MP_TOKEN_INDENT) {
exc = mp_obj_new_exception_msg(&mp_type_IndentationError,
"unexpected indent");
} else if (lex->tok_kind == MP_TOKEN_DEDENT_MISMATCH) {
exc = mp_obj_new_exception_msg(&mp_type_IndentationError,
"unindent does not match any outer indentation level");
} else {
exc = mp_obj_new_exception_msg(&mp_type_SyntaxError,
"invalid syntax");
}
// add traceback to give info about file name and location
// we don't have a 'block' name, so just pass the NULL qstr to indicate this
mp_obj_exception_add_traceback(exc, lex->source_name, lex->tok_line, MP_QSTR_NULL);
nlr_raise(exc);
}
// get the root parse node that we created
assert(parser.result_stack_top == 1);
parser.tree.root = parser.result_stack[0];
// free the memory that we don't need anymore
m_del(rule_stack_t, parser.rule_stack, parser.rule_stack_alloc);
m_del(mp_parse_node_t, parser.result_stack, parser.result_stack_alloc);
// we also free the lexer on behalf of the caller
mp_lexer_free(lex);
return parser.tree;
}
// returns standard error codes: 0 for success, 1 for all other errors
STATIC int execute_from_lexer(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, bool is_repl) {
if (lex == NULL) {
return 1;
}
if (0) {
// just tokenise
while (!mp_lexer_is_kind(lex, MP_TOKEN_END)) {
mp_token_show(mp_lexer_cur(lex));
mp_lexer_to_next(lex);
}
mp_lexer_free(lex);
return 0;
}
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, input_kind, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return 1;
}
qstr source_name = mp_lexer_source_name(lex);
#if MICROPY_PY___FILE__
if (input_kind == MP_PARSE_FILE_INPUT) {
mp_store_global(MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name));
}
#endif
mp_lexer_free(lex);
/*
printf("----------------\n");
mp_parse_node_print(pn, 0);
printf("----------------\n");
*/
mp_obj_t module_fun = mp_compile(pn, source_name, emit_opt, is_repl);
if (module_fun == mp_const_none) {
// compile error
return 1;
}
if (compile_only) {
return 0;
}
// execute it
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
mp_call_function_0(module_fun);
nlr_pop();
return 0;
} else {
// uncaught exception
// check for SystemExit
mp_obj_t exc = (mp_obj_t)nlr.ret_val;
if (mp_obj_is_subclass_fast(mp_obj_get_type(exc), &mp_type_SystemExit)) {
mp_obj_t exit_val = mp_obj_exception_get_value(exc);
mp_int_t val;
if (!mp_obj_get_int_maybe(exit_val, &val)) {
val = 0;
}
exit(val);
}
mp_obj_print_exception((mp_obj_t)nlr.ret_val);
return 1;
}
}
// parses, compiles and executes the code in the lexer
// frees the lexer before returning
bool parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, bool is_repl) {
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, input_kind, &parse_error_kind);
qstr source_name = mp_lexer_source_name(lex);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return false;
}
mp_lexer_free(lex);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, is_repl);
if (mp_obj_is_exception_instance(module_fun)) {
mp_obj_print_exception(module_fun);
return false;
}
nlr_buf_t nlr;
bool ret;
uint32_t start = HAL_GetTick();
if (nlr_push(&nlr) == 0) {
usb_vcp_set_interrupt_char(VCP_CHAR_CTRL_C); // allow ctrl-C to interrupt us
mp_call_function_0(module_fun);
usb_vcp_set_interrupt_char(VCP_CHAR_NONE); // disable interrupt
nlr_pop();
ret = true;
} else {
// uncaught exception
// FIXME it could be that an interrupt happens just before we disable it here
usb_vcp_set_interrupt_char(VCP_CHAR_NONE); // disable interrupt
mp_obj_print_exception((mp_obj_t)nlr.ret_val);
ret = false;
}
// display debugging info if wanted
if (is_repl && repl_display_debugging_info) {
uint32_t ticks = HAL_GetTick() - start; // TODO implement a function that does this properly
printf("took %lu ms\n", ticks);
gc_collect();
// qstr info
{
mp_uint_t n_pool, n_qstr, n_str_data_bytes, n_total_bytes;
qstr_pool_info(&n_pool, &n_qstr, &n_str_data_bytes, &n_total_bytes);
printf("qstr:\n n_pool=" UINT_FMT "\n n_qstr=" UINT_FMT "\n n_str_data_bytes=" UINT_FMT "\n n_total_bytes=" UINT_FMT "\n", n_pool, n_qstr, n_str_data_bytes, n_total_bytes);
}
// GC info
{
gc_info_t info;
gc_info(&info);
printf("GC:\n");
printf(" " UINT_FMT " total\n", info.total);
printf(" " UINT_FMT " : " UINT_FMT "\n", info.used, info.free);
printf(" 1=" UINT_FMT " 2=" UINT_FMT " m=" UINT_FMT "\n", info.num_1block, info.num_2block, info.max_block);
}
}
return ret;
}