mp_lexer_t *mp_lexer_new(qstr src_name, void *stream_data, mp_lexer_stream_next_byte_t stream_next_byte, mp_lexer_stream_close_t stream_close) { mp_lexer_t *lex = m_new_obj_maybe(mp_lexer_t); // check for memory allocation error if (lex == NULL) { if (stream_close) { stream_close(stream_data); } return NULL; } lex->source_name = src_name; lex->stream_data = stream_data; lex->stream_next_byte = stream_next_byte; lex->stream_close = stream_close; lex->line = 1; lex->column = 1; lex->emit_dent = 0; lex->nested_bracket_level = 0; lex->alloc_indent_level = MICROPY_ALLOC_LEXER_INDENT_INIT; lex->num_indent_level = 1; lex->indent_level = m_new_maybe(uint16_t, lex->alloc_indent_level); vstr_init(&lex->vstr, 32); // check for memory allocation error if (lex->indent_level == NULL || vstr_had_error(&lex->vstr)) { mp_lexer_free(lex); return NULL; } // store sentinel for first indentation level lex->indent_level[0] = 0; // preload characters lex->chr0 = stream_next_byte(stream_data); lex->chr1 = stream_next_byte(stream_data); lex->chr2 = stream_next_byte(stream_data); // if input stream is 0, 1 or 2 characters long and doesn't end in a newline, then insert a newline at the end if (lex->chr0 == MP_LEXER_EOF) { lex->chr0 = '\n'; } else if (lex->chr1 == MP_LEXER_EOF) { if (lex->chr0 == '\r') { lex->chr0 = '\n'; } else if (lex->chr0 != '\n') { lex->chr1 = '\n'; } } else if (lex->chr2 == MP_LEXER_EOF) { if (lex->chr1 == '\r') { lex->chr1 = '\n'; } else if (lex->chr1 != '\n') { lex->chr2 = '\n'; } } // preload first token mp_lexer_next_token_into(lex, true); return lex; }
mp_lexer_t *mp_lexer_new(const char *src_name, void *stream_data, mp_lexer_stream_next_char_t stream_next_char, mp_lexer_stream_close_t stream_close) { mp_lexer_t *lex = m_new(mp_lexer_t, 1); lex->name = src_name; // TODO do we need to strdup this? lex->stream_data = stream_data; lex->stream_next_char = stream_next_char; lex->stream_close = stream_close; lex->line = 1; lex->column = 1; lex->emit_dent = 0; lex->nested_bracket_level = 0; lex->alloc_indent_level = 16; lex->num_indent_level = 1; lex->indent_level = m_new(uint16_t, lex->alloc_indent_level); lex->indent_level[0] = 0; vstr_init(&lex->vstr); // preload characters lex->chr0 = stream_next_char(stream_data); lex->chr1 = stream_next_char(stream_data); lex->chr2 = stream_next_char(stream_data); // if input stream is 0, 1 or 2 characters long and doesn't end in a newline, then insert a newline at the end if (lex->chr0 == MP_LEXER_CHAR_EOF) { lex->chr0 = '\n'; } else if (lex->chr1 == MP_LEXER_CHAR_EOF) { if (lex->chr0 != '\n' && lex->chr0 != '\r') { lex->chr1 = '\n'; } } else if (lex->chr2 == MP_LEXER_CHAR_EOF) { if (lex->chr1 != '\n' && lex->chr1 != '\r') { lex->chr2 = '\n'; } } // preload first token mp_lexer_next_token_into(lex, &lex->tok_cur, true); return lex; }
void mp_lexer_to_next(mp_lexer_t *lex) { mp_lexer_next_token_into(lex, &lex->tok_cur, false); }
void mp_lexer_to_next(mp_lexer_t *lex) { mp_lexer_next_token_into(lex, false); }