void setup() { s1.line = 0; s1.column = 1; s2.line = 2; s2.column = 3; t1 = token_new( value, length, type, s1, s2 ); t2 = token_new( value, length, type, s1, s2 ); }
Token *run_addop() { char cur = next_char(); switch(cur) { case '+': return token_new(TOK_ADDOP, ADDOP_ADD); case '-': return token_new(TOK_ADDOP, ADDOP_SUBTRACT); default: return token_new(TOK_BLOCKED, 0); } }
Token *run_assignop() { char cur = next_char(); if(cur == ':') { cur = next_char(); if(cur == '=') { return token_new(TOK_ASSIGNOP, 0); } } return token_new(TOK_BLOCKED, 0); }
PyContextToken * PyContextVar_Set(PyContextVar *var, PyObject *val) { if (!PyContextVar_CheckExact(var)) { PyErr_SetString( PyExc_TypeError, "an instance of ContextVar was expected"); return NULL; } PyContext *ctx = context_get(); if (ctx == NULL) { return NULL; } PyObject *old_val = NULL; int found = _PyHamt_Find(ctx->ctx_vars, (PyObject *)var, &old_val); if (found < 0) { return NULL; } Py_XINCREF(old_val); PyContextToken *tok = token_new(ctx, var, old_val); Py_XDECREF(old_val); if (contextvar_set(var, val)) { Py_DECREF(tok); return NULL; } return tok; }
inline static grn_cell * get_phrase(grn_ctx *ctx, grn_query *q) { char *start, *s, *d; start = s = d = q->cur; while (1) { unsigned int len; if (s >= q->str_end) { q->cur = s; break; } len = grn_charlen(ctx, s, q->str_end); if (len == 0) { /* invalid string containing malformed multibyte char */ return NULL; } else if (len == 1) { if (*s == GRN_QUERY_QUOTER) { q->cur = s + 1; break; } else if (*s == GRN_QUERY_ESCAPE && s + 1 < q->str_end) { s++; len = grn_charlen(ctx, s, q->str_end); } } while (len--) { *d++ = *s++; } } return token_new(q, start, d); }
/* parser_get_token */ int parser_get_token(Parser * parser, Token ** token) { int ret = 0; /* XXX not sure */ size_t i; ParserCallbackData * pcd; if((*token = token_new(parser->filename, parser->line, parser->col)) == NULL) return 1; if(parser->last == EOF) parser_scan_filter(parser); #ifdef DEBUG fprintf(stderr, "DEBUG: %s() %c\n", __func__, parser->last); #endif for(i = 0; i < parser->callbacks_cnt; i++) { pcd = &parser->callbacks[i]; if((ret = pcd->callback(parser, *token, parser->last, pcd->data)) <= 0) break; } if(ret == 0 && i != parser->callbacks_cnt) return 0; /* there is a token and no error */ token_delete(*token); *token = NULL; return (ret >= 0 && parser->last == EOF) ? 0 : 1; }
ssize_t tokenize(const char *string, struct s_vec *queue) { size_t i; size_t len; struct s_token *token; assert(string != NULL); assert(queue != NULL); if (vec_init(queue, 0, &free_token_queue) == -1) return (-1); len = strlen(string); i = 0; while (i < len) { if (string[i] == ' ' || string[i] == '\t') { ++i; continue; } if (token_new(&token) != 0 || token_init(token) != 0) return (-1); if (!call_tokenizers(string, queue, token, &i)) i += 1; } return (queue->size); }
inline static grn_cell * get_word(grn_ctx *ctx, grn_query *q, int *prefixp) { char *start = q->cur, *end; unsigned int len; for (end = q->cur;; ) { /* null check and length check */ if (!(len = grn_charlen(ctx, end, q->str_end))) { q->cur = q->str_end; break; } if (grn_isspace(end, q->encoding) || *end == GRN_QUERY_PARENR) { q->cur = end; break; } if (*end == GRN_QUERY_PREFIX) { *prefixp = 1; q->cur = end + 1; break; } end += len; } return token_new(q, start, end); }
Token *run_ws() { char c = next_char(); if(!match_ws(c)) { Token *blocked_token = token_new(TOK_BLOCKED, 0); return blocked_token; } while(match_ws(c)) { c = next_char(); } move_f_back(); Token *my_token = token_new(TOK_WS, 0); return my_token; }
END_TEST START_TEST (test_token_new) { struct s_token *t; ck_assert_int_eq(token_new(&t), 0); }
END_TEST START_TEST (test_token_new_malloc_fail) { struct s_token *t; set_mock_malloc(&my_mock_malloc_null); ck_assert_int_ne(token_new(&t), 0); set_mock_malloc(&malloc); }
// Process any deferred token we have static void process_deferred_ast(parser_t* parser, rule_state_t* state) { assert(parser != NULL); assert(state != NULL); if(state->deferred) { token_t* deferred_token = token_new(state->deferred_id); token_set_pos(deferred_token, parser->source, state->line, state->pos); state->ast = ast_token(deferred_token); state->deferred = false; } }
Token *parseComment( Tokenizer *tokenizer ) { StatefulString *ss = tokenizer->ss_; assert( isCommentStart( ss, 0 ) ); int start, length; StatefulStringPosition pos1, pos2; Token *token; wchar_t *error = malloc( 201 * sizeof( wchar_t ) ); error[ 0 ] = L'\0'; start = ss->next_index; pos1 = ss->next_position; length = 2; TokenType type = COMMENT; ss_getchar( ss ); ss_getchar( ss ); // Throw away `/*` while ( ss_peek( ss ) != WEOF && ( ss_peek( ss ) != L'*' || ss_peekx( ss, 1 ) != L'/' ) ) { length++; if ( ss_getchar( ss ) == L'\\' ) { ss_getchar( ss ); length++; } } if ( ss_peek( ss ) == WEOF ) { swprintf( error, 200, L"Encountered end-of-file while parsing a comment. Probably a forgotten `*/`." ); } else { ss_getchar( ss ); ss_getchar( ss ); // Throw away `*/` length += 2; } // Return the token. pos2 = ss->next_position; token = token_new( ss_substr( ss, start, length ), length, type, pos1, pos2 ); if ( wcscmp( error, L"" ) != 0 ) { tokenizer_error( tokenizer, error, token ); } else { free( error ); } return token; }
Token* addtoken( Tokenizer *tokenizer, const wchar_t *value, const unsigned int length, const TokenType type, const StatefulStringPosition start, const StatefulStringPosition end ) { Token *token = token_new( value, length, type, start, end ); if ( tokenizer->numtokens >= tokenizer->maxtokens_ ) { tokenizer->maxtokens_ *= 2; tokenizer->tokens_ = realloc( tokenizer->tokens_, ( tokenizer->maxtokens_ + 1 ) * sizeof( Token* ) ); if ( tokenizer->tokens_ == NULL ) { allocationerror( tokenizer->maxtokens_, L"Tokenizer::addtoken" ); exit( EXIT_FAILURE ); } } tokenizer->tokens_[ tokenizer->numtokens++ ] = token; return token; }
static Token *advance() { char cur = next_char(); Token *result_token; switch(state) { case 0: if(cur == '<') { state = 1; return advance(); } if(cur == '>') { state = 2; return advance(); } if(cur == '=') { result_token = token_new(TOK_RELOP, RELOP_EQUAL); return result_token; } result_token = token_new(TOK_BLOCKED, 0); return result_token; case 1: if(cur == '>') { result_token = token_new(TOK_RELOP, RELOP_NOT_EQUAL); return result_token; } if(cur == '=') { result_token = token_new(TOK_RELOP, RELOP_LESS_THAN_OR_EQUAL); return result_token; } move_f_back(); result_token = token_new(TOK_RELOP, RELOP_LESS_THAN); return result_token; case 2: if(cur == '=') { result_token = token_new(TOK_RELOP, RELOP_GREATER_THAN_OR_EQUAL); return result_token; } move_f_back(); result_token = token_new(TOK_RELOP, RELOP_GREATER_THAN); return result_token; default: result_token = token_new(TOK_BLOCKED, 0); return result_token; } }
/////////////////////////////////////////////////////////////////////////// // // Parsing Functions // ///////////////////////////////////// // // Whitespace // Token *parseWhitespace( Tokenizer *tokenizer ) { StatefulString *ss = tokenizer->ss_; assert( isWhitespaceStart( ss, 0 ) ); int start, length; StatefulStringPosition pos1, pos2; start = ss->next_index; length = 0; pos1 = ss->next_position; while ( isWhitespace( ss_peek( ss ) ) ) { ss_getchar( ss ); length++; } pos2 = ss->next_position; return token_new( ss_substr( ss, start, length ), length, WHITESPACE, pos1, pos2 ); }
///////////////////////////////////// // // Strings // Token *parseString( Tokenizer *tokenizer ) { StatefulString *ss = tokenizer->ss_; assert( isStringStart( ss, 0 ) ); int start, length; StatefulStringPosition pos1, pos2; wchar_t temp; Token *token; length = 0; pos1 = ss->next_position; wchar_t quote = ss_getchar( ss ); start = ss->next_index; // Exclude the quotes while ( ( temp = ss_getchar( ss ) ) != WEOF && !isNewline( temp ) && temp != quote ) { length++; if ( temp == L'\\' ) { ss_getchar( ss ); length++; } } pos2 = ss->next_position; token = token_new( ss_substr( ss, start, length ), length, STRING, pos1, pos2 ); wchar_t *error = malloc( 201 * sizeof( wchar_t ) ); error[ 0 ] = L'\0'; if ( temp == WEOF ) { swprintf( error, 200, L"Encountered end-of-file while parsing a string. Looks like a `%C` is missing.", quote ); } else if ( isNewline( temp ) ) { error = L"Encountered a newline while parsing a string. Strings can only contain newlines if they're properly escaped (e.g. `\\[\\n]`)"; } if ( wcscmp( error, L"" ) != 0 ) { tokenizer_error( tokenizer, error, token ); } else { free( error ); } return token; }
Token *parseNumber( Tokenizer *tokenizer ) { StatefulString *ss = tokenizer->ss_; assert( isNumberStart( ss, 0 ) ); int start, length; StatefulStringPosition pos1, pos2; start = ss->next_index; length = 0; pos1 = ss->next_position; TokenType type = NUMBER; int isFloat = 0; wchar_t temp; while ( isNumeric( ss_peek( ss ) ) || ( ss_peek( ss ) == L'-' && length == 0 ) || ( ss_peek( ss ) == L'.' && !isFloat ) ) { temp = ss_getchar( ss ); if ( temp == L'.' ) { isFloat = 1; } length++; } // Is the number followed by a percentage? if ( ss_peek( ss ) == L'%' ) { ss_getchar( ss ); length++; type = PERCENTAGE; } // Is the number followed by a dimension? else if ( isIdentifierStart( ss, 0 ) ) { while ( isNameChar( ss_peek( ss ) ) ) { ss_getchar( ss ); length++; } type = DIMENSION; } pos2 = ss->next_position; return token_new( ss_substr( ss, start, length ), length, type, pos1, pos2 ); }
int assembler(struct asm_context_t *ctx, char *infile) { struct token_t *token; int error; token = token_new(&ctx->tokens); token_prepare(token, infile); while (1) { token_drop(token); if (lang_eof(token) == 0) break; if (lang_comment(token) == 0) continue; if (lang_label(ctx, token) == 0) continue; if (ctx->pass) { if (lang_directive(ctx, token) == 0) continue; if (lang_instruction(ctx, token) == 0) continue; } else { if (token_get(token, TOKEN_TYPE_LINE, TOKEN_CURRENT)) continue; } debug_emsg("Unknown program construction"); goto error; } error = 0; goto noerror; error: error = -1; token_print_rollback(token); debug_emsgf("Error in file", "%s" NL, infile); noerror: token_remove(&ctx->tokens, token); return error; }
Token *parseName( Tokenizer *tokenizer ) { StatefulString *ss = tokenizer->ss_; assert( isNameChar( ss_peek( ss ) ) || isUnicodeSequenceStart( ss, 0 ) ); int start, length; StatefulStringPosition pos1, pos2; start = ss->next_index; length = 0; pos1 = ss->next_position; while ( isNameChar( ss_peek( ss ) ) || isUnicodeSequenceStart( ss, 0 ) ) { length += processChar( ss ); } if ( ss_peek( ss ) == L'(' ) { ss_getchar( ss ); length++; } pos2 = ss->next_position; return token_new( ss_substr( ss, start, length ), length, NAME, pos1, pos2 ); }
Token *parseSGMLComment( Tokenizer *tokenizer ) { StatefulString *ss = tokenizer->ss_; assert( isCDOStart( ss, 0 ) || isCDCStart( ss, 0 ) ); int start, length; StatefulStringPosition pos1, pos2; start = ss->next_index; pos1 = ss->next_position; length = 4; TokenType type = SGML_COMMENT_OPEN; if ( isCDCStart( ss, 0 ) ) { type = SGML_COMMENT_CLOSE; length = 3; } for ( int i = 0; i<length; i++ ) { ss_getchar( ss ); } pos2 = ss->next_position; return token_new( ss_substr( ss, start, length ), length, type, pos1, pos2 ); }
static int read_conllx(FILE* fp) { int num_sent = 0; unsigned int line_num = 0; char buff[BUF_SIZE]; const char* sep = "\t"; char *t, *line, *tofree; size_t len; const char** seq = xmalloc(sizeof(char *) * CONLLX_TOKEN_NUM_FIELDS); struct blocks* blocks = new_blocks(256); struct sentence* sent = sentence_new(); while (fgets(buff, sizeof(buff), fp)) { line_num++; if (buff[0] == '\n') { latex_print_dep_tree(sent); sentence_destroy(sent); sent = sentence_new(); num_sent++; continue; } len = strnlen(buff, BUF_SIZE); tofree = line = strndup(buff, len-1); blocks_add_elem(blocks, tofree); int i = 0; while ((t = strsep(&line, sep)) != NULL) { seq[i] = t; i++; } sentence_add_token(sent, token_new(seq, CONLLX_TOKEN_NUM_FIELDS)); } destroy_blocks(blocks); sentence_destroy(sent); free(seq); fprintf(stderr, "INFO: Number of sentences = %d\n", num_sent); return 0; }
///////////////////////////////////// // // Operators, et al. // Token *parseEverythingElse( Tokenizer *tokenizer ) { StatefulString *ss = tokenizer->ss_; int start, length; StatefulStringPosition pos1, pos2; start = ss->next_index; pos1 = ss->next_position; TokenType type = DELIM; length = 1; wchar_t temp = ss_getchar( ss ); if ( ( temp== L'~' || temp == L'|' || temp == L'*' || temp == L'^' || temp == L'$' ) && ss_peek( ss ) == L'=' ) { ss_getchar( ss ); length++; } if ( length == 1 ) { switch ( temp ) { case L'{': type = CURLY_BRACE_OPEN; break; case L'}': type = CURLY_BRACE_CLOSE; break; case L'[': type = SQUARE_BRACE_OPEN; break; case L']': type = SQUARE_BRACE_CLOSE; break; case L'(': type = PAREN_OPEN; break; case L')': type = PAREN_CLOSE; break; case L':': type = COLON; break; case L';': type = SEMICOLON; break; case L'@': type = AT; break; case L'#': type = HASH; break; case L'%': type = PERCENT; break; case L'.': type = DOT; break; } } else if ( length == 2 ) { switch ( temp ) { case L'~': type = INCLUDES; break; case L'|': type = DASHMATCH; break; case L'^': type = PREFIXMATCH; break; case L'$': type = SUFFIXMATCH; break; case L'*': type = SUBSTRINGMATCH; break; } } pos2 = ss->next_position; return token_new( ss_substr( ss, start, length ), length, type, pos1, pos2 ); }
struct token token_stream_read(struct token_stream * ts) { if(ts->end) return token_new(TOKEN_TYPE_END); enum states{ READ_NEXT = 0, SCAN, SCAN_ERR, BUILD_SECTION, BUILD_SECTION1, BUILD_SECTION_END, BUILD_NAME, BUILD_NAME1, BUILD_NAME_END, BUILD_ASSIGNMENT, BUILD_STRING, BUILD_STRING1, BUILD_STRING_END, BUILD_COMMENT, BUILD_NEWLINE, BUILD_END, ERROR, SKIP_LINE, } state = ts->next_state; struct varstr * buf = varstr_new(); struct char_stream * cs = ts->cs; struct token tok; int running = 1; do switch(state){ case READ_NEXT: next_char(cs); state = SCAN; case SCAN: if(!cs->escaped) { if(cs->c == ' ' || cs->c == '\t') state = READ_NEXT; else if(cs->c == '[') state = BUILD_SECTION; else if(char_is_name(cs)) state = BUILD_NAME; else if(cs->c == '=') state = BUILD_ASSIGNMENT; else if(cs->c == '"') state = BUILD_STRING; else if(cs->c == '#') state = BUILD_COMMENT; else if(cs->c == '\n') state = BUILD_NEWLINE; else if(cs->c == EOF) state = BUILD_END; else state = SCAN_ERR; } else state = SCAN_ERR; break; case SCAN_ERR: fprintf(stderr,"next_token: " "bad token at row %d, column %d\n", cs->row, cs->col); state = SKIP_LINE; break; case BUILD_SECTION: // [section] varstr_clear(buf); tok = token_new(TOKEN_TYPE_SECTION); state = BUILD_SECTION1; case BUILD_SECTION1: next_char(cs); if(char_is_name(cs)) varstr_append(buf, cs->c); else state = BUILD_SECTION_END; break; case BUILD_SECTION_END: if(cs->c == ']' && !cs->escaped) { ts->next_state = READ_NEXT; tok.text = cstr_dup(varstr_view(buf)); running = 0; } else { fprintf(stderr, "next_token: " "bad section name at row %d, column %d, " "expected ']'\n", cs->row, cs->col); state = ERROR; } break; case BUILD_NAME: // name tok = token_new(TOKEN_TYPE_NAME); varstr_clear(buf); varstr_append(buf, cs->c); state = BUILD_NAME1; case BUILD_NAME1: next_char(cs); if(char_is_name(cs)) varstr_append(buf, cs->c); else state = BUILD_NAME_END; break; case BUILD_NAME_END: ts->next_state = SCAN; tok.text = cstr_dup(varstr_view(buf)); running = 0; break; case BUILD_ASSIGNMENT: // = tok = token_new(TOKEN_TYPE_ASSIGNMENT); ts->next_state = READ_NEXT; running = 0; break; case BUILD_STRING: /* "multi line string with escape /"/n" */ tok = token_new(TOKEN_TYPE_STRING); varstr_clear(buf); state = BUILD_STRING1; case BUILD_STRING1: next_char(cs); if(cs->c == EOF) state = BUILD_STRING_END; else if(cs->c == '"' && !cs->escaped) state = BUILD_STRING_END; else varstr_append(buf, cs->c); break; case BUILD_STRING_END: if(cs->c == '"' && !cs->escaped) { ts->next_state = READ_NEXT; tok.text = cstr_dup(varstr_view(buf)); running = 0; } else if(cs->c == EOF) { fprintf(stderr, "next_token" "bad string at row %d, column %d, " "expected '\"'\n", cs->row, cs->col); state = ERROR; } break; case BUILD_COMMENT: // # comment state = SKIP_LINE; break; case BUILD_NEWLINE: tok = token_new(TOKEN_TYPE_NEWLINE); ts->next_state = READ_NEXT; running = 0; break; case BUILD_END: tok = token_new(TOKEN_TYPE_NEWLINE); ts->end = 1; running = 0; break; case ERROR: state = SKIP_LINE; case SKIP_LINE: if(cs->c == '\n' && !cs->escaped) state = BUILD_NEWLINE; else if(cs->c == EOF) state = BUILD_END; else next_char(cs); break; }while(running); varstr_del(buf); return tok; }
token_t *token_new_no_data(token_id_t id) { return token_new(id, NULL, NULL, NULL, NULL); }
static token_t *error_new(error_info_t *info) { return token_new(TOKEN_ERROR, info, NULL, error_info_free_func, error_info_to_string_func); }
// Make a token with the specified ID and no token text static token_t* make_token(lexer_t* lexer, token_id id) { token_t* t = token_new(id); token_set_pos(t, lexer->source, lexer->token_line, lexer->token_pos); return t; }
///////////////////////////////////// // // Url // Token *parseUrl( Tokenizer *tokenizer ) { StatefulString *ss = tokenizer->ss_; assert( isUrlStart( ss, 0 ) ); int start, length; StatefulStringPosition pos1, pos2; Token *token; wchar_t *error = malloc( 201 * sizeof( wchar_t ) ); error[ 0 ] = L'\0'; start = ss->next_index; pos1 = ss->next_position; TokenType type = URL; // Throw away `url(` for ( int i = 0; i<4; i++ ) { ss_getchar( ss ); } length = 4; // Throw away leading whitespace while ( isWhitespaceStart( ss, 0 ) ) { ss_getchar( ss ); length++; } // Process the url if ( isStringStart( ss, 0 ) ) { token = parseString( tokenizer ); length += token->length + 2; // +2 for the quotes if ( token->error != NULL ) { length -= 1; // Error'd strings don't have a trailing quote pos2 = ss->next_position; Token *t = token_new( ss_substr( ss, start, length ), length, type, pos1, pos2 ); t->error = token->error; ( token->error )->token = t; free( token ); return t; } else { free( token ); } } else if ( isUrlChar( ss_peek( ss ) ) || isUnicodeSequenceStart( ss, 0 ) ) { while ( isUrlChar( ss_peek( ss ) ) || isUnicodeSequenceStart( ss, 0 ) ) { length += processChar( ss ); } } else { // ERROR: if ( ss_peek( ss ) == WEOF ) { swprintf( error, 200, L"Encountered end-of-file while parsing a URL." ); } else { swprintf( error, 200, L"Encountered an invalid character (`%C`) while parsing a URL.", ss_peek( ss ) ); } } // Throw away trailing whitespace while ( isWhitespaceStart( ss, 0 ) ) { ss_getchar( ss ); length++; } // Grab the trailing `)` if ( ss_peek( ss ) == L')' ) { ss_getchar( ss ); length++; } else { // ERROR: if ( ss_peek( ss ) == WEOF ) { swprintf( error, 200, L"Encountered end-of-file while parsing a URL. Expected a `)`." ); } else { swprintf( error, 200, L"Encountered an invalid character (`%C`) while parsing a URL. Expected a `)`. Perhaps a typo?", ss_peek( ss ) ); } } // Return the token. pos2 = ss->next_position; token = token_new( ss_substr( ss, start, length ), length, type, pos1, pos2 ); if ( wcscmp( error, L"" ) != 0 ) { tokenizer_error( tokenizer, error, token ); } else { free( error ); } return token; }
// returns a token on the heap, or a null pointer Token *scan_string(char **buf) { char *starting_buf = *buf; static int in_multiline_comment = false; char c = read_character(buf); if (in_multiline_comment) { if (c == '*') { if (read_character(buf) == '/') { in_multiline_comment = false; return token_new(COMMENT, "/* ... */", 9); } unread_character(buf); } if (c != '\0') { return scan_string(buf); } } switch(c) { case ':': if (read_character(buf) == '=') { return token_new(ASSIGN, ":=", 2); } unread_character(buf); case '+': return token_new(PLUS, "+", 1); case '-': return token_new(MINUS, "-", 1); case '*': return token_new(TIMES, "*", 1); case '/': c = read_character(buf); if (c == '/') { int len = 2; while ((c = read_character(buf)) && c != '\n' && c != EOF) { len++; } unread_character(buf); return token_new(COMMENT, *buf - len, len); } else if (c == '*' && !in_multiline_comment) { in_multiline_comment = true; return scan_string(buf); } unread_character(buf); return token_new(DIV, "/", 1); case '(': return token_new(LPAREN, "(", 1); case ')': return token_new(RPAREN, ")", 1); case ' ': case '\t': case '\n': return scan_string(buf); case '\0': case EOF: return NULL; } // try and parse an identifer or keyword if (isalpha(c)) { // check if keyword char *keywords[] = {"read", "write"}; int keywords_len = 2; for (int i = 0; i < keywords_len; i++) { if (strncmp(*buf - 1, keywords[i], strlen(keywords[i])) == 0) { *buf += strlen(keywords[i]) - 1; return token_new(KEYWORD, keywords[i], strlen(keywords[i])); } } // parse as identifer int len = 1; while ((c = read_character(buf)) && isalnum(c)) { len++; } // unread the character that broke the while loop unread_character(buf); return token_new(IDENTIFIER, *buf - len, len); } if (isdigit(c) || c == '.') { // parse as digit int len = 1; int dot_found = (c == '.'); while ((c = read_character(buf))) { if (c == '.') { if (dot_found) { break; } dot_found = true; } else if (!isdigit(c)) { break; } len++; } // unread the character that broke the while loop unread_character(buf); return token_new(NUMBER, *buf - len, len); } char debug_str[2] = {c, '\0'}; return token_new(INVALID, debug_str, 1); }
ast_t* ast_blank(token_id id) { return ast_token(token_new(id)); }