static bool cf_lexer_nexttoken(struct cf_lexer *lex, struct cf_token *out_token) { struct base_token token, start_token; bool wrote_data = false; base_token_clear(&token); base_token_clear(&start_token); cf_token_clear(out_token); while (lexer_getbasetoken(&lex->base_lexer, &token, false)) { /* reclassify underscore as alpha for alnum tokens */ if (*token.text.array == '_') token.type = BASETOKEN_ALPHA; /* ignore escaped newlines to merge spliced lines */ if (cf_is_splice(token.text.array)) { lex->base_lexer.offset += newline_size(token.text.array+1); continue; } if (!wrote_data) { out_token->unmerged_str.array = token.text.array; out_token->str.array = lex->write_offset; /* if comment then output a space */ if (cf_lexer_process_comment(lex, out_token)) return true; /* process string tokens if any */ if (cf_lexer_process_string(lex, out_token)) return true; base_token_copy(&start_token, &token); wrote_data = true; } else if (cf_is_token_break(&start_token, &token)) { lex->base_lexer.offset -= token.text.len; break; } /* write token to CF lexer to account for splicing/comments */ cf_lexer_write_strref(lex, &token.text); out_token->str.len += token.text.len; } if (wrote_data) { out_token->unmerged_str.len = (size_t)(lex->base_lexer.offset - out_token->unmerged_str.array); out_token->type = cf_get_token_type(out_token, &start_token); } return wrote_data; }
bool cf_lexer_lex(struct cf_lexer *lex, const char *str, const char *file) { struct cf_token token; struct cf_token *last_token = NULL; cf_lexer_free(lex); if (!str || !*str) return false; if (file) lex->file = bstrdup(file); lexer_start(&lex->base_lexer, str); cf_token_clear(&token); lex->reformatted = bmalloc(strlen(str) + 1); lex->reformatted[0] = 0; lex->write_offset = lex->reformatted; while (cf_lexer_nexttoken(lex, &token)) { if (last_token && is_space_or_tab(*last_token->str.array) && is_space_or_tab(*token.str.array)) { cf_token_add(last_token, &token); continue; } token.lex = lex; last_token = da_push_back_new(lex->tokens); memcpy(last_token, &token, sizeof(struct cf_token)); } cf_token_clear(&token); token.str.array = lex->write_offset; token.unmerged_str.array = lex->base_lexer.offset; token.lex = lex; da_push_back(lex->tokens, &token); return !lex->unexpected_eof; }
static inline int ep_parse_pass_command_call(struct effect_parser *ep, struct darray *call) { struct cf_token end_token; cf_token_clear(&end_token); while (!cf_token_is(&ep->cfp, ";")) { if (cf_token_is(&ep->cfp, "}")) { cf_adderror_expecting(&ep->cfp, ";"); return PARSE_CONTINUE; } darray_push_back(sizeof(struct cf_token), call, ep->cfp.cur_token); if (!cf_next_valid_token(&ep->cfp)) return PARSE_EOF; } darray_push_back(sizeof(struct cf_token), call, ep->cfp.cur_token); darray_push_back(sizeof(struct cf_token), call, &end_token); return PARSE_SUCCESS; }
static bool sp_parse_func_params(struct shader_parser *sp, struct shader_func *func) { struct cf_token peek; int code; cf_token_clear(&peek); if (!cf_peek_valid_token(&sp->cfp, &peek)) return false; if (*peek.str.array == ')') { cf_next_token(&sp->cfp); goto exit; } do { struct shader_var var; shader_var_init(&var); if (!cf_token_is(&sp->cfp, "(") && !cf_token_is(&sp->cfp, ",")) cf_adderror_syntax_error(&sp->cfp); code = sp_parse_func_param(sp, &var); if (code != PARSE_SUCCESS) { shader_var_free(&var); if (code == PARSE_CONTINUE) goto exit; else if (code == PARSE_EOF) return false; } da_push_back(func->params, &var); } while (!cf_token_is(&sp->cfp, ")")); exit: return true; }
static inline void append_end_token(struct darray *tokens) { struct cf_token end; cf_token_clear(&end); darray_push_back(sizeof(struct cf_token), tokens, &end); }
static inline void macro_param_free(struct macro_param *param) { cf_token_clear(¶m->name); da_free(param->tokens); }