static bool cf_lexer_nexttoken(struct cf_lexer *lex, struct cf_token *out_token)
{
struct base_token token, start_token;
bool wrote_data = false;
base_token_clear(&token);
base_token_clear(&start_token);
cf_token_clear(out_token);
while (lexer_getbasetoken(&lex->base_lexer, &token, false)) {
/* reclassify underscore as alpha for alnum tokens */
if (*token.text.array == '_')
token.type = BASETOKEN_ALPHA;
/* ignore escaped newlines to merge spliced lines */
if (cf_is_splice(token.text.array)) {
lex->base_lexer.offset +=
newline_size(token.text.array+1);
continue;
}
if (!wrote_data) {
out_token->unmerged_str.array = token.text.array;
out_token->str.array = lex->write_offset;
/* if comment then output a space */
if (cf_lexer_process_comment(lex, out_token))
return true;
/* process string tokens if any */
if (cf_lexer_process_string(lex, out_token))
return true;
base_token_copy(&start_token, &token);
wrote_data = true;
} else if (cf_is_token_break(&start_token, &token)) {
lex->base_lexer.offset -= token.text.len;
break;
}
/* write token to CF lexer to account for splicing/comments */
cf_lexer_write_strref(lex, &token.text);
out_token->str.len += token.text.len;
}
if (wrote_data) {
out_token->unmerged_str.len = (size_t)(lex->base_lexer.offset -
out_token->unmerged_str.array);
out_token->type = cf_get_token_type(out_token, &start_token);
}
return wrote_data;
}
bool cf_lexer_lex(struct cf_lexer *lex, const char *str, const char *file)
{
struct cf_token token;
struct cf_token *last_token = NULL;
cf_lexer_free(lex);
if (!str || !*str)
return false;
if (file)
lex->file = bstrdup(file);
lexer_start(&lex->base_lexer, str);
cf_token_clear(&token);
lex->reformatted = bmalloc(strlen(str) + 1);
lex->reformatted[0] = 0;
lex->write_offset = lex->reformatted;
while (cf_lexer_nexttoken(lex, &token)) {
if (last_token &&
is_space_or_tab(*last_token->str.array) &&
is_space_or_tab(*token.str.array)) {
cf_token_add(last_token, &token);
continue;
}
token.lex = lex;
last_token = da_push_back_new(lex->tokens);
memcpy(last_token, &token, sizeof(struct cf_token));
}
cf_token_clear(&token);
token.str.array = lex->write_offset;
token.unmerged_str.array = lex->base_lexer.offset;
token.lex = lex;
da_push_back(lex->tokens, &token);
return !lex->unexpected_eof;
}
static inline int ep_parse_pass_command_call(struct effect_parser *ep,
struct darray *call)
{
struct cf_token end_token;
cf_token_clear(&end_token);
while (!cf_token_is(&ep->cfp, ";")) {
if (cf_token_is(&ep->cfp, "}")) {
cf_adderror_expecting(&ep->cfp, ";");
return PARSE_CONTINUE;
}
darray_push_back(sizeof(struct cf_token), call,
ep->cfp.cur_token);
if (!cf_next_valid_token(&ep->cfp)) return PARSE_EOF;
}
darray_push_back(sizeof(struct cf_token), call, ep->cfp.cur_token);
darray_push_back(sizeof(struct cf_token), call, &end_token);
return PARSE_SUCCESS;
}
static bool sp_parse_func_params(struct shader_parser *sp,
struct shader_func *func)
{
struct cf_token peek;
int code;
cf_token_clear(&peek);
if (!cf_peek_valid_token(&sp->cfp, &peek))
return false;
if (*peek.str.array == ')') {
cf_next_token(&sp->cfp);
goto exit;
}
do {
struct shader_var var;
shader_var_init(&var);
if (!cf_token_is(&sp->cfp, "(") && !cf_token_is(&sp->cfp, ","))
cf_adderror_syntax_error(&sp->cfp);
code = sp_parse_func_param(sp, &var);
if (code != PARSE_SUCCESS) {
shader_var_free(&var);
if (code == PARSE_CONTINUE)
goto exit;
else if (code == PARSE_EOF)
return false;
}
da_push_back(func->params, &var);
} while (!cf_token_is(&sp->cfp, ")"));
exit:
return true;
}
static inline void append_end_token(struct darray *tokens)
{
struct cf_token end;
cf_token_clear(&end);
darray_push_back(sizeof(struct cf_token), tokens, &end);
}
static inline void macro_param_free(struct macro_param *param)
{
cf_token_clear(¶m->name);
da_free(param->tokens);
}