static int dag_parse_variable(struct lexer *bk, struct dag_node *n)
{
struct token *t = lexer_next_token(bk);
char mode = t->lexeme[0]; //=, or + (assign or append)
lexer_free_token(t);
t = lexer_next_token(bk);
if(t->type != TOKEN_LITERAL)
{
lexer_report_error(bk, "Literal variable name expected.");
}
char *name = xxstrdup(t->lexeme);
lexer_free_token(t);
t = lexer_next_token(bk);
if(t->type != TOKEN_LITERAL)
{
lexer_report_error(bk, "Expected LITERAL token, got: %s\n", lexer_print_token(t));
}
char *value = xxstrdup(t->lexeme);
lexer_free_token(t);
struct hash_table *current_table;
int nodeid;
if(n)
{
current_table = n->variables;
nodeid = n->nodeid;
}
else
{
current_table = bk->d->variables;
nodeid = bk->d->nodeid_counter;
}
int result = 1;
switch(mode)
{
case '=':
dag_variable_add_value(name, current_table, nodeid, value);
debug(D_MAKEFLOW_PARSER, "%s appending to variable name=%s, value=%s", (n ? "node" : "dag"), name, value);
break;
case '+':
dag_parse_append_variable(bk, nodeid, n, name, value);
debug(D_MAKEFLOW_PARSER, "%s variable name=%s, value=%s", (n ? "node" : "dag"), name, value);
break;
default:
lexer_report_error(bk, "Unknown variable operator.");
result = 0;
}
dag_parse_process_special_variable(bk, n, nodeid, name, value);
free(name);
free(value);
return result;
}
int lexer_read_command(struct lexer *lx)
{
struct list *tokens = lexer_read_command_aux(lx);
struct token *t;
if(list_size(tokens) < 2) {
/* If the only token in the list is a NEWLINE, then this is an empty line. */
while((t = list_pop_head(tokens)))
lexer_free_token(t);
list_delete(tokens);
return 1;
}
/* Add command start marker.*/
lexer_push_token(lx, lexer_pack_token(lx, TOKEN_COMMAND));
/* Merge command tokens into main queue. */
/* First merge command modifiers, if any. */
list_first_item(tokens);
while((t = list_peek_head(tokens))) {
if(t->type == TOKEN_LITERAL &&
((strcmp(t->lexeme, "LOCAL") == 0) ||
(strcmp(t->lexeme, "MAKEFLOW") == 0) )) {
t = list_pop_head(tokens);
lexer_push_token(lx, t);
} else if(t->type == TOKEN_SPACE) {
//Discard spaces between modifiers.
t = list_pop_head(tokens);
lexer_free_token(t);
} else {
break;
}
}
/* Mark end of modifiers. */
lexer_push_token(lx, lexer_pack_token(lx, TOKEN_COMMAND_MOD_END));
/* Now merge tha actual command tokens */
/* Gives the number of actual command tokens, not taking into account command modifiers. */
int count = 0;
while((t = list_pop_head(tokens))) {
count++;
lexer_push_token(lx, t);
}
list_delete(tokens);
if(count < 1)
lexer_report_error(lx, "Command is empty.\n");
return count;
}
static int dag_parse_node_nested_makeflow(struct lexer *bk, struct dag_node *n)
{
struct token *t, *start;
dag_parse_drop_spaces(bk);
//Get the dag's file name.
t = lexer_next_token(bk);
if(t->type == TOKEN_LITERAL) {
n->makeflow_dag = xxstrdup(t->lexeme);
start = t;
} else {
lexer_report_error(bk, "At least the name of the Makeflow file should be specified in a recursive call.\n");
return 0; // not reached, silences warning
}
dag_parse_drop_spaces(bk);
//Get dag's working directory.
t = lexer_peek_next_token(bk);
if(t->type == TOKEN_LITERAL) {
t = lexer_next_token(bk);
n->makeflow_cwd = xxstrdup(t->lexeme);
lexer_free_token(t);
} else {
n->makeflow_cwd = xxstrdup(".");
}
dag_parse_drop_spaces(bk);
//Get wrapper's name
char *wrapper = NULL;
t = lexer_peek_next_token(bk);
if(t->type == TOKEN_LITERAL) {
wrapper = xxstrdup(t->lexeme);
lexer_free_token(t);
} else {
wrapper = xxstrdup("");
}
free(start->lexeme);
start->lexeme = string_format("cd %s && %s %s %s",
n->makeflow_cwd,
wrapper,
"makeflow",
n->makeflow_dag);
free(wrapper);
dag_parse_drop_spaces(bk);
lexer_preppend_token(bk, start);
return dag_parse_node_regular_command(bk, n);
}
static int dag_parse_node_command(struct lexer *bk, struct dag_node *n)
{
struct token *t;
//Jump COMMAND token.
t = lexer_next_token(bk);
lexer_free_token(t);
char *local = dag_variable_lookup_string("BATCH_LOCAL", bk->environment);
if(local) {
if(string_istrue(local))
n->local_job = 1;
free(local);
}
/* Read command modifiers. */
while((t = lexer_peek_next_token(bk)) && t->type != TOKEN_COMMAND_MOD_END)
{
t = lexer_next_token(bk);
if(strcmp(t->lexeme, "LOCAL") == 0)
{
n->local_job = 1;
}
else if(strcmp(t->lexeme, "MAKEFLOW") == 0)
{
n->nested_job = 1;
}
else
{
lexer_report_error(bk, "Parser does not know about modifier: %s.\n", t->lexeme);
}
lexer_free_token(t);
}
if(!t)
{
lexer_report_error(bk, "Malformed command.");
}
//Free COMMAND_MOD_END token.
t = lexer_next_token(bk);
lexer_free_token(t);
if(n->nested_job)
{
return dag_parse_node_nested_makeflow(bk, n);
}
else
{
return dag_parse_node_regular_command(bk, n);
}
}
static int dag_parse_export(struct lexer *bk)
{
struct token *t, *vtoken, *vname;
const char *name;
int count = 0;
while((t = lexer_peek_next_token(bk)) && t->type != TOKEN_NEWLINE)
{
switch(t->type)
{
case TOKEN_VARIABLE:
vtoken = lexer_next_token(bk); //Save VARIABLE token.
vname = lexer_peek_next_token(bk);
if(vname->type == TOKEN_LITERAL) {
name = xxstrdup(vname->lexeme);
} else {
lexer_report_error(bk, "Variable definition has name missing.\n");
}
lexer_preppend_token(bk, vtoken); //Restore VARIABLE token.
dag_parse_variable(bk, NULL);
break;
case TOKEN_LITERAL:
t = lexer_next_token(bk);
name = xxstrdup(t->lexeme);
lexer_free_token(t);
break;
default:
lexer_report_error(bk, "Malformed export syntax.\n");
break;
}
set_insert(bk->d->export_vars, name);
count++;
debug(D_MAKEFLOW_PARSER, "export variable: %s", name);
}
if(t) {
//Free newline
t = lexer_next_token(bk);
lexer_free_token(t);
}
if(count < 1) {
lexer_report_error(bk, "The export syntax needs the explicit name of the variables to be exported.\n");
}
return 1;
}
struct list *lexer_read_command_aux(struct lexer *lx)
{
int spaces_deleted = lexer_discard_white_space(lx);
struct list *tokens = list_create();
//Preserve space in substitutions.
if(spaces_deleted && lx->depth > 0) {
list_push_tail(tokens, lexer_pack_token(lx, TOKEN_SPACE));
}
/* Read all command tokens. Note that we read from lx, but put in lx_c. */
while(1) {
struct token *t = lexer_read_command_argument(lx);
if(!t)
break;
if(t->type == TOKEN_SUBSTITUTION) {
tokens = list_splice(tokens, lexer_expand_substitution(lx, t, lexer_read_command_aux));
lexer_free_token(t);
continue;
} else {
list_push_tail(tokens, t);
if(t->type==TOKEN_NEWLINE) break;
}
}
return tokens;
}
struct token *lexer_next_token(struct lexer_book *bk, struct dag_lookup_set *s)
{
struct token *head;
if(list_size(bk->token_queue) == 0) {
if(bk->eof)
return NULL;
lexer_read_line(bk);
return lexer_next_token(bk, s);
}
head = list_pop_head(bk->token_queue);
if(head->type == SUBSTITUTION) {
char *substitution = dag_lookup(head->lexeme, s);
struct lexer_book *bk_s;
if(!substitution) {
debug(D_NOTICE, "Variable %s has not yet been defined at line %d.\n", head->lexeme, bk->line_number);
return lexer_next_token(bk, s);
}
bk_s = lexer_init_book(STRING, dag_lookup(head->lexeme, s), bk->line_number, bk->column_number);
lexer_append_all_tokens(bk, bk_s);
lexer_free_book(bk_s);
lexer_free_token(head);
head = list_pop_head(bk->token_queue);
}
return head;
}
void dag_parse_drop_spaces(struct lexer *bk)
{
struct token *t;
while((t = lexer_peek_next_token(bk)) && t->type == TOKEN_SPACE) {
t = lexer_next_token(bk);
lexer_free_token(t);
}
}
void lexer_concatenate_consecutive_literals(struct list *tokens)
{
struct list *tmp = list_create();
struct token *t, *prev = NULL;
list_first_item(tokens);
while((t = list_pop_head(tokens))) {
if(t->type != TOKEN_LITERAL) {
list_push_tail(tmp, t);
continue;
}
prev = list_pop_tail(tmp);
if(!prev) {
list_push_tail(tmp, t);
continue;
}
if(prev->type != TOKEN_LITERAL) {
list_push_tail(tmp, prev);
list_push_tail(tmp, t);
continue;
}
char *merge = string_format("%s%s", prev->lexeme, t->lexeme);
lexer_free_token(t);
free(prev->lexeme);
prev->lexeme = merge;
list_push_tail(tmp, prev);
}
/* Copy to tokens, drop spaces. */
list_first_item(tmp);
while((t = list_pop_head(tmp)))
if(t->type != TOKEN_SPACE) {
list_push_tail(tokens, t);
} else {
lexer_free_token(t);
}
list_delete(tmp);
}
static int dag_parse_node(struct lexer *bk)
{
struct token *t = lexer_next_token(bk);
if(t->type != TOKEN_FILES)
{
lexer_report_error(bk, "Error reading rule.");
}
lexer_free_token(t);
struct dag_node *n;
n = dag_node_create(bk->d, bk->line_number);
if(verbose_parsing && bk->d->nodeid_counter % parsing_rule_mod_counter == 0)
{
fprintf(stdout, "\rRules parsed: %d", bk->d->nodeid_counter + 1);
fflush(stdout);
}
n->category = bk->category;
list_push_tail(n->category->nodes, n);
dag_parse_node_filelist(bk, n);
bk->environment->node = n;
/* Read variables, if any */
while((t = lexer_peek_next_token(bk)) && t->type != TOKEN_COMMAND)
{
switch (t->type) {
case TOKEN_VARIABLE:
dag_parse_variable(bk, n);
break;
default:
lexer_report_error(bk, "Expected COMMAND or VARIABLE, got: %s", lexer_print_token(t));
break;
}
}
if(!t)
{
lexer_report_error(bk, "Rule does not have a command.\n");
}
dag_parse_node_command(bk, n);
bk->environment->node = NULL;
n->next = bk->d->nodes;
bk->d->nodes = n;
itable_insert(bk->d->node_table, n->nodeid, n);
debug(D_MAKEFLOW_PARSER, "Setting resource category '%s' for rule %d.\n", n->category->label, n->nodeid);
dag_node_fill_resources(n);
dag_node_print_debug_resources(n);
return 1;
}
static int dag_parse_syntax(struct lexer *bk)
{
struct token *t = lexer_next_token(bk);
if(strcmp(t->lexeme, "export") == 0) {
lexer_free_token(t);
dag_parse_export(bk);
} else {
lexer_report_error(bk, "Unknown syntax keyboard.\n");
}
return 1;
}
static int dag_parse(struct dag *d, FILE *stream)
{
struct lexer *bk = lexer_create(STREAM, stream, 1, 1);
bk->d = d;
bk->stream = stream;
bk->category = dag_task_category_lookup_or_create(d, "default");
struct dag_variable_lookup_set s = { d, NULL, NULL, NULL };
bk->environment = &s;
struct token *t;
while((t = lexer_peek_next_token(bk)))
{
s.category = bk->category;
s.node = NULL;
s.table = NULL;
switch (t->type) {
case TOKEN_NEWLINE:
case TOKEN_SPACE:
/* Skip newlines, spaces at top level. */
lexer_free_token(lexer_next_token(bk));
break;
case TOKEN_SYNTAX:
dag_parse_syntax(bk);
break;
case TOKEN_FILES:
dag_parse_node(bk);
break;
case TOKEN_VARIABLE:
dag_parse_variable(bk, NULL);
break;
default:
lexer_report_error(bk, "Unexpected token. Expected one of NEWLINE, SPACE, SYNTAX, FILES, or VARIABLE, but got: %s\n:", lexer_print_token(t));
break;
}
}
dag_close_over_environment(d);
dag_compile_ancestors(d);
free(bk);
return 1;
}
void lexer_free(s_lexer *lexer)
{
s_token_list *token_list = NULL;
s_token_list *token_list_next = NULL;
if (lexer == NULL)
return;
token_list = lexer->token_list;
while (token_list != NULL)
{
token_list_next = token_list->next;
lexer_free_token(token_list->token);
mfree(token_list);
token_list = token_list_next;
}
mfree(lexer);
}
static int dag_parse_node_regular_command(struct lexer *bk, struct dag_node *n)
{
struct buffer b;
buffer_init(&b);
struct token *t;
while((t = lexer_next_token(bk)) && t->type != TOKEN_NEWLINE)
{
switch(t->type)
{
case TOKEN_SPACE:
buffer_printf(&b, " ");
break;
case TOKEN_LITERAL:
buffer_printf(&b, "%s", t->lexeme);
break;
case TOKEN_IO_REDIRECT:
buffer_printf(&b, "%s", t->lexeme);
break;
default:
lexer_report_error(bk, "Unexpected command token: %s.\n", lexer_print_token(t));
break;
}
lexer_free_token(t);
}
if(!t)
{
lexer_report_error(bk, "Command does not end with newline.\n");
}
n->command = xxstrdup(buffer_tostring(&b));
buffer_free(&b);
debug(D_MAKEFLOW_PARSER, "node command=%s", n->command);
return 1;
}
struct token *lexer_concat_expandable(struct lexer *lx, struct list *tokens)
{
struct token *t;
struct buffer b;
buffer_init(&b);
char *substitution;
list_first_item(tokens);
while((t = list_pop_head(tokens))) {
switch(t->type) {
case TOKEN_SUBSTITUTION:
substitution = dag_variable_lookup_string(t->lexeme, lx->environment);
if(!substitution)
fatal("Variable %s has not yet been defined at line % " PRId64 ".\n", t->lexeme, lx->line_number);
buffer_printf(&b, "%s", substitution);
free(substitution);
break;
case TOKEN_LITERAL:
if(strcmp(t->lexeme, "") != 0) // Skip empty strings.
buffer_printf(&b, "%s", t->lexeme);
break;
default:
lexer_report_error(lx, "Error in expansion, got: %s.\n", lexer_print_token(t));
break;
}
lexer_free_token(t);
}
t = lexer_pack_token(lx, TOKEN_LITERAL);
t->lexeme = xxstrdup(buffer_tostring(&b));
buffer_free(&b);
return t;
}
struct list *lexer_read_file_list_aux(struct lexer *lx)
{
struct list *tokens = list_create();
lexer_discard_white_space(lx);
while(1) {
struct token *t = lexer_read_file(lx);
if(!t) break;
//Do substitution recursively
if(t->type == TOKEN_SUBSTITUTION) {
tokens = list_splice(tokens, lexer_expand_substitution(lx, t, lexer_read_file_list_aux));
lexer_free_token(t);
continue;
} else {
list_push_tail(tokens, t);
if(t->type==TOKEN_NEWLINE) break;
}
}
return tokens;
}
static int dag_parse_node_filelist(struct lexer *bk, struct dag_node *n)
{
int before_colon = 1;
char *filename;
char *newname;
struct token *t, *arrow, *rename;
while((t = lexer_next_token(bk)))
{
filename = NULL;
newname = NULL;
switch (t->type) {
case TOKEN_COLON:
before_colon = 0;
lexer_free_token(t);
break;
case TOKEN_NEWLINE:
/* Finished reading file list */
lexer_free_token(t);
return 1;
break;
case TOKEN_LITERAL:
rename = NULL;
arrow = lexer_peek_next_token(bk);
if(!arrow)
{
lexer_report_error(bk, "Rule specification is incomplete.");
}
else if(arrow->type == TOKEN_REMOTE_RENAME) //Is the arrow really an arrow?
{
lexer_free_token(lexer_next_token(bk)); //Jump arrow.
rename = lexer_next_token(bk);
if(!rename)
{
lexer_report_error(bk, "Remote name specification is incomplete.");
}
}
filename = t->lexeme;
newname = rename ? rename->lexeme : NULL;
if(before_colon)
dag_node_add_target_file(n, filename, newname);
else
dag_node_add_source_file(n, filename, newname);
lexer_free_token(t);
if(rename)
{
lexer_free_token(rename);
}
break;
default:
lexer_report_error(bk, "Error reading file list. %s", lexer_print_token(t));
break;
}
}
return 0;
}
