void makeflow_gc_prepare( struct dag *d )
{
/* Files to be collected:
* ((all_files \minus sink_files)) \union collect_list) \minus preserve_list) \minus source_files
*/
/* Parse GC_*_LIST and record which target files should be
* garbage collected. */
char *collect_list = dag_variable_lookup_global_string("GC_COLLECT_LIST", d);
char *preserve_list = dag_variable_lookup_global_string("GC_PRESERVE_LIST", d);
struct dag_file *f;
char *filename;
/* add all files, but sink_files */
hash_table_firstkey(d->files);
while((hash_table_nextkey(d->files, &filename, (void **) &f)))
if(!dag_file_is_sink(f)) {
set_insert(d->collect_table, f);
}
int i, argc;
char **argv;
/* add collect_list, for sink_files that should be removed */
string_split_quotes(collect_list, &argc, &argv);
for(i = 0; i < argc; i++) {
f = dag_file_lookup_or_create(d, argv[i]);
set_insert(d->collect_table, f);
debug(D_MAKEFLOW_RUN, "Added %s to garbage collection list", f->filename);
}
free(argv);
/* remove files from preserve_list */
string_split_quotes(preserve_list, &argc, &argv);
for(i = 0; i < argc; i++) {
/* Must initialize to non-zero for hash_table functions to work properly. */
f = dag_file_lookup_or_create(d, argv[i]);
set_remove(d->collect_table, f);
debug(D_MAKEFLOW_RUN, "Removed %s from garbage collection list", f->filename);
}
free(argv);
/* remove source_files from collect_table */
hash_table_firstkey(d->files);
while((hash_table_nextkey(d->files, &filename, (void **) &f)))
if(dag_file_is_source(f)) {
set_remove(d->collect_table, f);
debug(D_MAKEFLOW_RUN, "Removed %s from garbage collection list", f->filename);
}
/* Print reference counts of files to be collected */
set_first_element(d->collect_table);
while((f = set_next_element(d->collect_table)))
debug(D_MAKEFLOW_RUN, "Added %s to garbage collection list (%d)", f->filename, f->ref_count);
}
/** Parse a node's input or output filelist.
Parse through a list of input or output files, adding each as a source or target file to the provided node.
@param d The DAG being constructed
@param n The node that the files are being added to
@param filelist The list of files, separated by whitespace
@param source a flag for whether the files are source or target files. 1 indicates source files, 0 indicates targets
*/
int dag_parse_node_filelist(struct lexer_book *bk, struct dag_node *n, char *filelist, int source)
{
char *filename;
char *newname;
char **argv;
int i, argc;
string_split_quotes(filelist, &argc, &argv);
for(i = 0; i < argc; i++) {
filename = argv[i];
newname = NULL;
debug(D_DEBUG, "node %s file=%s", (source ? "input" : "output"), filename);
// remote renaming
if((newname = strstr(filename, "->"))) {
*newname = '\0';
newname += 2;
}
if(source)
dag_node_add_source_file(n, filename, newname);
else
dag_node_add_target_file(n, filename, newname);
}
free(argv);
return 1;
}
int ast_simple_execute( struct ast_simple *s, time_t stoptime )
{
int result=0;
int fds[3];
int argc;
char **argv;
char *line;
fds[0] = 0;
fds[1] = 1;
fds[2] = 2;
if( stoptime && (time(0)>stoptime) ) return 0;
if(ast_redirect_open(s->redirects,s->line,fds)) {
line = ast_word_list_execute(s->line,s->words);
if(line) {
if(string_split_quotes(line,&argc,&argv)) {
result = ast_do_simple(s->line,argc,argv,fds,stoptime);
free(argv);
}
free(line);
}
ast_redirect_close(s->redirects);
}
return result;
}
int ast_forloop_execute( struct ast_forloop *f, time_t stoptime )
{
char *loopname;
char *name;
char *line;
int result=1;
switch(f->type) {
case AST_FOR:
loopname = "FOR";
break;
case AST_FORALL:
loopname = "FORALL";
break;
case AST_FORANY:
loopname = "FORANY";
break;
}
ftsh_error(FTSH_ERROR_STRUCTURE,f->for_line,"%s %s",loopname,f->name->text);
name = ast_word_execute(f->for_line,f->name);
if(name) {
line = ast_expr_list_execute(f->for_line,f->list,stoptime);
if(line) {
int argc;
char **argv;
if(string_split_quotes( line, &argc, &argv )) {
switch(f->type) {
case AST_FOR:
result = ast_for_execute(f,stoptime,name,argc,argv);
break;
case AST_FORANY:
result = ast_forany_execute(f,stoptime,name,argc,argv);
break;
case AST_FORALL:
result = ast_forall_execute(f,stoptime,name,argc,argv);
break;
}
free(argv);
} else {
ftsh_error(FTSH_ERROR_FAILURE,f->for_line,"out of memory!");
result = 0;
}
free(line);
}
free(name);
} else {
result = 0;
}
ftsh_error(FTSH_ERROR_STRUCTURE,f->end_line,"END");
return result;
}
static char * ast_bareword_execute( int linenum, char *line )
{
int argc;
char **argv;
char *result=0;
if(string_split_quotes(line,&argc,&argv)) {
if(argc==1) {
result = xxstrdup(argv[0]);
} else if(argc>1) {
ftsh_error(FTSH_ERROR_SYNTAX,linenum,"expected only one word here, but got garbage following '%s'",argv[0]);
} else {
ftsh_error(FTSH_ERROR_SYNTAX,linenum,"expected a word here, but found nothing");
}
free(argv);
}
return result;
}
int dag_parse_node_makeflow_command(struct lexer_book *bk, struct dag_node *n, char *line)
{
int argc;
char **argv;
char *wrapper = NULL;
char *command = NULL;
n->nested_job = 1;
string_split_quotes(line, &argc, &argv);
switch (argc) {
case 1:
n->makeflow_dag = xxstrdup(argv[0]);
n->makeflow_cwd = xxstrdup(".");
break;
case 2:
n->makeflow_dag = xxstrdup(argv[0]);
n->makeflow_cwd = xxstrdup(argv[1]);
break;
case 3:
n->makeflow_dag = xxstrdup(argv[0]);
n->makeflow_cwd = xxstrdup(argv[1]);
wrapper = argv[2];
break;
default:
dag_parse_error(bk, "node makeflow command");
goto failure;
}
wrapper = wrapper ? wrapper : "";
command = xxmalloc(sizeof(char) * (strlen(n->makeflow_cwd) + strlen(wrapper) + strlen(makeflow_exe) + strlen(n->makeflow_dag) + 20));
sprintf(command, "cd %s && %s %s %s", n->makeflow_cwd, wrapper, makeflow_exe, n->makeflow_dag);
dag_parse_node_filelist(bk, n, argv[0], 1);
dag_parse_node_set_command(bk, n, command);
free(argv);
free(command);
return 1;
failure:
free(argv);
return 0;
}
int dag_parse_export(struct lexer_book *bk, char *line)
{
int i, argc;
char *end_export, *equal;
char **argv;
end_export = strstr(line, "export ");
if(!end_export)
return 0;
else
end_export += strlen("export ");
while(isblank(*end_export))
end_export++;
if(end_export == '\0')
return 0;
string_split_quotes(end_export, &argc, &argv);
for(i = 0; i < argc; i++) {
equal = strchr(argv[i], '=');
if(equal) {
if(!dag_parse_variable(bk, NULL, argv[i])) {
return 0;
} else {
*equal = '\0';
setenv(argv[i], equal + 1, 1); //this shouldn't be here...
}
}
set_insert(bk->d->export_vars, xxstrdup(argv[i]));
debug(D_DEBUG, "export variable=%s", argv[i]);
}
free(argv);
return 1;
}
void makeflow_parse_input_outputs( struct dag *d )
{
/* Check if GC_*_LIST is specified and warn user about deprecated usage */
char *collect_list = dag_variable_lookup_global_string("GC_COLLECT_LIST" , d);
if(collect_list)
debug(D_NOTICE, "GC_COLLECT_LIST is specified: Please refer to manual about MAKEFLOW_INPUTS/OUTPUTS");
char *preserve_list = dag_variable_lookup_global_string("GC_PRESERVE_LIST", d);
if(preserve_list)
debug(D_NOTICE, "GC_PRESERVE_LIST is specified: Please refer to manual about MAKEFLOW_INPUTS/OUTPUTS");
/* Parse INPUT and OUTPUT lists */
struct dag_file *f;
char *filename;
int i, argc;
char **argv;
char *input_list = dag_variable_lookup_global_string("MAKEFLOW_INPUTS" , d);
char *output_list = dag_variable_lookup_global_string("MAKEFLOW_OUTPUTS", d);
if(input_list) {
/* add collect_list, for sink_files that should be removed */
string_split_quotes(input_list, &argc, &argv);
for(i = 0; i < argc; i++) {
d->completed_files += 1;
f = dag_file_lookup_or_create(d, argv[i]);
set_insert(d->inputs, f);
debug(D_MAKEFLOW_RUN, "Added %s to input list", f->filename);
}
free(argv);
} else {
debug(D_NOTICE, "MAKEFLOW_INPUTS is not specified");
}
/* add all source files */
hash_table_firstkey(d->files);
while((hash_table_nextkey(d->files, &filename, (void **) &f)))
if(dag_file_is_source(f)) {
set_insert(d->inputs, f);
debug(D_MAKEFLOW_RUN, "Added %s to input list", f->filename);
}
if(output_list) {
/* remove files from preserve_list */
string_split_quotes(output_list, &argc, &argv);
for(i = 0; i < argc; i++) {
/* Must initialize to non-zero for hash_table functions to work properly. */
f = dag_file_lookup_or_create(d, argv[i]);
set_remove(d->outputs, f);
debug(D_MAKEFLOW_RUN, "Added %s to output list", f->filename);
}
free(argv);
} else {
debug(D_NOTICE, "MAKEFLOW_OUTPUTS is not specified");
/* add all sink if OUTPUTS not specified */
hash_table_firstkey(d->files);
while((hash_table_nextkey(d->files, &filename, (void **) &f)))
if(dag_file_is_sink(f)) {
set_insert(d->outputs, f);
debug(D_MAKEFLOW_RUN, "Added %s to output list", f->filename);
}
}
}
