void makeflow_gc_prepare( struct dag *d ) { /* Files to be collected: * ((all_files \minus sink_files)) \union collect_list) \minus preserve_list) \minus source_files */ /* Parse GC_*_LIST and record which target files should be * garbage collected. */ char *collect_list = dag_variable_lookup_global_string("GC_COLLECT_LIST", d); char *preserve_list = dag_variable_lookup_global_string("GC_PRESERVE_LIST", d); struct dag_file *f; char *filename; /* add all files, but sink_files */ hash_table_firstkey(d->files); while((hash_table_nextkey(d->files, &filename, (void **) &f))) if(!dag_file_is_sink(f)) { set_insert(d->collect_table, f); } int i, argc; char **argv; /* add collect_list, for sink_files that should be removed */ string_split_quotes(collect_list, &argc, &argv); for(i = 0; i < argc; i++) { f = dag_file_lookup_or_create(d, argv[i]); set_insert(d->collect_table, f); debug(D_MAKEFLOW_RUN, "Added %s to garbage collection list", f->filename); } free(argv); /* remove files from preserve_list */ string_split_quotes(preserve_list, &argc, &argv); for(i = 0; i < argc; i++) { /* Must initialize to non-zero for hash_table functions to work properly. */ f = dag_file_lookup_or_create(d, argv[i]); set_remove(d->collect_table, f); debug(D_MAKEFLOW_RUN, "Removed %s from garbage collection list", f->filename); } free(argv); /* remove source_files from collect_table */ hash_table_firstkey(d->files); while((hash_table_nextkey(d->files, &filename, (void **) &f))) if(dag_file_is_source(f)) { set_remove(d->collect_table, f); debug(D_MAKEFLOW_RUN, "Removed %s from garbage collection list", f->filename); } /* Print reference counts of files to be collected */ set_first_element(d->collect_table); while((f = set_next_element(d->collect_table))) debug(D_MAKEFLOW_RUN, "Added %s to garbage collection list (%d)", f->filename, f->ref_count); }
/** Parse a node's input or output filelist. Parse through a list of input or output files, adding each as a source or target file to the provided node. @param d The DAG being constructed @param n The node that the files are being added to @param filelist The list of files, separated by whitespace @param source a flag for whether the files are source or target files. 1 indicates source files, 0 indicates targets */ int dag_parse_node_filelist(struct lexer_book *bk, struct dag_node *n, char *filelist, int source) { char *filename; char *newname; char **argv; int i, argc; string_split_quotes(filelist, &argc, &argv); for(i = 0; i < argc; i++) { filename = argv[i]; newname = NULL; debug(D_DEBUG, "node %s file=%s", (source ? "input" : "output"), filename); // remote renaming if((newname = strstr(filename, "->"))) { *newname = '\0'; newname += 2; } if(source) dag_node_add_source_file(n, filename, newname); else dag_node_add_target_file(n, filename, newname); } free(argv); return 1; }
int ast_simple_execute( struct ast_simple *s, time_t stoptime ) { int result=0; int fds[3]; int argc; char **argv; char *line; fds[0] = 0; fds[1] = 1; fds[2] = 2; if( stoptime && (time(0)>stoptime) ) return 0; if(ast_redirect_open(s->redirects,s->line,fds)) { line = ast_word_list_execute(s->line,s->words); if(line) { if(string_split_quotes(line,&argc,&argv)) { result = ast_do_simple(s->line,argc,argv,fds,stoptime); free(argv); } free(line); } ast_redirect_close(s->redirects); } return result; }
int ast_forloop_execute( struct ast_forloop *f, time_t stoptime ) { char *loopname; char *name; char *line; int result=1; switch(f->type) { case AST_FOR: loopname = "FOR"; break; case AST_FORALL: loopname = "FORALL"; break; case AST_FORANY: loopname = "FORANY"; break; } ftsh_error(FTSH_ERROR_STRUCTURE,f->for_line,"%s %s",loopname,f->name->text); name = ast_word_execute(f->for_line,f->name); if(name) { line = ast_expr_list_execute(f->for_line,f->list,stoptime); if(line) { int argc; char **argv; if(string_split_quotes( line, &argc, &argv )) { switch(f->type) { case AST_FOR: result = ast_for_execute(f,stoptime,name,argc,argv); break; case AST_FORANY: result = ast_forany_execute(f,stoptime,name,argc,argv); break; case AST_FORALL: result = ast_forall_execute(f,stoptime,name,argc,argv); break; } free(argv); } else { ftsh_error(FTSH_ERROR_FAILURE,f->for_line,"out of memory!"); result = 0; } free(line); } free(name); } else { result = 0; } ftsh_error(FTSH_ERROR_STRUCTURE,f->end_line,"END"); return result; }
static char * ast_bareword_execute( int linenum, char *line ) { int argc; char **argv; char *result=0; if(string_split_quotes(line,&argc,&argv)) { if(argc==1) { result = xxstrdup(argv[0]); } else if(argc>1) { ftsh_error(FTSH_ERROR_SYNTAX,linenum,"expected only one word here, but got garbage following '%s'",argv[0]); } else { ftsh_error(FTSH_ERROR_SYNTAX,linenum,"expected a word here, but found nothing"); } free(argv); } return result; }
int dag_parse_node_makeflow_command(struct lexer_book *bk, struct dag_node *n, char *line) { int argc; char **argv; char *wrapper = NULL; char *command = NULL; n->nested_job = 1; string_split_quotes(line, &argc, &argv); switch (argc) { case 1: n->makeflow_dag = xxstrdup(argv[0]); n->makeflow_cwd = xxstrdup("."); break; case 2: n->makeflow_dag = xxstrdup(argv[0]); n->makeflow_cwd = xxstrdup(argv[1]); break; case 3: n->makeflow_dag = xxstrdup(argv[0]); n->makeflow_cwd = xxstrdup(argv[1]); wrapper = argv[2]; break; default: dag_parse_error(bk, "node makeflow command"); goto failure; } wrapper = wrapper ? wrapper : ""; command = xxmalloc(sizeof(char) * (strlen(n->makeflow_cwd) + strlen(wrapper) + strlen(makeflow_exe) + strlen(n->makeflow_dag) + 20)); sprintf(command, "cd %s && %s %s %s", n->makeflow_cwd, wrapper, makeflow_exe, n->makeflow_dag); dag_parse_node_filelist(bk, n, argv[0], 1); dag_parse_node_set_command(bk, n, command); free(argv); free(command); return 1; failure: free(argv); return 0; }
int dag_parse_export(struct lexer_book *bk, char *line) { int i, argc; char *end_export, *equal; char **argv; end_export = strstr(line, "export "); if(!end_export) return 0; else end_export += strlen("export "); while(isblank(*end_export)) end_export++; if(end_export == '\0') return 0; string_split_quotes(end_export, &argc, &argv); for(i = 0; i < argc; i++) { equal = strchr(argv[i], '='); if(equal) { if(!dag_parse_variable(bk, NULL, argv[i])) { return 0; } else { *equal = '\0'; setenv(argv[i], equal + 1, 1); //this shouldn't be here... } } set_insert(bk->d->export_vars, xxstrdup(argv[i])); debug(D_DEBUG, "export variable=%s", argv[i]); } free(argv); return 1; }
void makeflow_parse_input_outputs( struct dag *d ) { /* Check if GC_*_LIST is specified and warn user about deprecated usage */ char *collect_list = dag_variable_lookup_global_string("GC_COLLECT_LIST" , d); if(collect_list) debug(D_NOTICE, "GC_COLLECT_LIST is specified: Please refer to manual about MAKEFLOW_INPUTS/OUTPUTS"); char *preserve_list = dag_variable_lookup_global_string("GC_PRESERVE_LIST", d); if(preserve_list) debug(D_NOTICE, "GC_PRESERVE_LIST is specified: Please refer to manual about MAKEFLOW_INPUTS/OUTPUTS"); /* Parse INPUT and OUTPUT lists */ struct dag_file *f; char *filename; int i, argc; char **argv; char *input_list = dag_variable_lookup_global_string("MAKEFLOW_INPUTS" , d); char *output_list = dag_variable_lookup_global_string("MAKEFLOW_OUTPUTS", d); if(input_list) { /* add collect_list, for sink_files that should be removed */ string_split_quotes(input_list, &argc, &argv); for(i = 0; i < argc; i++) { d->completed_files += 1; f = dag_file_lookup_or_create(d, argv[i]); set_insert(d->inputs, f); debug(D_MAKEFLOW_RUN, "Added %s to input list", f->filename); } free(argv); } else { debug(D_NOTICE, "MAKEFLOW_INPUTS is not specified"); } /* add all source files */ hash_table_firstkey(d->files); while((hash_table_nextkey(d->files, &filename, (void **) &f))) if(dag_file_is_source(f)) { set_insert(d->inputs, f); debug(D_MAKEFLOW_RUN, "Added %s to input list", f->filename); } if(output_list) { /* remove files from preserve_list */ string_split_quotes(output_list, &argc, &argv); for(i = 0; i < argc; i++) { /* Must initialize to non-zero for hash_table functions to work properly. */ f = dag_file_lookup_or_create(d, argv[i]); set_remove(d->outputs, f); debug(D_MAKEFLOW_RUN, "Added %s to output list", f->filename); } free(argv); } else { debug(D_NOTICE, "MAKEFLOW_OUTPUTS is not specified"); /* add all sink if OUTPUTS not specified */ hash_table_firstkey(d->files); while((hash_table_nextkey(d->files, &filename, (void **) &f))) if(dag_file_is_sink(f)) { set_insert(d->outputs, f); debug(D_MAKEFLOW_RUN, "Added %s to output list", f->filename); } } }