int makeflow_clean_file( struct dag *d, struct batch_queue *queue, struct dag_file *f, int silent) { if(!f) return 1; if(batch_fs_unlink(queue, f->filename) == 0) { makeflow_log_file_state_change(d, f, DAG_FILE_STATE_DELETE); debug(D_MAKEFLOW_RUN, "File deleted %s\n", f->filename); } else if(errno != ENOENT) { if(f->state == DAG_FILE_STATE_EXPECT || dag_file_should_exist(f)) makeflow_log_file_state_change(d, f, DAG_FILE_STATE_DELETE); debug(D_MAKEFLOW_RUN, "Makeflow: Couldn't delete %s: %s\n", f->filename, strerror(errno)); return 1; } return 0; }
static int node_submit( void * instance_struct, struct dag_node *n, struct batch_task *t){ struct vc3_definition *v = (struct vc3_definition*)instance_struct; struct batch_wrapper *wrapper = batch_wrapper_create(); batch_wrapper_prefix(wrapper, "./vc3_builder_"); char * executable = NULL; // If the queue supports remote_renaming add as remote rename. if (batch_queue_supports_feature(makeflow_get_queue(n), "remote_rename")) { executable = string_format("./%s", path_basename(v->exe)); } else { // Else just use executable in path executable = string_format("%s", v->exe); } /* Assumes a /disk dir in the image. */ char *log = string_format("%s_%d", v->log, t->taskid); char *task_cmd = string_escape_shell(t->command); char *cmd = string_format("%s --home $PWD %s -- %s > %s", executable, v->opt, task_cmd, log); makeflow_hook_add_input_file(n->d, t, v->exe, executable, DAG_FILE_TYPE_GLOBAL); makeflow_hook_add_output_file(n->d, t, log, log, DAG_FILE_TYPE_INTERMEDIATE); free(log); free(executable); free(task_cmd); batch_wrapper_cmd(wrapper, cmd); free(cmd); cmd = batch_wrapper_write(wrapper, t); if(cmd){ batch_task_set_command(t, cmd); struct dag_file *df = makeflow_hook_add_input_file(n->d, t, cmd, cmd, DAG_FILE_TYPE_TEMP); debug(D_MAKEFLOW_HOOK, "Wrapper written to %s", df->filename); makeflow_log_file_state_change(n->d, df, DAG_FILE_STATE_EXISTS); } else { debug(D_MAKEFLOW_HOOK, "Failed to create wrapper: errno %d, %s", errno, strerror(errno)); return MAKEFLOW_HOOK_FAILURE; } free(cmd); return MAKEFLOW_HOOK_SUCCESS; }
static int makeflow_module_sandbox_node_submit( void * instance_struct, struct dag_node *node, struct batch_task *task){ struct batch_wrapper *wrapper = batch_wrapper_create(); char *wrap_name = string_format("./task_%d_sandbox", task->taskid); batch_wrapper_prefix(wrapper, wrap_name); /* Save the directory we were originally working in. */ batch_wrapper_pre(wrapper, "export CUR_WORK_DIR=$(pwd)"); /* Create sandbox. This should probably have a hex or random tail to be unique. */ char *cmd = string_format("export SANDBOX=$(mktemp -d %s_XXXXXX)", wrap_name); batch_wrapper_pre(wrapper, cmd); free(cmd); free(wrap_name); struct batch_file *f; list_first_item(task->input_files); while((f = list_next_item(task->input_files))){ /* Skip if absolute path. */ if(f->inner_name[0] == '/') continue; /* Add a cp for each file. Not linking as wq may already have done this. Not moving as it may be local. */ cmd = string_format("mkdir -p $(dirname $SANDBOX/%s) && cp -r %s $SANDBOX/%s", f->inner_name, f->inner_name, f->inner_name); batch_wrapper_pre(wrapper, cmd); free(cmd); } /* Enter into sandbox_dir. */ batch_wrapper_pre(wrapper, "cd $SANDBOX"); /* Execute the previous levels commmand. */ batch_wrapper_cmd(wrapper, task->command); /* Once the command is finished go back to working dir. */ batch_wrapper_post(wrapper, "cd $CUR_WORK_DIR"); list_first_item(task->output_files); while((f = list_next_item(task->output_files))){ /* Skip if absolute path. */ if(f->inner_name[0] == '/') continue; /* Copy out results to expected location. OR TRUE so that lack of one file does not prevent other files from being sent back.*/ cmd = string_format("mkdir -p $(dirname %s) && cp -r $SANDBOX/%s %s || true", f->inner_name, f->inner_name, f->inner_name); batch_wrapper_post(wrapper, cmd); free(cmd); } /* Remove and fully wipe out sandbox. */ batch_wrapper_post(wrapper, "rm -rf $SANDBOX"); cmd = batch_wrapper_write(wrapper, task); if(cmd){ batch_task_set_command(task, cmd); struct dag_file *df = makeflow_hook_add_input_file(node->d, task, cmd, cmd, DAG_FILE_TYPE_TEMP); debug(D_MAKEFLOW_HOOK, "Wrapper written to %s", df->filename); makeflow_log_file_state_change(node->d, df, DAG_FILE_STATE_EXISTS); } else { debug(D_MAKEFLOW_HOOK, "Failed to create wrapper: errno %d, %s", errno, strerror(errno)); return MAKEFLOW_HOOK_FAILURE; } free(cmd); return MAKEFLOW_HOOK_SUCCESS; }
/** The clean_mode variable was added so that we could better print out error messages * apply in the situation. Currently only used to silence node rerun checking. */ void makeflow_log_recover(struct dag *d, const char *filename, int verbose_mode, struct batch_queue *queue, makeflow_clean_depth clean_mode) { char *line, *name, file[MAX_BUFFER_SIZE]; int nodeid, state, jobid, file_state; int first_run = 1; struct dag_node *n; struct dag_file *f; struct stat buf; timestamp_t previous_completion_time; d->logfile = fopen(filename, "r"); if(d->logfile) { int linenum = 0; first_run = 0; printf("recovering from log file %s...\n",filename); while((line = get_line(d->logfile))) { linenum++; if(sscanf(line, "# %d %s %" SCNu64 "", &file_state, file, &previous_completion_time) == 3) { f = dag_file_lookup_or_create(d, file); f->state = file_state; if(file_state == DAG_FILE_STATE_EXISTS){ d->completed_files += 1; f->creation_logged = (time_t) (previous_completion_time / 1000000); } else if(file_state == DAG_FILE_STATE_DELETE){ d->deleted_files += 1; } continue; } if(line[0] == '#') continue; if(sscanf(line, "%" SCNu64 " %d %d %d", &previous_completion_time, &nodeid, &state, &jobid) == 4) { n = itable_lookup(d->node_table, nodeid); if(n) { n->state = state; n->jobid = jobid; /* Log timestamp is in microseconds, we need seconds for diff. */ n->previous_completion = (time_t) (previous_completion_time / 1000000); continue; } } fprintf(stderr, "makeflow: %s appears to be corrupted on line %d\n", filename, linenum); exit(1); } fclose(d->logfile); } d->logfile = fopen(filename, "a"); if(!d->logfile) { fprintf(stderr, "makeflow: couldn't open logfile %s: %s\n", filename, strerror(errno)); exit(1); } if(setvbuf(d->logfile, NULL, _IOLBF, BUFSIZ) != 0) { fprintf(stderr, "makeflow: couldn't set line buffer on logfile %s: %s\n", filename, strerror(errno)); exit(1); } if(first_run && verbose_mode) { struct dag_file *f; struct dag_node *p; for(n = d->nodes; n; n = n->next) { /* Record node information to log */ fprintf(d->logfile, "# NODE\t%d\t%s\n", n->nodeid, n->command); /* Record the node category to the log */ fprintf(d->logfile, "# CATEGORY\t%d\t%s\n", n->nodeid, n->category->label); fprintf(d->logfile, "# SYMBOL\t%d\t%s\n", n->nodeid, n->category->label); /* also write the SYMBOL as alias of CATEGORY, deprecated. */ /* Record node parents to log */ fprintf(d->logfile, "# PARENTS\t%d", n->nodeid); list_first_item(n->source_files); while( (f = list_next_item(n->source_files)) ) { p = f->created_by; if(p) fprintf(d->logfile, "\t%d", p->nodeid); } fputc('\n', d->logfile); /* Record node inputs to log */ fprintf(d->logfile, "# SOURCES\t%d", n->nodeid); list_first_item(n->source_files); while( (f = list_next_item(n->source_files)) ) { fprintf(d->logfile, "\t%s", f->filename); } fputc('\n', d->logfile); /* Record node outputs to log */ fprintf(d->logfile, "# TARGETS\t%d", n->nodeid); list_first_item(n->target_files); while( (f = list_next_item(n->target_files)) ) { fprintf(d->logfile, "\t%s", f->filename); } fputc('\n', d->logfile); /* Record translated command to log */ fprintf(d->logfile, "# COMMAND\t%d\t%s\n", n->nodeid, n->command); } } dag_count_states(d); // Check for log consistency if(!first_run) { hash_table_firstkey(d->files); while(hash_table_nextkey(d->files, &name, (void **) &f)) { if(dag_file_should_exist(f) && !dag_file_is_source(f) && !(batch_fs_stat(queue, f->filename, &buf) >= 0)){ fprintf(stderr, "makeflow: %s is reported as existing, but does not exist.\n", f->filename); makeflow_log_file_state_change(d, f, DAG_FILE_STATE_UNKNOWN); continue; } if(S_ISDIR(buf.st_mode)) continue; if(dag_file_should_exist(f) && !dag_file_is_source(f) && difftime(buf.st_mtime, f->creation_logged) > 0) { fprintf(stderr, "makeflow: %s is reported as existing, but has been modified (%" SCNu64 " ,%" SCNu64 ").\n", f->filename, (uint64_t)buf.st_mtime, (uint64_t)f->creation_logged); makeflow_clean_file(d, queue, f, 0); makeflow_log_file_state_change(d, f, DAG_FILE_STATE_UNKNOWN); } } } int silent = 0; if(clean_mode != MAKEFLOW_CLEAN_NONE) silent = 1; // Decide rerun tasks if(!first_run) { struct itable *rerun_table = itable_create(0); for(n = d->nodes; n; n = n->next) { makeflow_node_decide_rerun(rerun_table, d, n, silent); } itable_delete(rerun_table); } //Update file reference counts from nodes in log for(n = d->nodes; n; n = n->next) { if(n->state == DAG_NODE_STATE_COMPLETE) { struct dag_file *f; list_first_item(n->source_files); while((f = list_next_item(n->source_files))) f->ref_count += -1; } } }