char *makeflow_wrap_umbrella(char *result, struct dag_node *n, struct makeflow_wrapper_umbrella *w, struct batch_queue *queue, char *input_files, char *output_files) {
if(!w || !w->spec) return result;
char *umbrella_command = NULL;
char *umbrella_input_opt = NULL;
char *umbrella_output_opt = NULL;
char *umbrella_logfile = NULL;
bool remote_rename_support = false;
if (batch_queue_supports_feature(queue, "remote_rename")) {
remote_rename_support = true;
}
umbrella_logfile = string_format("%s.%d", w->log_prefix, n->nodeid);
debug(D_MAKEFLOW_RUN, "input_files: %s\n", input_files);
umbrella_input_opt = create_umbrella_opt(remote_rename_support, input_files, false, umbrella_logfile);
debug(D_MAKEFLOW_RUN, "umbrella input opt: %s\n", umbrella_input_opt);
debug(D_MAKEFLOW_RUN, "output_files: %s\n", output_files);
umbrella_output_opt = create_umbrella_opt(remote_rename_support, output_files, true, umbrella_logfile);
debug(D_MAKEFLOW_RUN, "umbrella output opt: %s\n", umbrella_output_opt);
// construct umbrella_command
if(!remote_rename_support) {
if(!w->binary) {
umbrella_command = string_format("umbrella --spec %s \
--localdir /tmp/umbrella_test \
--inputs \"%s\" \
--output \"%s\" \
--sandbox_mode \"%s\" \
--log \"%s\" \
run \'{}\'", w->spec, umbrella_input_opt, umbrella_output_opt, w->mode, umbrella_logfile);
} else {
void makeflow_wrapper_umbrella_set_input_files(struct makeflow_wrapper_umbrella *w, struct batch_queue *queue, struct dag_node *n) {
bool remote_rename_support = false;
if(!w) return;
// every rule may have its own umbrella spec file.
// to avoid w->wrapper->input_files accumulating umbrella spec files for different rules, first delete it and then recreate it.
list_delete(w->wrapper->input_files);
w->wrapper->input_files = list_create();
if(!n->umbrella_spec) return;
if (batch_queue_supports_feature(queue, "remote_rename")) {
remote_rename_support = true;
}
// add umbrella_spec (if specified) and umbrella_binary (if specified) into the input file list of w->wrapper
if (!remote_rename_support) {
makeflow_wrapper_add_input_file(w->wrapper, n->umbrella_spec);
if(w->binary) makeflow_wrapper_add_input_file(w->wrapper, w->binary);
} else {
{
char *s = string_format("%s=%s", n->umbrella_spec, path_basename(n->umbrella_spec));
if(!s) fatal("string_format for umbrella spec failed: %s.\n", strerror(errno));
makeflow_wrapper_add_input_file(w->wrapper, s);
free(s);
}
if(w->binary) {
char *s = string_format("%s=%s", w->binary, path_basename(w->binary));
if(!s) fatal("string_format for umbrella binary failed: %s.\n", strerror(errno));
makeflow_wrapper_add_input_file(w->wrapper, s);
free(s);
}
}
}
static int node_submit( void * instance_struct, struct dag_node *n, struct batch_task *t){
struct vc3_definition *v = (struct vc3_definition*)instance_struct;
struct batch_wrapper *wrapper = batch_wrapper_create();
batch_wrapper_prefix(wrapper, "./vc3_builder_");
char * executable = NULL;
// If the queue supports remote_renaming add as remote rename.
if (batch_queue_supports_feature(makeflow_get_queue(n), "remote_rename")) {
executable = string_format("./%s", path_basename(v->exe));
} else {
// Else just use executable in path
executable = string_format("%s", v->exe);
}
/* Assumes a /disk dir in the image. */
char *log = string_format("%s_%d", v->log, t->taskid);
char *task_cmd = string_escape_shell(t->command);
char *cmd = string_format("%s --home $PWD %s -- %s > %s", executable, v->opt, task_cmd, log);
makeflow_hook_add_input_file(n->d, t, v->exe, executable, DAG_FILE_TYPE_GLOBAL);
makeflow_hook_add_output_file(n->d, t, log, log, DAG_FILE_TYPE_INTERMEDIATE);
free(log);
free(executable);
free(task_cmd);
batch_wrapper_cmd(wrapper, cmd);
free(cmd);
cmd = batch_wrapper_write(wrapper, t);
if(cmd){
batch_task_set_command(t, cmd);
struct dag_file *df = makeflow_hook_add_input_file(n->d, t, cmd, cmd, DAG_FILE_TYPE_TEMP);
debug(D_MAKEFLOW_HOOK, "Wrapper written to %s", df->filename);
makeflow_log_file_state_change(n->d, df, DAG_FILE_STATE_EXISTS);
} else {
debug(D_MAKEFLOW_HOOK, "Failed to create wrapper: errno %d, %s", errno, strerror(errno));
return MAKEFLOW_HOOK_FAILURE;
}
free(cmd);
return MAKEFLOW_HOOK_SUCCESS;
}
void makeflow_wrapper_umbrella_preparation(struct makeflow_wrapper_umbrella *w, struct batch_queue *queue, struct dag *d) {
struct dag_node *cur;
bool remote_rename_support = false;
if (batch_queue_supports_feature(queue, "remote_rename")) {
remote_rename_support = true;
}
if(!w->binary) {
debug(D_MAKEFLOW_RUN, "the --umbrella-binary option is not set, therefore an umbrella binary should be available on an execution node.\n");
fprintf(stdout, "the --umbrella-binary option is not set, therefore an umbrella binary should be available on an execution node.\n");
}
// add umbrella_spec (if specified) and umbrella_binary (if specified) into the input file list of w->wrapper
if (!remote_rename_support) {
if(w->spec) makeflow_wrapper_add_input_file(w->wrapper, w->spec);
if(w->binary) makeflow_wrapper_add_input_file(w->wrapper, w->binary);
} else {
if(w->spec) {
char *s = string_format("%s=%s", w->spec, path_basename(w->spec));
if(!s) fatal("string_format for umbrella spec failed: %s.\n", strerror(errno));
makeflow_wrapper_add_input_file(w->wrapper, s);
free(s);
}
if(w->binary) {
char *s = string_format("%s=%s", w->binary, path_basename(w->binary));
if(!s) fatal("string_format for umbrella binary failed: %s.\n", strerror(errno));
makeflow_wrapper_add_input_file(w->wrapper, s);
free(s);
}
}
// set wrapper_umbrella->log_prefix to the default value
if(!w->log_prefix) {
w->log_prefix = string_format("%s.umbrella.log", d->filename);
debug(D_MAKEFLOW_RUN, "setting wrapper_umbrella->log_prefix to %s\n", w->log_prefix);
}
// check whether the umbrella log files already exist
debug(D_MAKEFLOW_RUN, "checking whether the umbrella log files already exist...\n");
cur = d->nodes;
while(cur) {
char *umbrella_logfile = NULL;
umbrella_logfile = string_format("%s.%d", w->log_prefix, cur->nodeid);
if(!access(umbrella_logfile, F_OK)) {
fprintf(stderr, "the umbrella log file for rule %d (`%s`) already exists!\n", cur->nodeid, umbrella_logfile);
exit(EXIT_FAILURE);
}
// add umbrella_logfile into the target files of a dag_node
if(remote_rename_support) {
dag_node_add_target_file(cur, umbrella_logfile, umbrella_logfile);
} else {
dag_node_add_target_file(cur, umbrella_logfile, NULL);
}
free(umbrella_logfile);
cur = cur->next;
}
if(!w->mode) {
w->mode = "local";
debug(D_MAKEFLOW_RUN, "setting wrapper_umbrella->mode to %s\n", w->mode);
}
}