static batch_job_id_t batch_job_amazon_batch_submit(struct batch_queue* q, const char* cmd, const char* extra_input_files, const char* extra_output_files, struct jx* envlist, const struct rmsummary* resources){
struct internal_amazon_batch_amazon_ids amazon_ids = initialize(q);
char* env_var = amazon_ids.master_env_prefix;
//so, we have the access keys, now we need to either set up the queues and exec environments, or add them.
unsigned int jobid = gen_guid();
char* job_name = string_format("%s_%u",queue_name,jobid);
//makeflow specifics
struct batch_job_info *info = malloc(sizeof(*info));
memset(info, 0, sizeof(*info));
//specs
int cpus=1;
long int mem=1000;
char* img = hash_table_lookup(q->options,"amazon-batch-img");
int disk = 1000;
if(resources){
cpus = resources->cores;
mem = resources->memory;
disk = resources->disk;
cpus = cpus > 1? cpus:1;
mem = mem > 1000? mem:1000;
disk = disk > 1000 ? disk : 1000;
}
//upload files to S3
upload_input_files_to_s3((char*)extra_input_files,job_name);
upload_cmd_file(bucket_name,(char*)extra_input_files,(char*)extra_output_files,(char*)cmd,jobid);
//create the fmd string to give to the command
char* fmt_cmd = string_format("%s aws s3 cp s3://%s/COMAND_FILE_%u.sh ./ && sh ./COMAND_FILE_%u.sh",env_var,bucket_name,jobid,jobid);
//combine all properties together
char* properties_string = string_format("{ \\\"image\\\": \\\"%s\\\", \\\"vcpus\\\": %i, \\\"memory\\\": %li, \\\"privileged\\\":true ,\\\"command\\\": [\\\"sh\\\",\\\"-c\\\",\\\"%s\\\"], \\\"environment\\\":[{\\\"name\\\":\\\"AWS_ACCESS_KEY_ID\\\",\\\"value\\\":\\\"%s\\\"},{\\\"name\\\":\\\"AWS_SECRET_ACCESS_KEY\\\",\\\"value\\\":\\\"%s\\\"},{\\\"name\\\":\\\"REGION\\\",\\\"value\\\":\\\"%s\\\"}] }", img,cpus,mem,fmt_cmd,amazon_ids.aws_access_key_id,amazon_ids.aws_secret_access_key,amazon_ids.aws_region);
char* jaid = aws_submit_job(job_name,properties_string);
itable_insert(amazon_job_ids,jobid,jaid);
debug(D_BATCH,"Job %u has amazon id: %s",jobid,jaid);
itable_insert(done_files,jobid,string_format("%s",extra_output_files));
debug(D_BATCH,"Job %u successfully Submitted",jobid);
//let makeflow know
info->submitted = time(0);
info->started = time(0);
itable_insert(q->job_table, jobid, info);
//cleanup
free(job_name);
free(fmt_cmd);
return jobid;
}
static int chirp_fuse_open(const char *path, struct fuse_file_info *fi)
{
static int file_number_counter = 1;
struct chirp_file *file;
int mode = 0;
char newpath[CHIRP_PATH_MAX];
char host[CHIRP_PATH_MAX];
parsepath(path, newpath, host);
pthread_mutex_lock(&mutex);
file = chirp_global_open(host, newpath, fi->flags, mode, time(0) + chirp_fuse_timeout);
if(file) {
int file_number = file_number_counter++;
itable_insert(file_table, file_number, file);
fi->fh = file_number;
}
pthread_mutex_unlock(&mutex);
if(!file)
return -errno;
return 0;
}
int histogram_insert(struct histogram *h, double value) {
uint64_t bucket = bucket_of(h, value);
struct box_count *box = itable_lookup(h->buckets, bucket);
if(!box) {
box = calloc(1, sizeof(*box));
itable_insert(h->buckets, bucket, box);
}
h->total_count++;
box->count++;
int mode_count = histogram_count(h, histogram_mode(h));
if(value > h->max_value || h->total_count < 1) {
h->max_value = value;
}
if(value < h->min_value || h->total_count < 1) {
h->min_value = value;
}
if(box->count > mode_count) {
h->mode = end_of(h, bucket);
}
return box->count;
}
INT64_T chirp_alloc_open(const char *path, INT64_T flags, INT64_T mode)
{
struct alloc_state *a;
int fd = -1;
if(!alloc_enabled)
return cfs->open(path, flags, mode);
a = alloc_state_cache(path);
if(a) {
INT64_T filesize = cfs_file_size(path);
if(filesize < 0)
filesize = 0;
fd = cfs->open(path, flags, mode);
if(fd >= 0) {
if(!fd_table)
fd_table = itable_create(0);
itable_insert(fd_table, fd, xstrdup(path));
if(flags & O_TRUNC) {
alloc_state_update(a, -space_consumed(filesize));
}
}
} else {
fd = -1;
}
return fd;
}
batch_job_id_t batch_job_submit_mpi_queue( struct batch_queue *q, const char *cmd, const char *args, const char *infile, const char *outfile, const char *errfile, const char *extra_input_files, const char *extra_output_files )
{
char *command = string_format("%s %s", cmd, args);
if(infile) {
char *new = string_format("%s <%s", infile);
free(command);
command = new;
}
struct mpi_queue_task *t = mpi_queue_task_create(command);
free(command);
if(infile)
mpi_queue_task_specify_file(t, infile, MPI_QUEUE_INPUT);
if(cmd)
mpi_queue_task_specify_file(t, cmd, MPI_QUEUE_INPUT);
specify_mpi_queue_task_files(t, extra_input_files, extra_output_files);
mpi_queue_submit(q->mpi_queue, t);
if(outfile) {
itable_insert(q->output_table, t->taskid, strdup(outfile));
}
return t->taskid;
}
void histogram_set_bucket(struct histogram *h, double value, int count) {
uint64_t bucket = bucket_of(h, value);
struct box_count *box = itable_lookup(h->buckets, bucket);
if(!box) {
box = calloc(1, sizeof(*box));
itable_insert(h->buckets, bucket, box);
}
}
static int dag_parse_node(struct lexer *bk)
{
struct token *t = lexer_next_token(bk);
if(t->type != TOKEN_FILES)
{
lexer_report_error(bk, "Error reading rule.");
}
lexer_free_token(t);
struct dag_node *n;
n = dag_node_create(bk->d, bk->line_number);
if(verbose_parsing && bk->d->nodeid_counter % parsing_rule_mod_counter == 0)
{
fprintf(stdout, "\rRules parsed: %d", bk->d->nodeid_counter + 1);
fflush(stdout);
}
n->category = bk->category;
list_push_tail(n->category->nodes, n);
dag_parse_node_filelist(bk, n);
bk->environment->node = n;
/* Read variables, if any */
while((t = lexer_peek_next_token(bk)) && t->type != TOKEN_COMMAND)
{
switch (t->type) {
case TOKEN_VARIABLE:
dag_parse_variable(bk, n);
break;
default:
lexer_report_error(bk, "Expected COMMAND or VARIABLE, got: %s", lexer_print_token(t));
break;
}
}
if(!t)
{
lexer_report_error(bk, "Rule does not have a command.\n");
}
dag_parse_node_command(bk, n);
bk->environment->node = NULL;
n->next = bk->d->nodes;
bk->d->nodes = n;
itable_insert(bk->d->node_table, n->nodeid, n);
debug(D_MAKEFLOW_PARSER, "Setting resource category '%s' for rule %d.\n", n->category->label, n->nodeid);
dag_node_fill_resources(n);
dag_node_print_debug_resources(n);
return 1;
}
void histogram_attach_data(struct histogram *h, double value, void *data) {
uint64_t bucket = bucket_of(h, value);
struct box_count *box = itable_lookup(h->buckets, bucket);
if(!box) {
box = calloc(1, sizeof(*box));
itable_insert(h->buckets, bucket, box);
}
box->data = data;
}
static int submit_workers( struct batch_queue *queue, struct itable *job_table, int count )
{
int i;
for(i=0;i<count;i++) {
int jobid = submit_worker(queue);
if(jobid>0) {
debug(D_WQ,"worker job %d submitted",jobid);
itable_insert(job_table,jobid,(void*)1);
} else {
break;
}
}
return i;
}
static batch_job_id_t batch_job_dryrun_submit (struct batch_queue *q, const char *cmd, const char *extra_input_files, const char *extra_output_files, struct jx *envlist, const struct rmsummary *resources )
{
FILE *log;
char *escaped_cmd;
char *env_assignment;
char *escaped_env_assignment;
struct batch_job_info *info;
batch_job_id_t jobid = random();
fflush(NULL);
debug(D_BATCH, "started dry run of job %" PRIbjid ": %s", jobid, cmd);
if ((log = fopen(q->logfile, "a"))) {
if (!(info = calloc(sizeof(*info), 1))) {
fclose(log);
return -1;
}
info->submitted = time(0);
info->started = time(0);
itable_insert(q->job_table, jobid, info);
if(envlist && jx_istype(envlist, JX_OBJECT) && envlist->u.pairs) {
struct jx_pair *p;
fprintf(log, "env ");
for(p=envlist->u.pairs;p;p=p->next) {
if(p->key->type==JX_STRING && p->value->type==JX_STRING) {
env_assignment = string_format("%s=%s", p->key->u.string_value,p->value->u.string_value);
escaped_env_assignment = string_escape_shell(env_assignment);
fprintf(log, "%s", escaped_env_assignment);
fprintf(log, " ");
free(env_assignment);
free(escaped_env_assignment);
}
}
}
escaped_cmd = string_escape_shell(cmd);
fprintf(log, "sh -c %s\n", escaped_cmd);
free(escaped_cmd);
fclose(log);
return jobid;
} else {
return -1;
}
}
static int batch_job_amazon_batch_remove(struct batch_queue *q, batch_job_id_t jobid){
struct internal_amazon_batch_amazon_ids amazon_ids = initialize(q);
char* env_var = amazon_ids.master_env_prefix;
if(itable_lookup(done_jobs,jobid)==NULL){
char* name = string_format("%s_%i",queue_name,(int)jobid);
itable_insert(done_jobs,jobid+1,name);
}
char* amazon_id;
if((amazon_id=itable_lookup(amazon_job_ids,jobid))==NULL){
return -1;
}
char* cmd = string_format("%s aws batch terminate-job --job-id %s --reason \"Makeflow Killed\"",env_var,amazon_id);
debug(D_BATCH,"Terminating the job: %s\n",cmd);
sh_system(cmd);
free(cmd);
return 0;
}
static int itable_double_buckets(struct itable *h)
{
struct itable *hn = itable_create(2 * h->bucket_count);
if(!hn)
return 0;
/* Move pairs to new hash */
uint64_t key;
void *value;
itable_firstkey(h);
while(itable_nextkey(h, &key, &value))
if(!itable_insert(hn, key, value))
{
itable_delete(hn);
return 0;
}
/* Delete all old pairs */
struct entry *e, *f;
int i;
for(i = 0; i < h->bucket_count; i++) {
e = h->buckets[i];
while(e) {
f = e->next;
free(e);
e = f;
}
}
/* Make the old point to the new */
free(h->buckets);
h->buckets = hn->buckets;
h->bucket_count = hn->bucket_count;
h->size = hn->size;
/* Delete reference to new, so old is safe */
free(hn);
return 1;
}
batch_job_id_t batch_job_submit_simple_local(struct batch_queue *q, const char *cmd, const char *extra_input_files, const char *extra_output_files)
{
batch_job_id_t jobid;
fflush(NULL);
jobid = fork();
if(jobid > 0) {
debug(D_BATCH, "started process %d: %s", jobid, cmd);
struct batch_job_info *info = malloc(sizeof(*info));
memset(info, 0, sizeof(*info));
info->submitted = time(0);
info->started = time(0);
itable_insert(q->job_table, jobid, info);
return jobid;
} else if(jobid < 0) {
debug(D_BATCH, "couldn't create new process: %s\n", strerror(errno));
return -1;
} else {
/** The following code works but would duplicates the current process because of the system() function.
int result = system(cmd);
if(WIFEXITED(result)) {
_exit(WEXITSTATUS(result));
} else {
_exit(1);
}*/
/** A note from "man system 3" as of Jan 2012:
* Do not use system() from a program with set-user-ID or set-group-ID
* privileges, because strange values for some environment variables
* might be used to subvert system integrity. Use the exec(3) family of
* functions instead, but not execlp(3) or execvp(3). system() will
* not, in fact, work properly from programs with set-user-ID or
* set-group-ID privileges on systems on which /bin/sh is bash version
* 2, since bash 2 drops privileges on startup. (Debian uses a modified
* bash which does not do this when invoked as sh.)
*/
execlp("sh", "sh", "-c", cmd, (char *) 0);
_exit(127); // Failed to execute the cmd.
}
return -1;
}
struct list *makeflow_wrapper_generate_files( struct list *result, struct list *input, struct dag_node *n, struct makeflow_wrapper *w)
{
char *f;
char *nodeid = string_format("%d",n->nodeid);
struct list *files = list_create();
list_first_item(input);
while((f = list_next_item(input)))
{
char *filename = string_replace_percents(f, nodeid);
char *f = xxstrdup(filename);
free(filename);
char *remote, *p;
struct dag_file *file;
p = strchr(f, '=');
if(p) {
*p = 0;
file = dag_file_lookup_or_create(n->d, f);
if(!n->local_job && !itable_lookup(w->remote_names, (uintptr_t) file)){
remote = xxstrdup(p+1);
itable_insert(w->remote_names, (uintptr_t) file, (void *)remote);
hash_table_insert(w->remote_names_inv, remote, (void *)file);
}
*p = '=';
} else {
file = dag_file_lookup_or_create(n->d, f);
}
free(f);
list_push_tail(files, file);
}
free(nodeid);
result = list_splice(result, files);
return result;
}
/* Adds remotename to the local name filename in the namespace of
* the given node. If remotename is NULL, then a new name is
* found using dag_node_translate_filename. If the remotename
* given is different from a previosly specified, a warning is
* written to the debug output, but otherwise this is ignored. */
const char *dag_node_add_remote_name(struct dag_node *n, const char *filename, const char *remotename)
{
char *oldname;
struct dag_file *f = dag_file_from_name(n->d, filename);
if(!f)
fatal("trying to add remote name %s to unknown file %s.\n", remotename, filename);
if(!remotename)
remotename = dag_node_translate_filename(n, filename);
else
remotename = xxstrdup(remotename);
oldname = hash_table_lookup(n->remote_names_inv, remotename);
if(oldname && strcmp(oldname, filename) == 0)
debug(D_DEBUG, "Remote name %s for %s already in use for %s\n", remotename, filename, oldname);
itable_insert(n->remote_names, (uintptr_t) f, remotename);
hash_table_insert(n->remote_names_inv, remotename, (void *) f);
return remotename;
}
struct cluster *nearest_neighbor_clustering(struct list *initial_clusters, double (*cmp)(struct cluster *, struct cluster *))
{
struct cluster *top, *closest, *subtop;
struct list *stack;
struct itable *active_clusters;
double dclosest, dsubtop;
int merge = 0;
list_first_item(initial_clusters);
top = list_next_item(initial_clusters);
/* Return immediately if top is NULL, or there is a unique
* initial cluster */
if(list_size(initial_clusters) < 2)
return top;
stack = list_create(0);
list_push_head(stack, top);
/* Add all of the initial clusters as active clusters. */
active_clusters = itable_create(0);
while( (top = list_next_item(initial_clusters)) )
itable_insert(active_clusters, (uintptr_t) top, (void *) top);
do
{
/* closest might be NULL if all of the clusters are in
* the stack now. subtop might be NULL if top was the
* only cluster in the stack */
top = list_pop_head( stack );
closest = cluster_nearest_neighbor(active_clusters, top, cmp);
subtop = list_peek_head( stack );
dclosest = -1;
dsubtop = -1;
if(closest)
dclosest = cluster_ward_distance(top, closest);
if(subtop)
dsubtop = cluster_ward_distance(top, subtop);
/* The nearest neighbor of top is either one of the
* remaining active clusters, or the second topmost
* cluster in the stack */
if( closest && subtop )
{
/* Use pointer address to systematically break ties. */
if(dclosest < dsubtop || ((dclosest == dsubtop) && (uintptr_t)closest < (uintptr_t)subtop))
merge = 0;
else
merge = 1;
}
else if( subtop )
merge = 1;
else if( closest )
merge = 0;
else
fatal("Zero clusters?\n"); //We should never reach here.
if(merge)
{
/* If the two topmost clusters in the stack are
* mutual nearest neighbors, merge them into a single
* cluster */
subtop = list_pop_head( stack );
list_push_head(stack, cluster_merge(top, subtop));
}
else
{
/* Otherwise, push the nearest neighbor of top to the
* stack */
itable_remove(active_clusters, (uintptr_t) closest);
list_push_head(stack, top);
list_push_head(stack, closest);
}
debug(D_DEBUG, "stack: %d active: %d closest: %lf subtop: %lf\n",
list_size(stack), itable_size(active_clusters), dclosest, dsubtop);
/* If there are no more active_clusters, but there is not
* a single cluster in the stack, we try again,
* converting the clusters in the stack into new active
* clusters. */
if(itable_size(active_clusters) == 0 && list_size(stack) > 3)
{
itable_delete(active_clusters);
return nearest_neighbor_clustering(stack, cmp);
}
}while( !(itable_size(active_clusters) == 0 && list_size(stack) == 1) );
/* top is now the root of a cluster hierarchy, of
* cluster->right, cluster->left. */
top = list_pop_head(stack);
list_delete(stack);
itable_delete(active_clusters);
return top;
}
int dag_parse_node(struct lexer_book *bk, char *line_org)
{
struct dag *d = bk->d;
char *line;
char *outputs = NULL;
char *inputs = NULL;
struct dag_node *n;
n = dag_node_create(bk->d, bk->line_number);
n->category = bk->category;
list_push_tail(n->category->nodes, n);
line = xxstrdup(line_org);
outputs = line;
inputs = strchr(line, ':');
*inputs = 0;
inputs = inputs + 1;
inputs = string_trim_spaces(inputs);
outputs = string_trim_spaces(outputs);
dag_parse_node_filelist(bk, n, outputs, 0);
dag_parse_node_filelist(bk, n, inputs, 1);
int ok;
char *comment;
//parse variables and comments
while((line = dag_parse_readline(bk, n)) != NULL) {
if(line[0] == '@' && strchr(line, '=')) {
ok = dag_parse_variable(bk, n, line);
free(line);
if(ok) {
continue;
} else {
dag_parse_error(bk, "node variable");
free(line);
return 0;
}
}
comment = strchr(line, '#');
if(comment)
{
*comment = '\0';
int n = strspn(line, " \t");
int m = strlen(line);
*comment = '#';
/* make sure that only spaces and tabs appear before the hash */
if(n == m) {
continue;
}
}
/* not a comment or a variable, so we break to parse the command */
break;
}
ok = dag_parse_node_command(bk, n, line);
free(line);
if(ok) {
n->next = d->nodes;
d->nodes = n;
itable_insert(d->node_table, n->nodeid, n);
} else {
dag_parse_error(bk, "node command");
return 0;
}
debug(D_DEBUG, "Setting resource category '%s' for rule %d.\n", n->category->label, n->nodeid);
dag_task_fill_resources(n);
dag_task_print_debug_resources(n);
return 1;
}
static batch_job_id_t batch_job_condor_submit (struct batch_queue *q, const char *cmd, const char *extra_input_files, const char *extra_output_files, struct jx *envlist, const struct rmsummary *resources )
{
FILE *file;
int njobs;
int jobid;
const char *options = hash_table_lookup(q->options, "batch-options");
if(setup_condor_wrapper("condor.sh") < 0) {
debug(D_BATCH, "could not create condor.sh: %s", strerror(errno));
return -1;
}
if(!string_istrue(hash_table_lookup(q->options, "skip-afs-check"))) {
char *cwd = path_getcwd();
if(!strncmp(cwd, "/afs", 4)) {
debug(D_NOTICE|D_BATCH, "The working directory is '%s':", cwd);
debug(D_NOTICE|D_BATCH, "This won't work because Condor is not able to write to files in AFS.");
debug(D_NOTICE|D_BATCH, "Instead, run makeflow from a local disk like /tmp.");
debug(D_NOTICE|D_BATCH, "Or, use the Work Queue with -T wq and condor_submit_workers.");
free(cwd);
exit(EXIT_FAILURE);
}
free(cwd);
}
file = fopen("condor.submit", "w");
if(!file) {
debug(D_BATCH, "could not create condor.submit: %s", strerror(errno));
return -1;
}
fprintf(file, "universe = vanilla\n");
fprintf(file, "executable = condor.sh\n");
char *escaped = string_escape_condor(cmd);
fprintf(file, "arguments = %s\n", escaped);
free(escaped);
if(extra_input_files)
fprintf(file, "transfer_input_files = %s\n", extra_input_files);
// Note that we do not use transfer_output_files, because that causes the job
// to get stuck in a system hold if the files are not created.
fprintf(file, "should_transfer_files = yes\n");
fprintf(file, "when_to_transfer_output = on_exit\n");
fprintf(file, "notification = never\n");
fprintf(file, "copy_to_spool = true\n");
fprintf(file, "transfer_executable = true\n");
fprintf(file, "keep_claim_idle = 30\n");
fprintf(file, "log = %s\n", q->logfile);
const char *c_req = batch_queue_get_option(q, "condor-requirements");
char *bexp = blacklisted_expression(q);
if(c_req && bexp) {
fprintf(file, "requirements = %s && %s\n", c_req, bexp);
} else if(c_req) {
fprintf(file, "requirements = %s\n", c_req);
} else if(bexp) {
fprintf(file, "requirements = %s\n", bexp);
}
if(bexp)
free(bexp);
/*
Getting environment variables formatted for a condor submit
file is very hairy, due to some strange quoting rules.
To avoid problems, we simply export vars to the environment,
and then tell condor getenv=true, which pulls in the environment.
*/
fprintf(file, "getenv = true\n");
if(envlist) {
jx_export(envlist);
}
if(options)
fprintf(file, "%s\n", options);
/* set same deafults as condor_submit_workers */
int64_t cores = 1;
int64_t memory = 1024;
int64_t disk = 1024;
if(resources) {
cores = resources->cores > -1 ? resources->cores : cores;
memory = resources->memory > -1 ? resources->memory : memory;
disk = resources->disk > -1 ? resources->disk : disk;
}
/* convert disk to KB */
disk *= 1024;
if(batch_queue_get_option(q, "autosize")) {
fprintf(file, "request_cpus = ifThenElse(%" PRId64 " > TotalSlotCpus, %" PRId64 ", TotalSlotCpus)\n", cores, cores);
fprintf(file, "request_memory = ifThenElse(%" PRId64 " > TotalSlotMemory, %" PRId64 ", TotalSlotMemory)\n", memory, memory);
fprintf(file, "request_disk = ifThenElse((%" PRId64 ") > TotalSlotDisk, (%" PRId64 "), TotalSlotDisk)\n", disk, disk);
}
else {
fprintf(file, "request_cpus = %" PRId64 "\n", cores);
fprintf(file, "request_memory = %" PRId64 "\n", memory);
fprintf(file, "request_disk = %" PRId64 "\n", disk);
}
fprintf(file, "queue\n");
fclose(file);
file = popen("condor_submit condor.submit", "r");
if(!file)
return -1;
char line[BATCH_JOB_LINE_MAX];
while(fgets(line, sizeof(line), file)) {
if(sscanf(line, "%d job(s) submitted to cluster %d", &njobs, &jobid) == 2) {
pclose(file);
debug(D_BATCH, "job %d submitted to condor", jobid);
struct batch_job_info *info;
info = malloc(sizeof(*info));
memset(info, 0, sizeof(*info));
info->submitted = time(0);
itable_insert(q->job_table, jobid, info);
return jobid;
}
}
pclose(file);
debug(D_BATCH, "failed to submit job to condor!");
return -1;
}
static batch_job_id_t batch_job_condor_wait (struct batch_queue * q, struct batch_job_info * info_out, time_t stoptime)
{
static FILE *logfile = 0;
if(!logfile) {
logfile = fopen(q->logfile, "r");
if(!logfile) {
debug(D_NOTICE, "couldn't open logfile %s: %s\n", q->logfile, strerror(errno));
return -1;
}
}
while(1) {
/*
Note: clearerr is necessary to clear any cached end-of-file condition,
otherwise some implementations of fgets (i.e. darwin) will read to end
of file once and then never look for any more data.
*/
clearerr(logfile);
char line[BATCH_JOB_LINE_MAX];
while(fgets(line, sizeof(line), logfile)) {
int type, proc, subproc;
batch_job_id_t jobid;
time_t current;
struct tm tm;
struct batch_job_info *info;
int logcode, exitcode;
if(sscanf(line, "%d (%" SCNbjid ".%d.%d) %d/%d %d:%d:%d", &type, &jobid, &proc, &subproc, &tm.tm_mon, &tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec) == 9) {
tm.tm_year = 2008 - 1900;
tm.tm_isdst = 0;
current = mktime(&tm);
info = itable_lookup(q->job_table, jobid);
if(!info) {
info = malloc(sizeof(*info));
memset(info, 0, sizeof(*info));
itable_insert(q->job_table, jobid, info);
}
debug(D_BATCH, "line: %s", line);
if(type == 0) {
info->submitted = current;
} else if(type == 1) {
info->started = current;
debug(D_BATCH, "job %" PRIbjid " running now", jobid);
} else if(type == 9) {
itable_remove(q->job_table, jobid);
info->finished = current;
info->exited_normally = 0;
info->exit_signal = SIGKILL;
debug(D_BATCH, "job %" PRIbjid " was removed", jobid);
memcpy(info_out, info, sizeof(*info));
free(info);
return jobid;
} else if(type == 5) {
itable_remove(q->job_table, jobid);
info->finished = current;
fgets(line, sizeof(line), logfile);
if(sscanf(line, " (%d) Normal termination (return value %d)", &logcode, &exitcode) == 2) {
debug(D_BATCH, "job %" PRIbjid " completed normally with status %d.", jobid, exitcode);
info->exited_normally = 1;
info->exit_code = exitcode;
} else if(sscanf(line, " (%d) Abnormal termination (signal %d)", &logcode, &exitcode) == 2) {
debug(D_BATCH, "job %" PRIbjid " completed abnormally with signal %d.", jobid, exitcode);
info->exited_normally = 0;
info->exit_signal = exitcode;
} else {
debug(D_BATCH, "job %" PRIbjid " completed with unknown status.", jobid);
info->exited_normally = 0;
info->exit_signal = 0;
}
memcpy(info_out, info, sizeof(*info));
free(info);
return jobid;
}
}
}
if(itable_size(q->job_table) <= 0)
return 0;
if(stoptime != 0 && time(0) >= stoptime)
return -1;
if(process_pending())
return -1;
sleep(1);
}
return -1;
}
int main( int argc, char *argv[] )
{
signed char c;
const char *progname = "wavefront";
debug_config(progname);
progress_log_file = stdout;
struct option long_options[] = {
{"help", no_argument, 0, 'h'},
{"version", no_argument, 0, 'v'},
{"debug", required_argument, 0, 'd'},
{"jobs", required_argument, 0, 'n'},
{"block-size", required_argument, 0, 'b'},
{"debug-file", required_argument, 0, 'o'},
{"log-file", required_argument, 0, 'l'},
{"bitmap", required_argument, 0, 'B'},
{"bitmap-interval", required_argument, 0, 'i'},
{"auto", no_argument, 0, 'A'},
{"local", no_argument, 0, 'L'},
{"batch-type", required_argument, 0, 'T'},
{"verify", no_argument, 0, 'V'},
{0,0,0,0}
};
while((c=getopt_long(argc,argv,"n:b:d:o:l:B:i:qALDT:VX:Y:vh", long_options, NULL)) > -1) {
switch(c) {
case 'n':
manual_max_jobs_running = atoi(optarg);
break;
case 'b':
manual_block_size = atoi(optarg);
break;
case 'd':
debug_flags_set(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'B':
progress_bitmap_file = optarg;
break;
case 'i':
progress_bitmap_interval = atoi(optarg);
break;
case 'l':
progress_log_file = fopen(optarg,"w");
if(!progress_log_file) {
fprintf(stderr,"couldn't open %s: %s\n",optarg,strerror(errno));
return 1;
}
break;
case 'A':
wavefront_mode = WAVEFRONT_MODE_AUTO;
break;
case 'L':
wavefront_mode = WAVEFRONT_MODE_MULTICORE;
break;
case 'T':
wavefront_mode = WAVEFRONT_MODE_DISTRIBUTED;
batch_system_type = batch_queue_type_from_string(optarg);
if(batch_system_type==BATCH_QUEUE_TYPE_UNKNOWN) {
fprintf(stderr,"unknown batch system type: %s\n",optarg);
exit(1);
}
break;
case 'V':
verify_mode = 1;
break;
case 'X':
xstart = atoi(optarg);
break;
case 'Y':
ystart = atoi(optarg);
break;
case 'v':
cctools_version_print(stdout, progname);
exit(0);
break;
case 'h':
show_help(progname);
exit(0);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if( (argc-optind<3) ) {
show_help(progname);
exit(1);
}
function = argv[optind];
xsize=atoi(argv[optind+1]);
ysize=atoi(argv[optind+2]);
total_cells = xsize*ysize;
if(!verify_mode && !check_configuration(function,xsize,ysize)) exit(1);
int ncpus = load_average_get_cpus();
if(wavefront_mode!=WAVEFRONT_MODE_MULTICORE) {
double task_time = measure_task_time();
printf("Each function takes %.02lfs to run.\n",task_time);
block_size = find_best_block_size(xsize,1000,2,task_time,average_dispatch_time);
double distributed_time = wavefront_distributed_model(xsize,1000,2,task_time,block_size,average_dispatch_time);
double multicore_time = wavefront_multicore_model(xsize,ncpus,task_time);
double ideal_multicore_time = wavefront_multicore_model(xsize,xsize,task_time);
double sequential_time = wavefront_multicore_model(xsize,1,task_time);
printf("---------------------------------\n");
printf("This workload would take:\n");
printf("%.02lfs sequentially\n",sequential_time);
printf("%.02lfs on this %d-core machine\n",multicore_time,ncpus);
printf("%.02lfs on a %d-core machine\n",ideal_multicore_time,xsize);
printf("%.02lfs on a 1000-node distributed system with block size %d\n",distributed_time,block_size);
printf("---------------------------------\n");
if(wavefront_mode==WAVEFRONT_MODE_AUTO) {
if(multicore_time < distributed_time*2) {
wavefront_mode = WAVEFRONT_MODE_MULTICORE;
} else {
wavefront_mode = WAVEFRONT_MODE_DISTRIBUTED;
}
}
}
if(wavefront_mode==WAVEFRONT_MODE_MULTICORE) {
batch_system_type = BATCH_QUEUE_TYPE_LOCAL;
max_jobs_running = ncpus;
} else {
max_jobs_running = 1000;
}
if(manual_block_size!=0) {
block_size = manual_block_size;
}
if(manual_max_jobs_running!=0) {
max_jobs_running = manual_max_jobs_running;
}
if(wavefront_mode==WAVEFRONT_MODE_MULTICORE) {
printf("Running in multicore mode with %d CPUs.\n",max_jobs_running);
} else {
printf("Running in distributed mode with block size %d on up to %d CPUs\n",block_size,max_jobs_running);
}
batch_q = batch_queue_create(batch_system_type);
if(verify_mode) exit(0);
struct bitmap * b = bitmap_create(xsize+1,ysize+1);
struct list *ready_list = list_create();
struct itable *running_table = itable_create(0);
struct batch_job_info info;
UINT64_T jobid;
struct wavefront_task *task;
wavefront_task_initialize(b,ready_list);
printf("Starting workload...\n");
fprintf(progress_log_file,"# elapsed time : waiting jobs / running jobs / cells complete (percent complete)\n");
while(1) {
if(abort_mode) {
while((task=list_pop_tail(ready_list))) {
wavefront_task_delete(task);
}
itable_firstkey(running_table);
while(itable_nextkey(running_table,&jobid,(void**)&task)) {
batch_job_remove(batch_q,jobid);
}
}
if(list_size(ready_list)==0 && itable_size(running_table)==0) break;
while(1) {
if(itable_size(running_table)>=max_jobs_running) break;
task = list_pop_tail(ready_list);
if(!task) break;
jobid = wavefront_task_submit(task);
if(jobid>0) {
itable_insert(running_table,jobid,task);
wavefront_task_mark_range(task,b,WAVEFRONT_TASK_STATE_RUNNING);
} else {
abort();
sleep(1);
list_push_head(ready_list,task);
}
}
save_status(b,ready_list,running_table);
jobid = batch_job_wait(batch_q,&info);
if(jobid>0) {
task = itable_remove(running_table,jobid);
if(task) {
if(info.exited_normally && info.exit_code==0) {
total_dispatch_time += info.started-info.submitted;
total_execute_time += MAX(info.finished-info.started,1);
total_cells_complete+=task->width*task->height;
total_jobs_complete++;
average_dispatch_time = 1.0*total_dispatch_time / total_jobs_complete;
average_task_time = 1.0*total_execute_time / total_cells_complete;
wavefront_task_complete(b,ready_list,task);
} else {
printf("job %" PRIu64 " failed, aborting this workload\n",jobid);
abort_mode = 1;
}
}
}
}
save_status(b,ready_list,running_table);
if(abort_mode) {
printf("Workload was aborted.\n");
} else {
printf("Workload complete.\n");
}
return 0;
}
int master_main(const char *host, int port, const char *addr) {
time_t idle_stoptime;
struct link *master = NULL;
int num_workers, i;
struct mpi_queue_job **workers;
struct itable *active_jobs = itable_create(0);
struct itable *waiting_jobs = itable_create(0);
struct list *complete_jobs = list_create();
MPI_Comm_size(MPI_COMM_WORLD, &num_workers);
workers = malloc(num_workers * sizeof(*workers));
memset(workers, 0, num_workers * sizeof(*workers));
idle_stoptime = time(0) + idle_timeout;
while(!abort_flag) {
char line[MPI_QUEUE_LINE_MAX];
if(time(0) > idle_stoptime) {
if(master) {
printf("mpi master: gave up after waiting %ds to receive a task.\n", idle_timeout);
} else {
printf("mpi master: gave up after waiting %ds to connect to %s port %d.\n", idle_timeout, host, port);
}
break;
}
if(!master) {
char working_dir[MPI_QUEUE_LINE_MAX];
master = link_connect(addr, port, idle_stoptime);
if(!master) {
sleep(5);
continue;
}
link_tune(master, LINK_TUNE_INTERACTIVE);
link_readline(master, line, sizeof(line), time(0) + active_timeout);
memset(working_dir, 0, MPI_QUEUE_LINE_MAX);
if(sscanf(line, "workdir %s", working_dir) == 1) {
MPI_Bcast(working_dir, MPI_QUEUE_LINE_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
link_close(master);
master = NULL;
continue;
}
}
if(link_readline(master, line, sizeof(line), time(0) + short_timeout)) {
struct mpi_queue_operation *op;
int jobid, mode;
INT64_T length;
char path[MPI_QUEUE_LINE_MAX];
op = NULL;
debug(D_MPI, "received: %s\n", line);
if(!strcmp(line, "get results")) {
struct mpi_queue_job *job;
debug(D_MPI, "results requested: %d available\n", list_size(complete_jobs));
link_putfstring(master, "num results %d\n", time(0) + active_timeout, list_size(complete_jobs));
while(list_size(complete_jobs)) {
job = list_pop_head(complete_jobs);
link_putfstring(master, "result %d %d %d %lld\n", time(0) + active_timeout, job->jobid, job->status, job->result, job->output_length);
if(job->output_length) {
link_write(master, job->output, job->output_length, time(0)+active_timeout);
}
mpi_queue_job_delete(job);
}
} else if(sscanf(line, "work %d %lld", &jobid, &length)) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_WORK;
op->buffer_length = length+1;
op->buffer = malloc(length+1);
op->buffer[op->buffer_length] = 0;
link_read(master, op->buffer, length, time(0) + active_timeout);
op->result = -1;
} else if(sscanf(line, "stat %d %s", &jobid, path) == 2) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_STAT;
sprintf(op->args, "%s", path);
op->result = -1;
} else if(sscanf(line, "unlink %d %s", &jobid, path) == 2) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_UNLINK;
sprintf(op->args, "%s", path);
op->result = -1;
} else if(sscanf(line, "mkdir %d %s %o", &jobid, path, &mode) == 3) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_MKDIR;
sprintf(op->args, "%s %o", path, mode);
op->result = -1;
} else if(sscanf(line, "close %d", &jobid) == 1) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_CLOSE;
op->result = -1;
// } else if(sscanf(line, "symlink %d %s %s", &jobid, path, filename) == 3) {
// } else if(sscanf(line, "put %d %s %lld %o", &jobid, filename, &length, &mode) == 4) {
// } else if(sscanf(line, "rget %d %s", &jobid, filename) == 2) {
// } else if(sscanf(line, "get %d %s", &jobid, filename) == 2) {
// } else if(sscanf(line, "thirdget %d %d %s %[^\n]", &jobid, &mode, filename, path) == 4) {
// } else if(sscanf(line, "thirdput %d %d %s %[^\n]", &jobid, &mode, filename, path) == 4) {
} else if(!strcmp(line, "exit")) {
break;
} else {
abort_flag = 1;
continue;
}
if(op) {
struct mpi_queue_job *job;
job = itable_lookup(active_jobs, jobid);
if(!job) {
job = itable_lookup(waiting_jobs, jobid);
}
if(!job) {
job = malloc(sizeof(*job));
memset(job, 0, sizeof(*job));
job->jobid = jobid;
job->operations = list_create();
job->status = MPI_QUEUE_JOB_WAITING;
job->worker_rank = -1;
itable_insert(waiting_jobs, jobid, job);
}
list_push_tail(job->operations, op);
}
idle_stoptime = time(0) + idle_timeout;
} else {
link_close(master);
master = 0;
sleep(5);
}
int num_waiting_jobs = itable_size(waiting_jobs);
int num_unvisited_jobs = itable_size(active_jobs);
for(i = 1; i < num_workers && (num_unvisited_jobs > 0 || num_waiting_jobs > 0); i++) {
struct mpi_queue_job *job;
struct mpi_queue_operation *op;
int flag = 0;
UINT64_T jobid;
if(!workers[i]) {
if(num_waiting_jobs) {
itable_firstkey(waiting_jobs);
itable_nextkey(waiting_jobs, &jobid, (void **)&job);
itable_remove(waiting_jobs, jobid);
itable_insert(active_jobs, jobid, job);
workers[i] = job;
num_waiting_jobs--;
job->worker_rank = i;
job->status = MPI_QUEUE_JOB_READY;
} else {
continue;
}
} else {
num_unvisited_jobs--;
if(workers[i]->status == MPI_QUEUE_JOB_BUSY) {
MPI_Test(&workers[i]->request, &flag, &workers[i]->mpi_status);
if(flag) {
op = list_pop_head(workers[i]->operations);
if(op->output_length) {
op->output_buffer = malloc(op->output_length);
MPI_Recv(op->output_buffer, op->output_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->mpi_status);
}
workers[i]->status = MPI_QUEUE_JOB_READY;
if(op->type == MPI_QUEUE_OP_WORK || op->result < 0) {
if(workers[i]->output)
free(workers[i]->output);
workers[i]->output = op->output_buffer;
op->output_buffer = NULL;
workers[i]->output_length = op->output_length;
workers[i]->result = op->result;
if(op->result < 0) {
workers[i]->status = MPI_QUEUE_JOB_FAILED | op->type;
op->type = MPI_QUEUE_OP_CLOSE;
list_push_head(workers[i]->operations, op);
op = NULL;
}
}
if(op) {
if(op->buffer)
free(op->buffer);
if(op->output_buffer)
free(op->output_buffer);
free(op);
}
}
}
}
if( workers[i]->status != MPI_QUEUE_JOB_BUSY && list_size(workers[i]->operations)) {
op = list_peek_head(workers[i]->operations);
if(op->type == MPI_QUEUE_OP_CLOSE) {
itable_remove(active_jobs, workers[i]->jobid);
list_push_tail(complete_jobs, workers[i]);
if(!(workers[i]->status & MPI_QUEUE_JOB_FAILED))
workers[i]->status = MPI_QUEUE_JOB_COMPLETE;
workers[i] = NULL;
i--;
continue;
}
MPI_Send(op, sizeof(*op), MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD);
if(op->buffer_length) {
MPI_Send(op->buffer, op->buffer_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD);
free(op->buffer);
op->buffer_length = 0;
op->buffer = NULL;
}
MPI_Irecv(op, sizeof(*op), MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->request);
workers[i]->status = MPI_QUEUE_JOB_BUSY;
}
}
}
/** Clean up waiting & complete jobs, send Exit commands to each worker */
if(!master) {
// If the master link hasn't been set up yet
// the workers will be waiting for the working directory
char line[MPI_QUEUE_LINE_MAX];
memset(line, 0, MPI_QUEUE_LINE_MAX);
MPI_Bcast(line, MPI_QUEUE_LINE_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
link_close(master);
}
for(i = 1; i < num_workers; i++) {
struct mpi_queue_operation *op, close;
memset(&close, 0, sizeof(close));
close.type = MPI_QUEUE_OP_EXIT;
if(workers[i]) {
if(workers[i]->status == MPI_QUEUE_JOB_BUSY) {
MPI_Wait(&workers[i]->request, &workers[i]->mpi_status);
op = list_peek_head(workers[i]->operations);
if(op->output_length) {
op->output_buffer = malloc(op->output_length);
MPI_Recv(op->output_buffer, op->output_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->mpi_status);
}
}
itable_remove(active_jobs, workers[i]->jobid);
list_push_tail(complete_jobs, workers[i]);
}
MPI_Send(&close, sizeof(close), MPI_BYTE, i, 0, MPI_COMM_WORLD);
}
itable_firstkey(waiting_jobs);
while(itable_size(waiting_jobs)) {
struct mpi_queue_job *job;
UINT64_T jobid;
itable_nextkey(waiting_jobs, &jobid, (void **)&job);
itable_remove(waiting_jobs, jobid);
list_push_tail(complete_jobs, job);
}
while(list_size(complete_jobs)) {
mpi_queue_job_delete(list_pop_head(complete_jobs));
}
MPI_Finalize();
return abort_flag;
}
static batch_job_id_t batch_job_mesos_submit (struct batch_queue *q, const char *cmd,
const char *extra_input_files, const char *extra_output_files,
struct jx *envlist, const struct rmsummary *resources )
{
// Get the path to mesos python site-packages
if (!is_mesos_py_path_known) {
mesos_py_path = batch_queue_get_option(q, "mesos-path");
if (mesos_py_path != NULL) {
debug(D_INFO, "Get mesos_path %s from command line\n", mesos_py_path);
}
is_mesos_py_path_known = 1;
}
// Get the mesos master address
if (!is_mesos_master_known) {
mesos_master = batch_queue_get_option(q, "mesos-master");
if (mesos_master == NULL) {
fatal("Please specify the hostname of mesos master by using --mesos-master");
} else {
debug(D_INFO, "Get mesos_path %s from command line\n", mesos_py_path);
is_mesos_master_known = 1;
}
}
mesos_preload = batch_queue_get_option(q, "mesos-preload");
if (is_mesos_py_path_known &&
is_mesos_master_known &&
!is_scheduler_running ) {
// start mesos scheduler if it is not running
start_mesos_scheduler(q);
is_scheduler_running = 1;
}
int task_id = ++counter;
debug(D_BATCH, "task %d is ready", task_id);
struct batch_job_info *info = calloc(1, sizeof(*info));
info->started = time(0);
info->submitted = time(0);
itable_insert(q->job_table, task_id, info);
// write the ready task information as
// "task_id, task_cmd, inputs, outputs" to
// mesos_task_info, which will be scanned by
// mf_mesos_scheduler later.
FILE *task_info_fp;
if(access(FILE_TASK_INFO, F_OK) != -1) {
task_info_fp = fopen(FILE_TASK_INFO, "a+");
} else {
task_info_fp = fopen(FILE_TASK_INFO, "w+");
}
struct mesos_task *mt = mesos_task_create(task_id, cmd, extra_input_files, extra_output_files);
fprintf(task_info_fp, "%d,%s,", mt->task_id, mt->task_cmd);
if (extra_input_files != NULL && strlen(extra_input_files) != 0) {
int j = 0;
int num_input_files = text_list_size(mt->task_input_files);
for (j = 0; j < (num_input_files-1); j++) {
fprintf(task_info_fp, "%s ", text_list_get(mt->task_input_files, j));
}
fprintf(task_info_fp, "%s,", text_list_get(mt->task_input_files, num_input_files-1));
} else {
fprintf(task_info_fp, ",");
}
if (extra_output_files != NULL && strlen(extra_output_files) != 0) {
int j = 0;
int num_output_files = text_list_size(mt->task_output_files);
for (j = 0; j < (num_output_files-1); j++) {
fprintf(task_info_fp, "%s ", text_list_get(mt->task_output_files, j));
}
fprintf(task_info_fp, "%s,",text_list_get(mt->task_output_files, num_output_files-1));
} else {
fprintf(task_info_fp, ",");
}
// The default resource requirements for each task
int64_t cores = -1;
int64_t memory = -1;
int64_t disk = -1;
if (resources) {
cores = resources->cores > -1 ? resources->cores : cores;
memory = resources->memory > -1 ? resources->memory : memory;
disk = resources->disk > -1 ? resources->disk : disk;
}
fprintf(task_info_fp, "%" PRId64 ",%" PRId64 ",%" PRId64 ",", cores, memory, disk);
fputs("submitted\n", task_info_fp);
mesos_task_delete(mt);
fclose(task_info_fp);
return task_id;
}
static batch_job_id_t batch_job_amazon_batch_wait(struct batch_queue *q, struct batch_job_info *info_out, time_t stoptime){
struct internal_amazon_batch_amazon_ids amazon_ids = initialize(q);
//succeeded check
int done = 0;
char* env_var = amazon_ids.master_env_prefix;
itable_firstkey(amazon_job_ids);
char* jaid;
UINT64_T jobid;
while(itable_nextkey(amazon_job_ids,&jobid,(void**)&jaid)){
done = describe_aws_job(jaid,env_var);
char* jobname = string_format("%s_%u",queue_name,(unsigned int)jobid);
unsigned int id = (unsigned int)jobid;
if(done == DESCRIBE_AWS_JOB_SUCCESS){
if(itable_lookup(done_jobs,id+1) == NULL){
//id is done, returning here
debug(D_BATCH,"Inserting id: %u into done_jobs",id);
itable_insert(done_jobs,id+1,jobname);
itable_remove(amazon_job_ids,jobid);
//pull files from s3
char* output_files = itable_lookup(done_files,id);
struct list* file_list = extract_file_names_from_list(output_files);
if(list_size(file_list)> 0){
list_first_item(file_list);
char* cur_file = NULL;
while((cur_file=list_next_item(file_list)) != NULL){
debug(D_BATCH,"Copying over %s",cur_file);
char* get_from_s3_cmd = string_format("%s aws s3 cp s3://%s/%s.txz ./%s.txz && tar -xvf %s.txz && rm %s.txz",env_var,bucket_name,cur_file,cur_file, cur_file, cur_file);
int outputcode = sh_system(get_from_s3_cmd);
debug(D_BATCH,"output code from calling S3 to pull file %s: %i",cur_file,outputcode);
FILE* tmpOut = fopen(cur_file,"r");
if(tmpOut){
debug(D_BATCH,"File does indeed exist: %s",cur_file);
fclose(tmpOut);
}else{
debug(D_BATCH,"File doesn't exist: %s",cur_file);
}
free(get_from_s3_cmd);
}
}
list_free(file_list);
list_delete(file_list);
//Let Makeflow know we're all done!
debug(D_BATCH,"Removing the job from the job_table");
struct batch_job_info* info = itable_remove(q->job_table, id);//got from batch_job_amazon.c
info->finished = time(0);//get now
info->exited_normally=1;
info->exit_code=finished_aws_job_exit_code(jaid,env_var);
debug(D_BATCH,"copying over the data to info_out");
memcpy(info_out, info, sizeof(struct batch_job_info));
free(info);
char* jobdef = aws_job_def(jaid);
del_job_def(jobdef);
free(jobdef);
return id;
}
}else if(done == DESCRIBE_AWS_JOB_FAILED || done == DESCRIBE_AWS_JOB_NON_EXIST){
if(itable_lookup(done_jobs,id+1)==NULL){
//id is done, returning here
itable_insert(done_jobs,id+1,jobname);
itable_remove(amazon_job_ids,jobid);
debug(D_BATCH,"Failed job: %i",id);
struct batch_job_info* info = itable_remove(q->job_table, id);//got from batch_job_amazon.c
info->finished = time(0); //get now
info->exited_normally=0;
int exc = finished_aws_job_exit_code(jaid,env_var);
info->exit_code= exc == 0 ? -1 : exc;
memcpy(info_out, info, sizeof(*info));
free(info);
char* jobdef = aws_job_def(jaid);
del_job_def(jobdef);
free(jobdef);
return id;
}
}else{
continue;
}
}
return -1;
}
static batch_job_id_t batch_job_mesos_wait (struct batch_queue * q, struct batch_job_info * info_out, time_t stoptime)
{
char line[MAX_BUF_SIZE];
FILE *task_state_fp;
int last_pos = 0;
int curr_pos = 0;
int read_len = 0;
if(!finished_tasks) {
finished_tasks = itable_create(0);
}
while(access(FILE_TASK_STATE, F_OK) == -1) {}
task_state_fp = fopen(FILE_TASK_STATE, "r");
while(1) {
char *task_id_str;
char *task_stat_str;
const char *task_exit_code;
int task_id;
while(fgets(line, MAX_BUF_SIZE, task_state_fp) != NULL) {
curr_pos = ftell(task_state_fp);
read_len = curr_pos - last_pos;
last_pos = curr_pos;
// trim the newline character
if (line[read_len-1] == '\n') {
line[read_len-1] = '\0';
--read_len;
}
task_id_str = strtok(line, ",");
task_id = atoi(task_id_str);
// There is a new task finished
if(itable_lookup(finished_tasks, task_id) == NULL) {
struct batch_job_info *info = itable_remove(q->job_table, task_id);
info->finished = time(0);
task_stat_str = strtok(NULL, ",");
if (strcmp(task_stat_str, "finished") == 0) {
info->exited_normally = 1;
} else if (strcmp(task_stat_str, "failed") == 0) {
info->exited_normally = 0;
task_exit_code = strtok(NULL, ",");
// 444 is an arbitrary exit code set in mf_mesos_scheduler,
// which means the task failed to retrieve the outpus
if(atoi(task_exit_code) == 444) {
info->exit_code = 444;
debug(D_BATCH, "Task %s failed to retrieve the output.", task_id_str);
}
info->exit_code = atoi(task_exit_code);
} else {
info->exited_normally = 0;
}
memcpy(info_out, info, sizeof(*info));
free(info);
fclose(task_state_fp);
int itable_val = 1;
itable_insert(finished_tasks, task_id, &itable_val);
return task_id;
}
}
sleep(1);
if(stoptime != 0 && time(0) >= stoptime) {
fclose(task_state_fp);
return -1;
}
}
}
static batch_job_id_t batch_job_cluster_submit (struct batch_queue * q, const char *cmd, const char *extra_input_files, const char *extra_output_files, struct jx *envlist, const struct rmsummary *resources )
{
batch_job_id_t jobid;
struct batch_job_info *info;
const char *options = hash_table_lookup(q->options, "batch-options");
if(!setup_batch_wrapper(q, cluster_name)) {
debug(D_NOTICE|D_BATCH,"couldn't setup wrapper file: %s",strerror(errno));
return 0;
}
/* Use the first word in the command line as a name for the job. */
char *name = xxstrdup(cmd);
{
char *s = strchr(name, ' ');
if(s)
*s = 0;
}
/*
Experiment shows that passing environment variables
through the command-line doesn't work, due to multiple
levels of quote interpretation. So, we export all
variables into the environment, and rely upon the -V
option to load the environment into the job.
*/
if(envlist) {
jx_export(envlist);
}
/*
Pass the command to run through the environment as well.
*/
setenv("BATCH_JOB_COMMAND", cmd, 1);
char *command = string_format("%s %s %s '%s' %s %s.wrapper",
cluster_submit_cmd,
cluster_options,
cluster_jobname_var,
path_basename(name),
options ? options : "",
cluster_name);
free(name);
debug(D_BATCH, "%s", command);
FILE *file = popen(command, "r");
free(command);
if(!file) {
debug(D_BATCH, "couldn't submit job: %s", strerror(errno));
return -1;
}
char line[BATCH_JOB_LINE_MAX] = "";
while(fgets(line, sizeof(line), file)) {
if(sscanf(line, "Your job %" SCNbjid, &jobid) == 1
|| sscanf(line, "Submitted batch job %" SCNbjid, &jobid) == 1
|| sscanf(line, "%" SCNbjid, &jobid) == 1 ) {
debug(D_BATCH, "job %" PRIbjid " submitted", jobid);
pclose(file);
info = malloc(sizeof(*info));
memset(info, 0, sizeof(*info));
info->submitted = time(0);
itable_insert(q->job_table, jobid, info);
return jobid;
}
}
if(strlen(line)) {
debug(D_NOTICE, "job submission failed: %s", line);
} else {
debug(D_NOTICE, "job submission failed: no output from %s", cluster_name);
}
pclose(file);
return -1;
}
struct mpi_queue_task *mpi_queue_wait(struct mpi_queue *q, int timeout)
{
struct mpi_queue_task *t;
time_t stoptime;
int result;
if(timeout == MPI_QUEUE_WAITFORTASK) {
stoptime = 0;
} else {
stoptime = time(0) + timeout;
}
while(1) {
// If a task is already complete, return it
t = list_pop_head(q->complete_list);
if(t)
return t;
if(list_size(q->ready_list) == 0 && itable_size(q->active_list) == 0)
break;
// Wait no longer than the caller's patience.
int msec;
int sec;
if(stoptime) {
sec = MAX(0, stoptime - time(0));
msec = sec * 1000;
} else {
sec = 5;
msec = 5000;
}
if(!q->mpi_link) {
q->mpi_link = link_accept(q->master_link, stoptime);
if(q->mpi_link) {
char working_dir[MPI_QUEUE_LINE_MAX];
link_tune(q->mpi_link, LINK_TUNE_INTERACTIVE);
link_usleep(q->mpi_link, msec, 0, 1);
getcwd(working_dir, MPI_QUEUE_LINE_MAX);
link_putfstring(q->mpi_link, "workdir %s\n", stoptime, working_dir);
result = link_usleep(q->mpi_link, msec, 1, 1);
} else {
result = 0;
}
} else {
debug(D_MPI, "Waiting for link to be ready\n");
result = link_usleep(q->mpi_link, msec, 1, 1);
}
// If nothing was awake, restart the loop or return without a task.
if(result <= 0) {
if(stoptime && time(0) >= stoptime) {
return 0;
} else {
continue;
}
}
debug(D_MPI, "sending %d tasks to the MPI master process\n", list_size(q->ready_list));
// Send all ready tasks to the MPI master process
while(list_size(q->ready_list)) {
struct mpi_queue_task *t = list_pop_head(q->ready_list);
result = dispatch_task(q->mpi_link, t, msec/1000);
if(result <= 0)
return 0;
itable_insert(q->active_list, t->taskid, t);
}
// Receive any results back
result = get_results(q->mpi_link, q->active_list, q->complete_list, msec/1000);
if(result < 0) {
return 0;
}
}
return 0;
}
void makeflow_wrapper_generate_files( struct batch_task *task, struct list *input, struct list *output, struct dag_node *n, struct makeflow_wrapper *w)
{
char *f;
char *nodeid = string_format("%d",n->nodeid);
list_first_item(input);
while((f = list_next_item(input)))
{
char *filename = string_replace_percents(f, nodeid);
char *f = xxstrdup(filename);
free(filename);
char *remote, *p;
struct dag_file *file;
p = strchr(f, '=');
if(p) {
*p = 0;
file = dag_file_lookup_or_create(n->d, f);
if(!n->local_job && !itable_lookup(w->remote_names, (uintptr_t) file)){
remote = xxstrdup(p+1);
itable_insert(w->remote_names, (uintptr_t) file, (void *)remote);
hash_table_insert(w->remote_names_inv, remote, (void *)file);
makeflow_hook_add_input_file(n->d, task, f, remote, file->type);
} else {
makeflow_hook_add_output_file(n->d, task, f, NULL, file->type);
}
*p = '=';
} else {
file = dag_file_lookup_or_create(n->d, f);
makeflow_hook_add_input_file(n->d, task, f, NULL, file->type);
}
free(f);
}
list_first_item(output);
while((f = list_next_item(output)))
{
char *filename = string_replace_percents(f, nodeid);
char *f = xxstrdup(filename);
free(filename);
char *remote, *p;
struct dag_file *file;
p = strchr(f, '=');
if(p) {
*p = 0;
file = dag_file_lookup_or_create(n->d, f);
if(!n->local_job && !itable_lookup(w->remote_names, (uintptr_t) file)){
remote = xxstrdup(p+1);
itable_insert(w->remote_names, (uintptr_t) file, (void *)remote);
hash_table_insert(w->remote_names_inv, remote, (void *)file);
makeflow_hook_add_output_file(n->d, task, f, remote, file->type);
} else {
makeflow_hook_add_output_file(n->d, task, f, NULL, file->type);
}
*p = '=';
} else {
file = dag_file_lookup_or_create(n->d, f);
makeflow_hook_add_output_file(n->d, task, f, NULL, file->type);
}
free(f);
}
free(nodeid);
}
