void work_queue_process_delete(struct work_queue_process *p)
{
if(p->task)
work_queue_task_delete(p->task);
if(p->output_fd) {
close(p->output_fd);
}
if(p->output_file_name) {
unlink(p->output_file_name);
free(p->output_file_name);
}
if(p->sandbox) {
if(p->loop_mount == 1) {
disk_alloc_delete(p->sandbox);
}
else {
delete_dir(p->sandbox);
}
free(p->sandbox);
}
if(p->tmpdir)
free(p->tmpdir);
free(p);
}
void wait_for_all_tasks( struct work_queue *q )
{
struct work_queue_task *t;
while(!work_queue_empty(q)) {
t = work_queue_wait(q,5);
if(t) work_queue_task_delete(t);
}
}
void wait_for_task(struct work_queue *q, int timeout) {
struct work_queue_task *t = work_queue_wait(q,timeout);
if(t) {
printf("task (id# %d) complete: %s (return code %d)\n", t->taskid, t->command_line, t->return_status);
work_queue_task_delete(t);
log_work_queue_status(q);
}
}
static void task_complete(struct work_queue_task *t)
{
checkpoint_task(t);
if(t->result == 0) {
debug(D_DEBUG, "task complete: %s: %s", t->tag, t->command_line);
if(strlen(t->output) > 0) {
char *out = strdup(t->output);
char *cand1 = malloc(sizeof(char) * 500);
char *cand2 = malloc(sizeof(char) * 500);
int dir, start1, start2;
char *line = strtok(out, "\n");
int result = sscanf(line, "%s\t%s\t%d\t%d\t%d", cand1, cand2, &dir, &start1, &start2);
while(result == 5) {
cand_count++;
line = strtok(NULL, "\n");
if(line == NULL) {
break;
}
result = sscanf(line, "%s\t%s\t%d\t%d\t%d", cand1, cand2, &dir, &start1, &start2);
}
free(out);
free(cand1);
free(cand2);
}
fputs(t->output, outfile);
fflush(outfile);
total_processed++;
tasks_runtime += (t->time_receive_output_finish - t->time_send_input_start);
tasks_filetime += t->total_transfer_time;
work_queue_task_delete(t);
} else {
debug(D_DEBUG, "task failed: %s: %s", t->tag, t->command_line);
if(retry_max > total_retried) {
debug(D_DEBUG, "retrying task %d/%d", total_retried, retry_max);
total_retried++;
work_queue_submit(q, t);
} else {
fprintf(stderr, "%s: giving up after retrying %d tasks.\n", progname, retry_max);
exit(1);
}
}
}
double wait_partition_tasks(struct work_queue *q, int timeout, char *task_times_file) {
struct work_queue_task *t;
FILE *task_times_fp = NULL;
double task_execution_times = 0;
int64_t total_transfered_bytes = 0;
time_t total_transfer_time = 0;
if(task_times_file) {
task_times_fp = fopen("wq_sort.tasktimes", "w");
if (!task_times_fp) {
fprintf(stderr, "Opening of wq_sort.tasktimes file failed!\n");
}
}
while(!work_queue_empty(q)) {
t = work_queue_wait(q, timeout);
if(t) {
fprintf(stdout, "Task (taskid# %d) complete in %llu: %s (return code %d)\n", t->taskid, (long long unsigned) t->cmd_execution_time, t->command_line, t->return_status);
total_transfered_bytes += t->total_bytes_transferred;
total_transfer_time += t->total_transfer_time;
fprintf(stderr, "Total bytes sent %" PRId64 " in %llu s\n", total_transfered_bytes, (long long unsigned) total_transfer_time);
fprintf(stderr, "Default bandwidth (Bps): %f\n", bandwidth_bytes_per_sec);
bandwidth_bytes_per_sec = total_transfered_bytes / (total_transfer_time/1000000.0);
fprintf(stderr, "Measured bandwidth (Bps): %f\n", bandwidth_bytes_per_sec);
task_execution_times += t->cmd_execution_time/1000000.00;
if(task_times_fp) {
fprintf(task_times_fp, "%d: %llu\n", t->taskid, (long long unsigned) t->cmd_execution_time);
}
work_queue_task_delete(t);
}
}
if(task_times_fp)
fclose(task_times_fp);
return task_execution_times;
}
void work_queue_process_delete( struct work_queue_process *p )
{
if(p->task) work_queue_task_delete(p->task);
if(p->output_fd) {
close(p->output_fd);
}
if(p->output_file_name) {
unlink(p->output_file_name);
free(p->output_file_name);
}
if(p->sandbox) {
delete_dir(p->sandbox);
free(p->sandbox);
}
free(p);
}
void task_complete( struct work_queue_task *t )
{
FILE *output = stdout;
if(output_filename)
{
output = fopen(output_filename, "a");
if(!output)
{
fprintf(stderr, "Cannot open %s for writing. Output to stdout instead.\n", output_filename);
output = stdout;
}
}
string_chomp(t->output);
fprintf(output, "%s\n",t->output);
if(output != stdout)
fclose(output);
work_queue_task_delete(t);
}
int main(int argc, char *argv[])
{
struct work_queue *q;
struct work_queue_task *t;
int port = WORK_QUEUE_DEFAULT_PORT;
int taskid;
int i;
if(argc < 2) {
printf("work_queue_example <executable> <file1> [file2] [file3] ...\n");
printf("Each file given on the command line will be compressed using a remote worker.\n");
return 0;
}
debug_flags_set("all");
q = work_queue_create(port);
if(!q) {
printf("couldn't listen on port %d: %s\n", port, strerror(errno));
return 1;
}
printf("listening on port %d...\n", work_queue_port(q));
for(i = 1; i < argc; i++) {
char infile[256], outfile[256], command[256];
sprintf(infile, "%s", argv[i]);
sprintf(outfile, "%s.gz", argv[i]);
sprintf(command, "./gzip < %s > %s", infile, outfile);
t = work_queue_task_create(command);
if (!work_queue_task_specify_file(t, "/usr/bin/gzip", "gzip", WORK_QUEUE_INPUT, WORK_QUEUE_CACHE)) {
printf("task_specify_file() failed for /usr/bin/gzip: check if arguments are null or remote name is an absolute path.\n");
return 1;
}
if (!work_queue_task_specify_file(t, infile, infile, WORK_QUEUE_INPUT, WORK_QUEUE_NOCACHE)) {
printf("task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", infile);
return 1;
}
if (!work_queue_task_specify_file(t, outfile, outfile, WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE)) {
printf("task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", outfile);
return 1;
}
taskid = work_queue_submit(q, t);
printf("submitted task (id# %d): %s\n", taskid, t->command_line);
}
printf("waiting for tasks to complete...\n");
while(!work_queue_empty(q)) {
t = work_queue_wait(q, 5);
if(t) {
printf("task (id# %d) complete: %s (return code %d)\n", t->taskid, t->command_line, t->return_status);
work_queue_task_delete(t);
}
}
printf("all tasks complete!\n");
work_queue_delete(q);
return 0;
}
int main( int argc, char *argv[] )
{
signed char c;
int work_queue_master_mode = WORK_QUEUE_MASTER_MODE_STANDALONE;
char *project = NULL;
int priority = 0;
const char *progname = "wavefront";
debug_config(progname);
struct option long_options[] = {
{"help", no_argument, 0, 'h'},
{"version", no_argument, 0, 'v'},
{"debug", required_argument, 0, 'd'},
{"advertise", no_argument, 0, 'a'},
{"project-name", required_argument, 0, 'N'},
{"debug-file", required_argument, 0, 'o'},
{"port", required_argument, 0, 'p'},
{"priority", required_argument, 0, 'P'},
{"estimated-time", required_argument, 0, 't'},
{"random-port", required_argument, 0, 'Z'},
{"bitmap", required_argument, 0, 'B'},
{0,0,0,0}
};
while((c=getopt_long(argc,argv,"aB:d:hN:p:P:o:v:Z:", long_options, NULL)) >= 0) {
switch(c) {
case 'a':
break;
case 'd':
debug_flags_set(optarg);
break;
case 'h':
show_help(progname);
exit(0);
break;
case 'N':
work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
free(project);
project = xxstrdup(optarg);
break;
case 'p':
port = atoi(optarg);
break;
case 'P':
priority = atoi(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'v':
cctools_version_print(stdout, progname);
exit(0);
break;
case 'Z':
port_file = optarg;
port = 0;
break;
case 'B':
progress_bitmap_file = optarg;
break;
default:
show_help(progname);
return 1;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if( (argc-optind)!=5 ) {
show_help(progname);
exit(1);
}
function = argv[optind];
xsize=atoi(argv[optind+1]);
ysize=atoi(argv[optind+2]);
infile=argv[optind+3];
outfile=argv[optind+4];
start_time = time(0);
last_display_time = 0;
cells_total = xsize*ysize;
xsize++;
ysize++;
array = text_array_create(xsize,ysize);
if(!text_array_load(array,infile)) {
fprintf(stderr,"couldn't load %s: %s",infile,strerror(errno));
return 1;
}
int count = text_array_load(array,outfile);
if(count>0) printf("recovered %d results from %s\n",count,outfile);
logfile = fopen(outfile,"a");
if(!logfile) {
fprintf(stderr,"couldn't open %s for append: %s\n",outfile,strerror(errno));
return 1;
}
if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) {
fprintf(stderr, "wavefront: wavefront master running in catalog mode. Please use '-N' option to specify the name of this project.\n");
fprintf(stderr, "wavefront: Run \"%s -h\" for help with options.\n", argv[0]);
return 1;
}
queue = work_queue_create(port);
//Read the port the queue is actually running, in case we just called
//work_queue_create(LINK_PORT_ANY)
port = work_queue_port(queue);
if(!queue) {
fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno));
return 1;
}
if(port_file)
opts_write_port_file(port_file, port);
// advanced work queue options
work_queue_specify_master_mode(queue, work_queue_master_mode);
work_queue_specify_name(queue, project);
work_queue_specify_priority(queue, priority);
fprintf(stdout, "%s: listening for workers on port %d...\n",progname,work_queue_port(queue));
if(progress_bitmap_file)
{
bmap = bitmap_create(xsize,ysize);
wavefront_bitmap_initialize(bmap);
}
task_prime();
struct work_queue_task *t;
while(1) {
if(time(0)!=last_display_time) display_progress(queue);
t = work_queue_wait(queue,WORK_QUEUE_WAITFORTASK);
if(!t) break;
if(t->return_status==0) {
int x,y;
if(sscanf(t->tag,"%d %d",&x,&y)==2) {
text_array_set(array,x,y,t->output);
task_complete(x,y);
fprintf(logfile,"%d %d %s\n",x,y,t->output);
fflush(logfile);
tasks_done++;
} else {
fprintf(stderr,"unexpected output: %s\nfrom command: %s\non host: %s",t->output,t->command_line,t->host);
}
} else {
fprintf(stderr,"function failed return value (%i) result (%i) on host %s. output:\n%s\n",t->return_status,t->result,t->host,t->output);
}
work_queue_task_delete(t);
if(work_queue_empty(queue))
break;
}
display_progress(queue);
return 0;
}
static batch_job_id_t batch_job_wq_wait (struct batch_queue * q, struct batch_job_info * info, time_t stoptime)
{
static int try_open_log = 0;
int timeout, taskid = -1;
if(!try_open_log)
{
try_open_log = 1;
if(!work_queue_specify_log(q->data, q->logfile))
{
return -1;
}
const char *transactions = batch_queue_get_option(q, "batch_log_transactions_name");
if(transactions) {
work_queue_specify_transactions_log(q->data, transactions);
}
}
if(stoptime == 0) {
timeout = WORK_QUEUE_WAITFORTASK;
} else {
timeout = MAX(0, stoptime - time(0));
}
struct work_queue_task *t = work_queue_wait(q->data, timeout);
if(t) {
info->submitted = t->time_when_submitted / 1000000;
info->started = t->time_when_commit_end / 1000000;
info->finished = t->time_when_done / 1000000;
info->exited_normally = 1;
info->exit_code = t->return_status;
info->exit_signal = 0;
info->disk_allocation_exhausted = t->disk_allocation_exhausted;
/*
If the standard ouput of the job is not empty,
then print it, because this is analogous to a Unix
job, and would otherwise be lost. Important for
capturing errors from the program.
*/
if(t->output && t->output[0]) {
if(t->output[1] || t->output[0] != '\n') {
string_chomp(t->output);
printf("%s\n", t->output);
}
}
char *outfile = itable_remove(q->output_table, t->taskid);
if(outfile) {
FILE *file = fopen(outfile, "w");
if(file) {
fwrite(t->output, strlen(t->output), 1, file);
fclose(file);
}
free(outfile);
}
taskid = t->taskid;
work_queue_task_delete(t);
}
if(taskid >= 0) {
return taskid;
}
if(work_queue_empty(q->data)) {
return 0;
} else {
return -1;
}
}
int main( int argc, char *argv[] )
{
char c;
const char *progname = "wavefront";
debug_config(progname);
while((c=getopt(argc,argv,"p:Pd:o:vh"))!=(char)-1) {
switch(c) {
case 'p':
port = atoi(optarg);
break;
case 'd':
debug_flags_set(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'v':
show_version(progname);
exit(0);
break;
case 'h':
show_help(progname);
exit(0);
break;
}
}
if( (argc-optind)!=5 ) {
show_help(progname);
exit(1);
}
function = argv[optind];
xsize=atoi(argv[optind+1]);
ysize=atoi(argv[optind+2]);
infile=argv[optind+3];
outfile=argv[optind+4];
start_time = time(0);
last_display_time = 0;
cells_total = xsize*ysize;
xsize++;
ysize++;
array = text_array_create(xsize,ysize);
if(!text_array_load(array,infile)) {
fprintf(stderr,"couldn't load %s: %s",infile,strerror(errno));
return 1;
}
int count = text_array_load(array,outfile);
if(count>0) printf("recovered %d results from %s\n",count,outfile);
logfile = fopen(outfile,"a");
if(!logfile) {
fprintf(stderr,"couldn't open %s for append: %s\n",outfile,strerror(errno));
return 1;
}
queue = work_queue_create(port);
task_prime();
struct work_queue_task *t;
while(1) {
if(time(0)!=last_display_time) display_progress(queue);
t = work_queue_wait(queue,WORK_QUEUE_WAITFORTASK);
if(!t) break;
if(t->return_status==0) {
int x,y;
if(sscanf(t->tag,"%d %d",&x,&y)==2) {
text_array_set(array,x,y,t->output);
task_complete(x,y);
fprintf(logfile,"%d %d %s\n",x,y,t->output);
fflush(logfile);
tasks_done++;
} else {
fprintf(stderr,"unexpected output: %s\nfrom command: %s\non host: %s",t->output,t->command_line,t->host);
}
} else {
fprintf(stderr,"function failed return value (%i) result (%i) on host %s. output:\n%s\n",t->return_status,t->result,t->host,t->output);
}
work_queue_task_delete(t);
if(work_queue_empty(queue))
break;
}
display_progress(queue);
return 0;
}
int main(int argc, char *argv[])
{
struct work_queue *q;
struct work_queue_task *t;
int port = WORK_QUEUE_DEFAULT_PORT;
int taskid;
int i;
char *gzip_path;
if(argc < 2) {
printf("work_queue_example <file1> [file2] [file3] ...\n");
printf("Each file given on the command line will be compressed using a remote worker.\n");
return 0;
}
/*
Usually, we can execute the gzip utility by simply typing its name at a
terminal. However, this is not enough for work queue; we have to specify
precisely which files need to be transmitted to the workers. We record
the location of gzip in 'gzip_path', which is usually found in /bin/gzip
or /usr/bin/gzip. We use the 'access' function (from unistd.h standard C
library), and test the path for execution (X_OK) and reading (R_OK)
permissions.
*/
gzip_path = "/bin/gzip";
if(access(gzip_path, X_OK | R_OK) != 0) {
gzip_path = "/usr/bin/gzip";
if(access(gzip_path, X_OK | R_OK) != 0) {
fprintf(stderr, "gzip was not found. Please modify the gzip_path variable accordingly. To determine the location of gzip, from the terminal type: which gzip (usual locations are /bin/gzip and /usr/bin/gzip)\n");
exit(1);
}
}
/* We create the tasks queue using the default port. If this port is
* already been used by another program, you can try setting port = 0 to
* use an available port. */
q = work_queue_create(port);
if(!q) {
printf("couldn't listen on port %d: %s\n", port, strerror(errno));
return 1;
}
printf("listening on port %d...\n", work_queue_port(q));
/* We create and dispatch a task for each filename given in the argument list */
for(i = 1; i < argc; i++) {
char infile[256], outfile[256], command[256];
sprintf(infile, "%s", argv[i]);
sprintf(outfile, "%s.gz", argv[i]);
/* Note that we write ./gzip here, to guarantee that the gzip version
* we are using is the one being sent to the workers. */
sprintf(command, "./gzip < %s > %s", infile, outfile);
t = work_queue_task_create(command);
/* gzip is the same across all tasks, so we can cache it in the
* workers. Note that when specifying a file, we have to name its local
* name (e.g. gzip_path), and its remote name (e.g. "gzip"). Unlike the
* following line, more often than not these are the same. */
work_queue_task_specify_file(t, gzip_path, "gzip", WORK_QUEUE_INPUT, WORK_QUEUE_CACHE);
/* files to be compressed are different across all tasks, so we do not
* cache them. This is, of course, application specific. Sometimes you
* may want to cache an output file if is the input of a later task.*/
work_queue_task_specify_file(t, infile, infile, WORK_QUEUE_INPUT, WORK_QUEUE_NOCACHE);
work_queue_task_specify_file(t, outfile, outfile, WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE);
/* Once all files has been specified, we are ready to submit the task to the queue. */
taskid = work_queue_submit(q, t);
printf("submitted task (id# %d): %s\n", taskid, t->command_line);
}
printf("waiting for tasks to complete...\n");
while(!work_queue_empty(q)) {
/* Application specific code goes here ... */
/* work_queue_wait waits at most 5 seconds for some task to return. */
t = work_queue_wait(q, 5);
if(t) {
printf("task (id# %d) complete: %s (return code %d)\n", t->taskid, t->command_line, t->return_status);
if(t->return_status != 0)
{
/* The task failed. Error handling (e.g., resubmit with new parameters) here. */
}
work_queue_task_delete(t);
}
/* Application specific code goes here ... */
}
printf("all tasks complete!\n");
work_queue_delete(q);
return 0;
}
void task_complete( struct work_queue_task *t )
{
string_chomp(t->output);
printf("%s\n",t->output);
work_queue_task_delete(t);
}
