int submit_tasks(struct work_queue *q, int input_size, int run_time, int output_size, int count ) { static int ntasks=0; char output_file[128]; char command[256]; char gen_input_cmd[256]; /* Note that bs=1m and similar are not portable across various implementations of dd, so we spell it out as bs=1048576 */ sprintf(gen_input_cmd, "dd if=/dev/zero of=input.0 bs=1048576 count=%d",input_size); system(gen_input_cmd); int i; for(i=0;i<count;i++) { sprintf(output_file, "output.%d",ntasks++); sprintf(command, "dd if=/dev/zero of=outfile bs=1048576 count=%d; sleep %d", output_size, run_time ); struct work_queue_task *t = work_queue_task_create(command); work_queue_task_specify_file(t, "input.0", "infile", WORK_QUEUE_INPUT, WORK_QUEUE_CACHE); work_queue_task_specify_file(t, output_file, "outfile", WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE); work_queue_task_specify_cores(t,1); work_queue_submit(q, t); } return 1; }
static batch_job_id_t batch_job_wq_submit (struct batch_queue * q, const char *cmd, const char *extra_input_files, const char *extra_output_files, struct jx *envlist, const struct rmsummary *resources) { struct work_queue_task *t; int caching_flag = WORK_QUEUE_CACHE; if(string_istrue(hash_table_lookup(q->options, "caching"))) { caching_flag = WORK_QUEUE_CACHE; } else { caching_flag = WORK_QUEUE_NOCACHE; } t = work_queue_task_create(cmd); specify_files(t, extra_input_files, extra_output_files, caching_flag); specify_envlist(t,envlist); if(envlist) { const char *category = jx_lookup_string(envlist, "CATEGORY"); if(category) { work_queue_task_specify_category(t, category); } } if(resources) { work_queue_task_specify_resources(t, resources); } work_queue_submit(q->data, t); return t->taskid; }
static batch_job_id_t batch_job_wq_submit (struct batch_queue * q, const char *cmd, const char *extra_input_files, const char *extra_output_files, struct nvpair *envlist ) { struct work_queue_task *t; int caching_flag = WORK_QUEUE_CACHE; if(string_istrue(hash_table_lookup(q->options, "caching"))) { caching_flag = WORK_QUEUE_CACHE; } else { caching_flag = WORK_QUEUE_NOCACHE; } t = work_queue_task_create(cmd); specify_files(t, extra_input_files, extra_output_files, caching_flag); specify_envlist(t,envlist); struct rmsummary *resources = parse_batch_options_resources(hash_table_lookup(q->options, "batch-options")); if(resources) { work_queue_task_specify_resources(t, resources); free(resources); } work_queue_submit(q->data, t); return t->taskid; }
static int task_consider( int x, int y ) { char command[WAVEFRONT_LINE_MAX]; char tag[WAVEFRONT_LINE_MAX]; struct work_queue_task* t; if(x>=xsize) return 1; if(y>=ysize) return 1; if(text_array_get(array,x,y)) return 0; const char *left = text_array_get(array,x-1,y); const char *bottom = text_array_get(array,x,y-1); const char *diag = text_array_get(array,x-1,y-1); if(!left || !bottom || !diag) return 1; sprintf(command,"./%s %d %d xfile yfile dfile",function,x,y); sprintf(tag,"%d %d",x,y); t = work_queue_task_create(command); work_queue_task_specify_tag(t,tag); work_queue_task_specify_input_file(t, function, function); work_queue_task_specify_input_buf(t, left, strlen(left), "xfile"); work_queue_task_specify_input_buf(t, bottom, strlen(bottom), "yfile"); work_queue_task_specify_input_buf(t, diag, strlen(diag), "dfile"); work_queue_submit(queue,t); return 1; }
int submit_task(struct work_queue *q, const char *command, const char *executable, const char *infile, off_t infile_offset_start, off_t infile_offset_end, const char *outfile) { struct work_queue_task *t; int taskid; char *infile_dup = strdup(infile); //basename() modifies its arguments. So we need to pass a duplicate. char *executable_dup = strdup(executable); char *outfile_dup = strdup(outfile); t = work_queue_task_create(command); if (!work_queue_task_specify_file_piece(t, infile, basename(infile_dup), infile_offset_start, infile_offset_end, WORK_QUEUE_INPUT, WORK_QUEUE_NOCACHE)) { fprintf(stderr, "task_specify_file_piece() failed for %s: start offset %ld, end offset %ld.\n", infile, infile_offset_start, infile_offset_end); return 0; } if (!work_queue_task_specify_file(t, executable, basename(executable_dup), WORK_QUEUE_INPUT, WORK_QUEUE_CACHE)) { fprintf(stderr, "task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", executable); return 0; } if (!work_queue_task_specify_file(t, outfile, basename(outfile_dup), WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE)) { fprintf(stderr, "task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", outfile); return 0; } taskid = work_queue_submit(q, t); fprintf(stdout, "submitted task (id# %d): %s\n", taskid, t->command_line); free(infile_dup); free(executable_dup); free(outfile_dup); return taskid; }
int submit_task_series(struct work_queue *q, struct task_series *ts, int series_id) { char input_file[128], output_file[128], command[256]; char gen_input_cmd[256]; sprintf(input_file, "input-%d", series_id); list_push_tail(created_files, xxstrdup(input_file)); sprintf(gen_input_cmd, "dd if=/dev/zero of=%s bs=1M count=%d", input_file, ts->input_size); system(gen_input_cmd); // submit tasks to the queue int i; for(i = 0; i < ts->num_of_tasks; i++) { sprintf(output_file, "output-%d-%d", series_id, i); list_push_tail(created_files, xxstrdup(output_file)); sprintf(command, "dd if=/dev/zero of=%s bs=1M count=%d; sleep %d", output_file, ts->output_size, ts->execution_time); struct work_queue_task *t = work_queue_task_create(command); if (!work_queue_task_specify_file(t, input_file, input_file, WORK_QUEUE_INPUT, WORK_QUEUE_CACHE)) { printf("task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", input_file); return 0; } if (!work_queue_task_specify_file(t, output_file, output_file, WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE)) { printf("task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", output_file); return 0; } int taskid = work_queue_submit(q, t); printf("submitted task (id# %d): %s\n", taskid, t->command_line); } return 1; // success }
static void task_submit(struct work_queue *q, int curr_rect_x, int curr_rect_y) { struct work_queue_task *t; char rname_x[32]; char rname_y[32]; char cmd[255]; char fname_x[255]; char fname_y[255]; char tag[32]; sprintf(tag, "%03d-%03d", curr_rect_y, curr_rect_x); sprintf(rname_x, "rect%03d.cfa", curr_rect_x); if(curr_rect_x != curr_rect_y) { sprintf(rname_y, "rect%03d.cfa", curr_rect_y); } else { sprintf(rname_y, "%s", ""); } sprintf(cmd, "./%s %s %s %s", filter_program_name, filter_program_args, rname_x, rname_y); // Create the task. t = work_queue_task_create(cmd); // Specify the tag for this task. Used for identifying which // ones are done. work_queue_task_specify_tag(t, tag); // Send the executable, if it's not already there. work_queue_task_specify_file(t, filter_program_path, filter_program_name, WORK_QUEUE_INPUT, WORK_QUEUE_CACHE); // Send the repeat file if we need it and it's not already there. if(repeat_filename) work_queue_task_specify_file(t, repeat_filename, string_basename(repeat_filename), WORK_QUEUE_INPUT, WORK_QUEUE_CACHE); // Add the rectangle. Add it as staged, so if the worker // already has these sequences, there's no need to send them again. sprintf(fname_x, "%s/%s", outdirname, rname_x); work_queue_task_specify_file(t, fname_x, rname_x, WORK_QUEUE_INPUT, WORK_QUEUE_CACHE); if(curr_rect_x != curr_rect_y) { sprintf(fname_y, "%s/%s", outdirname, rname_y); work_queue_task_specify_file(t, fname_y, rname_y, WORK_QUEUE_INPUT, WORK_QUEUE_CACHE); } work_queue_submit(q, t); total_submitted++; debug(D_DEBUG, "Submitted task for rectangle (%d, %d)\n", curr_rect_y, curr_rect_x); }
static void task_complete(struct work_queue_task *t) { checkpoint_task(t); if(t->result == 0) { debug(D_DEBUG, "task complete: %s: %s", t->tag, t->command_line); if(strlen(t->output) > 0) { char *out = strdup(t->output); char *cand1 = malloc(sizeof(char) * 500); char *cand2 = malloc(sizeof(char) * 500); int dir, start1, start2; char *line = strtok(out, "\n"); int result = sscanf(line, "%s\t%s\t%d\t%d\t%d", cand1, cand2, &dir, &start1, &start2); while(result == 5) { cand_count++; line = strtok(NULL, "\n"); if(line == NULL) { break; } result = sscanf(line, "%s\t%s\t%d\t%d\t%d", cand1, cand2, &dir, &start1, &start2); } free(out); free(cand1); free(cand2); } fputs(t->output, outfile); fflush(outfile); total_processed++; tasks_runtime += (t->time_receive_output_finish - t->time_send_input_start); tasks_filetime += t->total_transfer_time; work_queue_task_delete(t); } else { debug(D_DEBUG, "task failed: %s: %s", t->tag, t->command_line); if(retry_max > total_retried) { debug(D_DEBUG, "retrying task %d/%d", total_retried, retry_max); total_retried++; work_queue_submit(q, t); } else { fprintf(stderr, "%s: giving up after retrying %d tasks.\n", progname, retry_max); exit(1); } } }
int main(int argc, char *argv[]) { struct work_queue *q; struct work_queue_task *t; int port = WORK_QUEUE_DEFAULT_PORT; int taskid; int i; if(argc < 2) { printf("work_queue_example <executable> <file1> [file2] [file3] ...\n"); printf("Each file given on the command line will be compressed using a remote worker.\n"); return 0; } debug_flags_set("all"); q = work_queue_create(port); if(!q) { printf("couldn't listen on port %d: %s\n", port, strerror(errno)); return 1; } printf("listening on port %d...\n", work_queue_port(q)); for(i = 1; i < argc; i++) { char infile[256], outfile[256], command[256]; sprintf(infile, "%s", argv[i]); sprintf(outfile, "%s.gz", argv[i]); sprintf(command, "./gzip < %s > %s", infile, outfile); t = work_queue_task_create(command); if (!work_queue_task_specify_file(t, "/usr/bin/gzip", "gzip", WORK_QUEUE_INPUT, WORK_QUEUE_CACHE)) { printf("task_specify_file() failed for /usr/bin/gzip: check if arguments are null or remote name is an absolute path.\n"); return 1; } if (!work_queue_task_specify_file(t, infile, infile, WORK_QUEUE_INPUT, WORK_QUEUE_NOCACHE)) { printf("task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", infile); return 1; } if (!work_queue_task_specify_file(t, outfile, outfile, WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE)) { printf("task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", outfile); return 1; } taskid = work_queue_submit(q, t); printf("submitted task (id# %d): %s\n", taskid, t->command_line); } printf("waiting for tasks to complete...\n"); while(!work_queue_empty(q)) { t = work_queue_wait(q, 5); if(t) { printf("task (id# %d) complete: %s (return code %d)\n", t->taskid, t->command_line, t->return_status); work_queue_task_delete(t); } } printf("all tasks complete!\n"); work_queue_delete(q); return 0; }
int main(int argc, char **argv) { signed char c; struct work_queue *q; int port = WORK_QUEUE_DEFAULT_PORT; static const char *port_file = NULL; int work_queue_master_mode = WORK_QUEUE_MASTER_MODE_STANDALONE; char *project = NULL; int priority = 0; debug_config("allpairs_master"); extra_files_list = list_create(); struct option long_options[] = { {"debug", required_argument, 0, 'd'}, {"help", no_argument, 0, 'h'}, {"version", no_argument, 0, 'v'}, {"port", required_argument, 0, 'p'}, {"random-port", required_argument, 0, 'Z'}, {"extra-args", required_argument, 0, 'e'}, {"width", required_argument, 0, 'x'}, {"height", required_argument, 0, 'y'}, {"advertise", no_argument, 0, 'a'}, //deprecated, left here for backwards compatibility {"project-name", required_argument, 0, 'N'}, {"debug-file", required_argument, 0, 'o'}, {"output-file", required_argument, 0, 'O'}, {"wqstats-file", required_argument, 0, 's'}, {"input-file", required_argument, 0, 'f'}, {"estimated-time", required_argument, 0, 't'}, {"priority", required_argument, 0, 'P'}, {0,0,0,0} }; while((c = getopt_long(argc, argv, "ad:e:f:hN:p:P:t:vx:y:Z:O:o:s:", long_options, NULL)) >= 0) { switch (c) { case 'a': work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG; break; case 'd': debug_flags_set(optarg); break; case 'e': extra_arguments = optarg; break; case 'f': list_push_head(extra_files_list,optarg); break; case 'o': debug_config_file(optarg); break; case 'O': free(output_filename); output_filename=xxstrdup(optarg); break; case 's': free(wqstats_filename); wqstats_filename=xxstrdup(optarg); break; case 'h': show_help(progname); exit(0); break; case 'N': work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG; free(project); project = xxstrdup(optarg); break; case 'p': port = atoi(optarg); break; case 'P': priority = atoi(optarg); break; case 't': compare_program_time = atof(optarg); break; case 'v': cctools_version_print(stdout, progname); exit(0); break; case 'x': xblock = atoi(optarg); break; case 'y': yblock = atoi(optarg); break; case 'Z': port_file = optarg; port = 0; break; default: show_help(progname); return 1; } } cctools_version_debug(D_DEBUG, argv[0]); if((argc - optind) < 3) { show_help(progname); exit(1); } struct text_list *seta = text_list_load(argv[optind]); if(!seta) { fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno)); return 1; } fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta)); struct text_list *setb = text_list_load(argv[optind+1]); if(!setb) { fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno)); return 1; } fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb)); if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) { fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname); return 1; } debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program); xstop = text_list_size(seta); ystop = text_list_size(setb); if(allpairs_compare_function_get(argv[optind+2])) { strcpy(allpairs_compare_program,argv[optind+2]); debug(D_DEBUG,"using internal function %s",allpairs_compare_program); use_external_program = 0; } else { if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) { fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program); return 1; } debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program); use_external_program = 1; } if(!xblock || !yblock) { estimate_block_size(seta,setb,&xblock,&yblock); } fprintf(stdout, "%s: using block size of %dx%d\n",progname,xblock,yblock); if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) { fprintf(stderr, "allpairs: allpairs master running in catalog mode. Please use '-N' option to specify the name of this project.\n"); fprintf(stderr, "allpairs: Run \"%s -h\" for help with options.\n", argv[0]); return 1; } q = work_queue_create(port); //Read the port the queue is actually running, in case we just called //work_queue_create(LINK_PORT_ANY) port = work_queue_port(q); if(!q) { fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno)); return 1; } if(port_file) opts_write_port_file(port_file, port); if(wqstats_filename) work_queue_specify_log(q, wqstats_filename); // advanced work queue options work_queue_specify_master_mode(q, work_queue_master_mode); work_queue_specify_name(q, project); work_queue_specify_priority(q, priority); fprintf(stdout, "%s: listening for workers on port %d...\n",progname,work_queue_port(q)); while(1) { struct work_queue_task *task = NULL; while(work_queue_hungry(q)) { task = ap_task_create(seta,setb); if(task) { work_queue_submit(q, task); } else { break; } } if(!task && work_queue_empty(q)) break; task = work_queue_wait(q,5); if(task) task_complete(task); } work_queue_delete(q); return 0; }
int main(int argc, char *argv[]) { struct work_queue *q; struct work_queue_task *t; int port = WORK_QUEUE_DEFAULT_PORT; int taskid; int i; char *gzip_path; if(argc < 2) { printf("work_queue_example <file1> [file2] [file3] ...\n"); printf("Each file given on the command line will be compressed using a remote worker.\n"); return 0; } /* Usually, we can execute the gzip utility by simply typing its name at a terminal. However, this is not enough for work queue; we have to specify precisely which files need to be transmitted to the workers. We record the location of gzip in 'gzip_path', which is usually found in /bin/gzip or /usr/bin/gzip. We use the 'access' function (from unistd.h standard C library), and test the path for execution (X_OK) and reading (R_OK) permissions. */ gzip_path = "/bin/gzip"; if(access(gzip_path, X_OK | R_OK) != 0) { gzip_path = "/usr/bin/gzip"; if(access(gzip_path, X_OK | R_OK) != 0) { fprintf(stderr, "gzip was not found. Please modify the gzip_path variable accordingly. To determine the location of gzip, from the terminal type: which gzip (usual locations are /bin/gzip and /usr/bin/gzip)\n"); exit(1); } } /* We create the tasks queue using the default port. If this port is * already been used by another program, you can try setting port = 0 to * use an available port. */ q = work_queue_create(port); if(!q) { printf("couldn't listen on port %d: %s\n", port, strerror(errno)); return 1; } printf("listening on port %d...\n", work_queue_port(q)); /* We create and dispatch a task for each filename given in the argument list */ for(i = 1; i < argc; i++) { char infile[256], outfile[256], command[256]; sprintf(infile, "%s", argv[i]); sprintf(outfile, "%s.gz", argv[i]); /* Note that we write ./gzip here, to guarantee that the gzip version * we are using is the one being sent to the workers. */ sprintf(command, "./gzip < %s > %s", infile, outfile); t = work_queue_task_create(command); /* gzip is the same across all tasks, so we can cache it in the * workers. Note that when specifying a file, we have to name its local * name (e.g. gzip_path), and its remote name (e.g. "gzip"). Unlike the * following line, more often than not these are the same. */ work_queue_task_specify_file(t, gzip_path, "gzip", WORK_QUEUE_INPUT, WORK_QUEUE_CACHE); /* files to be compressed are different across all tasks, so we do not * cache them. This is, of course, application specific. Sometimes you * may want to cache an output file if is the input of a later task.*/ work_queue_task_specify_file(t, infile, infile, WORK_QUEUE_INPUT, WORK_QUEUE_NOCACHE); work_queue_task_specify_file(t, outfile, outfile, WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE); /* Once all files has been specified, we are ready to submit the task to the queue. */ taskid = work_queue_submit(q, t); printf("submitted task (id# %d): %s\n", taskid, t->command_line); } printf("waiting for tasks to complete...\n"); while(!work_queue_empty(q)) { /* Application specific code goes here ... */ /* work_queue_wait waits at most 5 seconds for some task to return. */ t = work_queue_wait(q, 5); if(t) { printf("task (id# %d) complete: %s (return code %d)\n", t->taskid, t->command_line, t->return_status); if(t->return_status != 0) { /* The task failed. Error handling (e.g., resubmit with new parameters) here. */ } work_queue_task_delete(t); } /* Application specific code goes here ... */ } printf("all tasks complete!\n"); work_queue_delete(q); return 0; }
int main(int argc, char **argv) { char c; struct work_queue *q; int port = WORK_QUEUE_DEFAULT_PORT; extra_files_list = list_create(); while((c = getopt(argc, argv, "e:f:t:x:y:p:N:E:d:vh")) != (char) -1) { switch (c) { case 'e': extra_arguments = optarg; break; case 'f': list_push_head(extra_files_list,optarg); break; case 't': compare_program_time = atof(optarg); break; case 'x': xblock = atoi(optarg); break; case 'y': yblock = atoi(optarg); break; case 'p': port = atoi(optarg); break; case 'N': setenv("WORK_QUEUE_NAME", optarg, 1); break; case 'E': setenv("WORK_QUEUE_PRIORITY", optarg, 1); break; case 'd': debug_flags_set(optarg); break; case 'v': show_version(progname); exit(0); break; case 'h': show_help(progname); exit(0); break; default: show_help(progname); return 1; } } if((argc - optind) < 3) { show_help(progname); exit(1); } struct text_list *seta = text_list_load(argv[optind]); if(!seta) { fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno)); return 1; } fprintf(stderr, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta)); struct text_list *setb = text_list_load(argv[optind+1]); if(!setb) { fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno)); return 1; } fprintf(stderr, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb)); if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) { fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname); return 1; } debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program); xstop = text_list_size(seta); ystop = text_list_size(setb); if(allpairs_compare_function_get(argv[optind+2])) { strcpy(allpairs_compare_program,argv[optind+2]); debug(D_DEBUG,"using internal function %s",allpairs_compare_program); use_external_program = 0; } else { if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) { fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program); return 1; } debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program); use_external_program = 1; } if(!xblock || !yblock) { estimate_block_size(seta,setb,&xblock,&yblock); } fprintf(stderr, "%s: using block size of %dx%d\n",progname,xblock,yblock); q = work_queue_create(port); if(!q) { fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno)); return 1; } fprintf(stderr, "%s: listening for workers on port %d...\n",progname,work_queue_port(q)); while(1) { struct work_queue_task *task = NULL; while(work_queue_hungry(q)) { task = ap_task_create(seta,setb); if(task) { work_queue_submit(q, task); } else { break; } } if(!task && work_queue_empty(q)) break; task = work_queue_wait(q,5); if(task) task_complete(task); } work_queue_delete(q); return 0; }