void wait_for_all_tasks( struct work_queue *q ) { struct work_queue_task *t; while(!work_queue_empty(q)) { t = work_queue_wait(q,5); if(t) work_queue_task_delete(t); } }
double wait_partition_tasks(struct work_queue *q, int timeout, char *task_times_file) { struct work_queue_task *t; FILE *task_times_fp = NULL; double task_execution_times = 0; int64_t total_transfered_bytes = 0; time_t total_transfer_time = 0; if(task_times_file) { task_times_fp = fopen("wq_sort.tasktimes", "w"); if (!task_times_fp) { fprintf(stderr, "Opening of wq_sort.tasktimes file failed!\n"); } } while(!work_queue_empty(q)) { t = work_queue_wait(q, timeout); if(t) { fprintf(stdout, "Task (taskid# %d) complete in %llu: %s (return code %d)\n", t->taskid, (long long unsigned) t->cmd_execution_time, t->command_line, t->return_status); total_transfered_bytes += t->total_bytes_transferred; total_transfer_time += t->total_transfer_time; fprintf(stderr, "Total bytes sent %" PRId64 " in %llu s\n", total_transfered_bytes, (long long unsigned) total_transfer_time); fprintf(stderr, "Default bandwidth (Bps): %f\n", bandwidth_bytes_per_sec); bandwidth_bytes_per_sec = total_transfered_bytes / (total_transfer_time/1000000.0); fprintf(stderr, "Measured bandwidth (Bps): %f\n", bandwidth_bytes_per_sec); task_execution_times += t->cmd_execution_time/1000000.00; if(task_times_fp) { fprintf(task_times_fp, "%d: %llu\n", t->taskid, (long long unsigned) t->cmd_execution_time); } work_queue_task_delete(t); } } if(task_times_fp) fclose(task_times_fp); return task_execution_times; }
void work_queue_wait(struct work_queue *q) { while (!work_queue_empty(q)) event_loop(-1); }
int main(int argc, char *argv[]) { struct work_queue *q; struct work_queue_task *t; int port = WORK_QUEUE_DEFAULT_PORT; int taskid; int i; if(argc < 2) { printf("work_queue_example <executable> <file1> [file2] [file3] ...\n"); printf("Each file given on the command line will be compressed using a remote worker.\n"); return 0; } debug_flags_set("all"); q = work_queue_create(port); if(!q) { printf("couldn't listen on port %d: %s\n", port, strerror(errno)); return 1; } printf("listening on port %d...\n", work_queue_port(q)); for(i = 1; i < argc; i++) { char infile[256], outfile[256], command[256]; sprintf(infile, "%s", argv[i]); sprintf(outfile, "%s.gz", argv[i]); sprintf(command, "./gzip < %s > %s", infile, outfile); t = work_queue_task_create(command); if (!work_queue_task_specify_file(t, "/usr/bin/gzip", "gzip", WORK_QUEUE_INPUT, WORK_QUEUE_CACHE)) { printf("task_specify_file() failed for /usr/bin/gzip: check if arguments are null or remote name is an absolute path.\n"); return 1; } if (!work_queue_task_specify_file(t, infile, infile, WORK_QUEUE_INPUT, WORK_QUEUE_NOCACHE)) { printf("task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", infile); return 1; } if (!work_queue_task_specify_file(t, outfile, outfile, WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE)) { printf("task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", outfile); return 1; } taskid = work_queue_submit(q, t); printf("submitted task (id# %d): %s\n", taskid, t->command_line); } printf("waiting for tasks to complete...\n"); while(!work_queue_empty(q)) { t = work_queue_wait(q, 5); if(t) { printf("task (id# %d) complete: %s (return code %d)\n", t->taskid, t->command_line, t->return_status); work_queue_task_delete(t); } } printf("all tasks complete!\n"); work_queue_delete(q); return 0; }
int main( int argc, char *argv[] ) { signed char c; int work_queue_master_mode = WORK_QUEUE_MASTER_MODE_STANDALONE; char *project = NULL; int priority = 0; const char *progname = "wavefront"; debug_config(progname); struct option long_options[] = { {"help", no_argument, 0, 'h'}, {"version", no_argument, 0, 'v'}, {"debug", required_argument, 0, 'd'}, {"advertise", no_argument, 0, 'a'}, {"project-name", required_argument, 0, 'N'}, {"debug-file", required_argument, 0, 'o'}, {"port", required_argument, 0, 'p'}, {"priority", required_argument, 0, 'P'}, {"estimated-time", required_argument, 0, 't'}, {"random-port", required_argument, 0, 'Z'}, {"bitmap", required_argument, 0, 'B'}, {0,0,0,0} }; while((c=getopt_long(argc,argv,"aB:d:hN:p:P:o:v:Z:", long_options, NULL)) >= 0) { switch(c) { case 'a': break; case 'd': debug_flags_set(optarg); break; case 'h': show_help(progname); exit(0); break; case 'N': work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG; free(project); project = xxstrdup(optarg); break; case 'p': port = atoi(optarg); break; case 'P': priority = atoi(optarg); break; case 'o': debug_config_file(optarg); break; case 'v': cctools_version_print(stdout, progname); exit(0); break; case 'Z': port_file = optarg; port = 0; break; case 'B': progress_bitmap_file = optarg; break; default: show_help(progname); return 1; } } cctools_version_debug(D_DEBUG, argv[0]); if( (argc-optind)!=5 ) { show_help(progname); exit(1); } function = argv[optind]; xsize=atoi(argv[optind+1]); ysize=atoi(argv[optind+2]); infile=argv[optind+3]; outfile=argv[optind+4]; start_time = time(0); last_display_time = 0; cells_total = xsize*ysize; xsize++; ysize++; array = text_array_create(xsize,ysize); if(!text_array_load(array,infile)) { fprintf(stderr,"couldn't load %s: %s",infile,strerror(errno)); return 1; } int count = text_array_load(array,outfile); if(count>0) printf("recovered %d results from %s\n",count,outfile); logfile = fopen(outfile,"a"); if(!logfile) { fprintf(stderr,"couldn't open %s for append: %s\n",outfile,strerror(errno)); return 1; } if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) { fprintf(stderr, "wavefront: wavefront master running in catalog mode. Please use '-N' option to specify the name of this project.\n"); fprintf(stderr, "wavefront: Run \"%s -h\" for help with options.\n", argv[0]); return 1; } queue = work_queue_create(port); //Read the port the queue is actually running, in case we just called //work_queue_create(LINK_PORT_ANY) port = work_queue_port(queue); if(!queue) { fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno)); return 1; } if(port_file) opts_write_port_file(port_file, port); // advanced work queue options work_queue_specify_master_mode(queue, work_queue_master_mode); work_queue_specify_name(queue, project); work_queue_specify_priority(queue, priority); fprintf(stdout, "%s: listening for workers on port %d...\n",progname,work_queue_port(queue)); if(progress_bitmap_file) { bmap = bitmap_create(xsize,ysize); wavefront_bitmap_initialize(bmap); } task_prime(); struct work_queue_task *t; while(1) { if(time(0)!=last_display_time) display_progress(queue); t = work_queue_wait(queue,WORK_QUEUE_WAITFORTASK); if(!t) break; if(t->return_status==0) { int x,y; if(sscanf(t->tag,"%d %d",&x,&y)==2) { text_array_set(array,x,y,t->output); task_complete(x,y); fprintf(logfile,"%d %d %s\n",x,y,t->output); fflush(logfile); tasks_done++; } else { fprintf(stderr,"unexpected output: %s\nfrom command: %s\non host: %s",t->output,t->command_line,t->host); } } else { fprintf(stderr,"function failed return value (%i) result (%i) on host %s. output:\n%s\n",t->return_status,t->result,t->host,t->output); } work_queue_task_delete(t); if(work_queue_empty(queue)) break; } display_progress(queue); return 0; }
static batch_job_id_t batch_job_wq_wait (struct batch_queue * q, struct batch_job_info * info, time_t stoptime) { static int try_open_log = 0; int timeout, taskid = -1; if(!try_open_log) { try_open_log = 1; if(!work_queue_specify_log(q->data, q->logfile)) { return -1; } const char *transactions = batch_queue_get_option(q, "batch_log_transactions_name"); if(transactions) { work_queue_specify_transactions_log(q->data, transactions); } } if(stoptime == 0) { timeout = WORK_QUEUE_WAITFORTASK; } else { timeout = MAX(0, stoptime - time(0)); } struct work_queue_task *t = work_queue_wait(q->data, timeout); if(t) { info->submitted = t->time_when_submitted / 1000000; info->started = t->time_when_commit_end / 1000000; info->finished = t->time_when_done / 1000000; info->exited_normally = 1; info->exit_code = t->return_status; info->exit_signal = 0; info->disk_allocation_exhausted = t->disk_allocation_exhausted; /* If the standard ouput of the job is not empty, then print it, because this is analogous to a Unix job, and would otherwise be lost. Important for capturing errors from the program. */ if(t->output && t->output[0]) { if(t->output[1] || t->output[0] != '\n') { string_chomp(t->output); printf("%s\n", t->output); } } char *outfile = itable_remove(q->output_table, t->taskid); if(outfile) { FILE *file = fopen(outfile, "w"); if(file) { fwrite(t->output, strlen(t->output), 1, file); fclose(file); } free(outfile); } taskid = t->taskid; work_queue_task_delete(t); } if(taskid >= 0) { return taskid; } if(work_queue_empty(q->data)) { return 0; } else { return -1; } }
int main(int argc, char **argv) { debug_config(progname); // By default, turn on fast abort option since we know each job is of very similar size (in terms of runtime). // One can also set the fast_abort_multiplier by the '-f' option. wq_option_fast_abort_multiplier = 10; get_options(argc, argv, progname); outfile = fopen(outfilename, "a+"); if(!outfile) { fprintf(stderr, "%s: couldn't open %s: %s\n", progname, outfilename, strerror(errno)); exit(1); } if(!find_executable(filter_program_name, "PATH", filter_program_path, sizeof(filter_program_path))) { fprintf(stderr, "%s: couldn't find %s in your PATH.\n", progname, filter_program_path); exit(1); } if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) { fprintf(stderr, "sand_filter: sand filter master running in catalog mode. Please use '-N' option to specify the name of this project.\n"); fprintf(stderr, "sand_filter: Run \"%s -h\" for help with options.\n", argv[0]); return 1; } q = work_queue_create(port); if(!q) { fprintf(stderr, "%s: couldn't listen on port %d: %s\n", progname, port, strerror(errno)); exit(1); } port = work_queue_port(q); if(port_file) { opts_write_port_file(port_file,port); } // advanced work queue options work_queue_specify_master_mode(q, work_queue_master_mode); work_queue_specify_name(q, project); work_queue_specify_priority(q, priority); load_sequences(sequence_filename); debug(D_DEBUG, "Sequence loaded.\n", curr_rect_y, curr_rect_x); init_checkpoint(); start_time = time(0); int curr_start_x = 0, curr_start_y = 0, curr_rect_x = 0, curr_rect_y = 0; while(1) { while(work_queue_hungry(q)) { if(curr_start_y >= num_seqs) break; display_progress(); if(checkpoint[curr_rect_y][curr_rect_x] != CHECKPOINT_STATUS_SUCCESS) task_submit(q, curr_rect_x, curr_rect_y); // Increment the x rectangle curr_rect_x++; curr_start_x += rectangle_size; // If we've reached the end of a row, move to the // next row by incrementing the y rectangle. if(curr_start_x >= num_seqs) { curr_rect_y++; curr_start_y += rectangle_size; curr_rect_x = curr_rect_y; curr_start_x = curr_rect_x * rectangle_size; } } if(work_queue_empty(q) && curr_start_y >= num_seqs) break; struct work_queue_task *t = work_queue_wait(q, 5); if(t) task_complete(t); display_progress(); } printf("%s: candidates generated: %lu\n", progname, cand_count); if(checkpoint_file) { fclose(checkpoint_file); } fprintf(outfile, "EOF\n"); fclose(outfile); work_queue_delete(q); if(!do_not_unlink) delete_dir(outdirname); return 0; }
int main( int argc, char *argv[] ) { char c; const char *progname = "wavefront"; debug_config(progname); while((c=getopt(argc,argv,"p:Pd:o:vh"))!=(char)-1) { switch(c) { case 'p': port = atoi(optarg); break; case 'd': debug_flags_set(optarg); break; case 'o': debug_config_file(optarg); break; case 'v': show_version(progname); exit(0); break; case 'h': show_help(progname); exit(0); break; } } if( (argc-optind)!=5 ) { show_help(progname); exit(1); } function = argv[optind]; xsize=atoi(argv[optind+1]); ysize=atoi(argv[optind+2]); infile=argv[optind+3]; outfile=argv[optind+4]; start_time = time(0); last_display_time = 0; cells_total = xsize*ysize; xsize++; ysize++; array = text_array_create(xsize,ysize); if(!text_array_load(array,infile)) { fprintf(stderr,"couldn't load %s: %s",infile,strerror(errno)); return 1; } int count = text_array_load(array,outfile); if(count>0) printf("recovered %d results from %s\n",count,outfile); logfile = fopen(outfile,"a"); if(!logfile) { fprintf(stderr,"couldn't open %s for append: %s\n",outfile,strerror(errno)); return 1; } queue = work_queue_create(port); task_prime(); struct work_queue_task *t; while(1) { if(time(0)!=last_display_time) display_progress(queue); t = work_queue_wait(queue,WORK_QUEUE_WAITFORTASK); if(!t) break; if(t->return_status==0) { int x,y; if(sscanf(t->tag,"%d %d",&x,&y)==2) { text_array_set(array,x,y,t->output); task_complete(x,y); fprintf(logfile,"%d %d %s\n",x,y,t->output); fflush(logfile); tasks_done++; } else { fprintf(stderr,"unexpected output: %s\nfrom command: %s\non host: %s",t->output,t->command_line,t->host); } } else { fprintf(stderr,"function failed return value (%i) result (%i) on host %s. output:\n%s\n",t->return_status,t->result,t->host,t->output); } work_queue_task_delete(t); if(work_queue_empty(queue)) break; } display_progress(queue); return 0; }
int main(int argc, char **argv) { signed char c; struct work_queue *q; int port = WORK_QUEUE_DEFAULT_PORT; static const char *port_file = NULL; int work_queue_master_mode = WORK_QUEUE_MASTER_MODE_STANDALONE; char *project = NULL; int priority = 0; debug_config("allpairs_master"); extra_files_list = list_create(); struct option long_options[] = { {"debug", required_argument, 0, 'd'}, {"help", no_argument, 0, 'h'}, {"version", no_argument, 0, 'v'}, {"port", required_argument, 0, 'p'}, {"random-port", required_argument, 0, 'Z'}, {"extra-args", required_argument, 0, 'e'}, {"width", required_argument, 0, 'x'}, {"height", required_argument, 0, 'y'}, {"advertise", no_argument, 0, 'a'}, //deprecated, left here for backwards compatibility {"project-name", required_argument, 0, 'N'}, {"debug-file", required_argument, 0, 'o'}, {"output-file", required_argument, 0, 'O'}, {"wqstats-file", required_argument, 0, 's'}, {"input-file", required_argument, 0, 'f'}, {"estimated-time", required_argument, 0, 't'}, {"priority", required_argument, 0, 'P'}, {0,0,0,0} }; while((c = getopt_long(argc, argv, "ad:e:f:hN:p:P:t:vx:y:Z:O:o:s:", long_options, NULL)) >= 0) { switch (c) { case 'a': work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG; break; case 'd': debug_flags_set(optarg); break; case 'e': extra_arguments = optarg; break; case 'f': list_push_head(extra_files_list,optarg); break; case 'o': debug_config_file(optarg); break; case 'O': free(output_filename); output_filename=xxstrdup(optarg); break; case 's': free(wqstats_filename); wqstats_filename=xxstrdup(optarg); break; case 'h': show_help(progname); exit(0); break; case 'N': work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG; free(project); project = xxstrdup(optarg); break; case 'p': port = atoi(optarg); break; case 'P': priority = atoi(optarg); break; case 't': compare_program_time = atof(optarg); break; case 'v': cctools_version_print(stdout, progname); exit(0); break; case 'x': xblock = atoi(optarg); break; case 'y': yblock = atoi(optarg); break; case 'Z': port_file = optarg; port = 0; break; default: show_help(progname); return 1; } } cctools_version_debug(D_DEBUG, argv[0]); if((argc - optind) < 3) { show_help(progname); exit(1); } struct text_list *seta = text_list_load(argv[optind]); if(!seta) { fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno)); return 1; } fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta)); struct text_list *setb = text_list_load(argv[optind+1]); if(!setb) { fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno)); return 1; } fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb)); if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) { fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname); return 1; } debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program); xstop = text_list_size(seta); ystop = text_list_size(setb); if(allpairs_compare_function_get(argv[optind+2])) { strcpy(allpairs_compare_program,argv[optind+2]); debug(D_DEBUG,"using internal function %s",allpairs_compare_program); use_external_program = 0; } else { if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) { fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program); return 1; } debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program); use_external_program = 1; } if(!xblock || !yblock) { estimate_block_size(seta,setb,&xblock,&yblock); } fprintf(stdout, "%s: using block size of %dx%d\n",progname,xblock,yblock); if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) { fprintf(stderr, "allpairs: allpairs master running in catalog mode. Please use '-N' option to specify the name of this project.\n"); fprintf(stderr, "allpairs: Run \"%s -h\" for help with options.\n", argv[0]); return 1; } q = work_queue_create(port); //Read the port the queue is actually running, in case we just called //work_queue_create(LINK_PORT_ANY) port = work_queue_port(q); if(!q) { fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno)); return 1; } if(port_file) opts_write_port_file(port_file, port); if(wqstats_filename) work_queue_specify_log(q, wqstats_filename); // advanced work queue options work_queue_specify_master_mode(q, work_queue_master_mode); work_queue_specify_name(q, project); work_queue_specify_priority(q, priority); fprintf(stdout, "%s: listening for workers on port %d...\n",progname,work_queue_port(q)); while(1) { struct work_queue_task *task = NULL; while(work_queue_hungry(q)) { task = ap_task_create(seta,setb); if(task) { work_queue_submit(q, task); } else { break; } } if(!task && work_queue_empty(q)) break; task = work_queue_wait(q,5); if(task) task_complete(task); } work_queue_delete(q); return 0; }
int main(int argc, char *argv[]) { struct work_queue *q; struct work_queue_task *t; int port = WORK_QUEUE_DEFAULT_PORT; int taskid; int i; char *gzip_path; if(argc < 2) { printf("work_queue_example <file1> [file2] [file3] ...\n"); printf("Each file given on the command line will be compressed using a remote worker.\n"); return 0; } /* Usually, we can execute the gzip utility by simply typing its name at a terminal. However, this is not enough for work queue; we have to specify precisely which files need to be transmitted to the workers. We record the location of gzip in 'gzip_path', which is usually found in /bin/gzip or /usr/bin/gzip. We use the 'access' function (from unistd.h standard C library), and test the path for execution (X_OK) and reading (R_OK) permissions. */ gzip_path = "/bin/gzip"; if(access(gzip_path, X_OK | R_OK) != 0) { gzip_path = "/usr/bin/gzip"; if(access(gzip_path, X_OK | R_OK) != 0) { fprintf(stderr, "gzip was not found. Please modify the gzip_path variable accordingly. To determine the location of gzip, from the terminal type: which gzip (usual locations are /bin/gzip and /usr/bin/gzip)\n"); exit(1); } } /* We create the tasks queue using the default port. If this port is * already been used by another program, you can try setting port = 0 to * use an available port. */ q = work_queue_create(port); if(!q) { printf("couldn't listen on port %d: %s\n", port, strerror(errno)); return 1; } printf("listening on port %d...\n", work_queue_port(q)); /* We create and dispatch a task for each filename given in the argument list */ for(i = 1; i < argc; i++) { char infile[256], outfile[256], command[256]; sprintf(infile, "%s", argv[i]); sprintf(outfile, "%s.gz", argv[i]); /* Note that we write ./gzip here, to guarantee that the gzip version * we are using is the one being sent to the workers. */ sprintf(command, "./gzip < %s > %s", infile, outfile); t = work_queue_task_create(command); /* gzip is the same across all tasks, so we can cache it in the * workers. Note that when specifying a file, we have to name its local * name (e.g. gzip_path), and its remote name (e.g. "gzip"). Unlike the * following line, more often than not these are the same. */ work_queue_task_specify_file(t, gzip_path, "gzip", WORK_QUEUE_INPUT, WORK_QUEUE_CACHE); /* files to be compressed are different across all tasks, so we do not * cache them. This is, of course, application specific. Sometimes you * may want to cache an output file if is the input of a later task.*/ work_queue_task_specify_file(t, infile, infile, WORK_QUEUE_INPUT, WORK_QUEUE_NOCACHE); work_queue_task_specify_file(t, outfile, outfile, WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE); /* Once all files has been specified, we are ready to submit the task to the queue. */ taskid = work_queue_submit(q, t); printf("submitted task (id# %d): %s\n", taskid, t->command_line); } printf("waiting for tasks to complete...\n"); while(!work_queue_empty(q)) { /* Application specific code goes here ... */ /* work_queue_wait waits at most 5 seconds for some task to return. */ t = work_queue_wait(q, 5); if(t) { printf("task (id# %d) complete: %s (return code %d)\n", t->taskid, t->command_line, t->return_status); if(t->return_status != 0) { /* The task failed. Error handling (e.g., resubmit with new parameters) here. */ } work_queue_task_delete(t); } /* Application specific code goes here ... */ } printf("all tasks complete!\n"); work_queue_delete(q); return 0; }
int main(int argc, char **argv) { char c; struct work_queue *q; int port = WORK_QUEUE_DEFAULT_PORT; extra_files_list = list_create(); while((c = getopt(argc, argv, "e:f:t:x:y:p:N:E:d:vh")) != (char) -1) { switch (c) { case 'e': extra_arguments = optarg; break; case 'f': list_push_head(extra_files_list,optarg); break; case 't': compare_program_time = atof(optarg); break; case 'x': xblock = atoi(optarg); break; case 'y': yblock = atoi(optarg); break; case 'p': port = atoi(optarg); break; case 'N': setenv("WORK_QUEUE_NAME", optarg, 1); break; case 'E': setenv("WORK_QUEUE_PRIORITY", optarg, 1); break; case 'd': debug_flags_set(optarg); break; case 'v': show_version(progname); exit(0); break; case 'h': show_help(progname); exit(0); break; default: show_help(progname); return 1; } } if((argc - optind) < 3) { show_help(progname); exit(1); } struct text_list *seta = text_list_load(argv[optind]); if(!seta) { fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno)); return 1; } fprintf(stderr, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta)); struct text_list *setb = text_list_load(argv[optind+1]); if(!setb) { fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno)); return 1; } fprintf(stderr, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb)); if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) { fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname); return 1; } debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program); xstop = text_list_size(seta); ystop = text_list_size(setb); if(allpairs_compare_function_get(argv[optind+2])) { strcpy(allpairs_compare_program,argv[optind+2]); debug(D_DEBUG,"using internal function %s",allpairs_compare_program); use_external_program = 0; } else { if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) { fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program); return 1; } debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program); use_external_program = 1; } if(!xblock || !yblock) { estimate_block_size(seta,setb,&xblock,&yblock); } fprintf(stderr, "%s: using block size of %dx%d\n",progname,xblock,yblock); q = work_queue_create(port); if(!q) { fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno)); return 1; } fprintf(stderr, "%s: listening for workers on port %d...\n",progname,work_queue_port(q)); while(1) { struct work_queue_task *task = NULL; while(work_queue_hungry(q)) { task = ap_task_create(seta,setb); if(task) { work_queue_submit(q, task); } else { break; } } if(!task && work_queue_empty(q)) break; task = work_queue_wait(q,5); if(task) task_complete(task); } work_queue_delete(q); return 0; }
int main(int argc, char *argv[]) { struct work_queue *q; int port = WORK_QUEUE_DEFAULT_PORT; if(argc != 4) { printf("Usage: work_queue_workload_simulator <workload_spec> <logfile> <proj_name> \n"); exit(1); } struct list *specs = get_workload_specs(argv[1]); if(!specs) { printf("Failed to load a non-empty workload specification.\n"); exit(1); } created_files = list_create(); if(!created_files) { printf("Failed to allocate memory for a list to store created files.\n"); exit(1); } // open log file logfile = fopen(argv[2], "a"); if(!logfile) { printf("Couldn't open logfile %s: %s\n", argv[2], strerror(errno)); exit(1); } q = work_queue_create(port); if(!q) { printf("couldn't listen on port %d: %s\n", port, strerror(errno)); goto fail; exit(1); } printf("listening on port %d...\n", work_queue_port(q)); // specifying the right modes work_queue_specify_master_mode(q, WORK_QUEUE_MASTER_MODE_CATALOG); work_queue_specify_name(q, argv[3]); work_queue_specify_estimate_capacity_on(q, 1); // report capacity on int time_elapsed = 0; // in seconds int series_id = 0; time_t start_time = time(0); log_work_queue_status(q); while(1) { struct task_series *ts = (struct task_series *)list_peek_tail(specs); if(!ts) { while(!work_queue_empty(q)) { // wait until all tasks to finish wait_for_task(q, 5); } break; } else { time_elapsed = time(0) - start_time; int time_until_next_submit = ts->submit_time - time_elapsed; if(time_until_next_submit <=0) { list_pop_tail(specs); printf("time elapsed: %d seconds\n", time_elapsed); if(!submit_task_series(q, ts, series_id)) { // failed to submit tasks fprintf(stderr, "Failed to submit tasks.\n"); goto fail; } free(ts); series_id++; } else { time_t stoptime = start_time + ts->submit_time; while(!work_queue_empty(q)) { int timeout = stoptime - time(0); if(timeout > 0) { wait_for_task(q, timeout); } else { break; } } time_t current_time = time(0); if(current_time < stoptime) { sleep(stoptime - current_time); } } } } printf("all tasks complete!\n"); work_queue_delete(q); remove_created_files(); fclose(logfile); return 0; fail: remove_created_files(); fclose(logfile); exit(1); }