Exemplo n.º 1
0
int submit_tasks(struct work_queue *q, int input_size, int run_time, int output_size, int count )
{
	static int ntasks=0;
	char output_file[128];
	char command[256];
	char gen_input_cmd[256];

	/*
	Note that bs=1m and similar are not portable across various
	implementations of dd, so we spell it out as bs=1048576
	*/

	sprintf(gen_input_cmd, "dd if=/dev/zero of=input.0 bs=1048576 count=%d",input_size);
	system(gen_input_cmd);

	int i;
	for(i=0;i<count;i++) {
		sprintf(output_file, "output.%d",ntasks++);
		sprintf(command, "dd if=/dev/zero of=outfile bs=1048576 count=%d; sleep %d", output_size, run_time );

		struct work_queue_task *t = work_queue_task_create(command);
		work_queue_task_specify_file(t, "input.0", "infile", WORK_QUEUE_INPUT, WORK_QUEUE_CACHE);
		work_queue_task_specify_file(t, output_file, "outfile", WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE);
		work_queue_task_specify_cores(t,1);
		work_queue_submit(q, t);
	}

	return 1;
}
Exemplo n.º 2
0
static batch_job_id_t batch_job_wq_submit (struct batch_queue * q, const char *cmd, const char *extra_input_files, const char *extra_output_files, struct jx *envlist, const struct rmsummary *resources)
{
	struct work_queue_task *t;

	int caching_flag = WORK_QUEUE_CACHE;

	if(string_istrue(hash_table_lookup(q->options, "caching"))) {
		caching_flag = WORK_QUEUE_CACHE;
	} else {
		caching_flag = WORK_QUEUE_NOCACHE;
	}

	t = work_queue_task_create(cmd);

	specify_files(t, extra_input_files, extra_output_files, caching_flag);
	specify_envlist(t,envlist);

	if(envlist) {
		const char *category = jx_lookup_string(envlist, "CATEGORY");
		if(category) {
			work_queue_task_specify_category(t, category);
		}
	}

	if(resources)
	{
		work_queue_task_specify_resources(t, resources);
	}

	work_queue_submit(q->data, t);

	return t->taskid;
}
Exemplo n.º 3
0
static batch_job_id_t batch_job_wq_submit (struct batch_queue * q, const char *cmd, const char *extra_input_files, const char *extra_output_files, struct nvpair *envlist )
{
	struct work_queue_task *t;

	int caching_flag = WORK_QUEUE_CACHE;

	if(string_istrue(hash_table_lookup(q->options, "caching"))) {
		caching_flag = WORK_QUEUE_CACHE;
	} else {
		caching_flag = WORK_QUEUE_NOCACHE;
	}

	t = work_queue_task_create(cmd);

	specify_files(t, extra_input_files, extra_output_files, caching_flag);
	specify_envlist(t,envlist);

	struct rmsummary *resources = parse_batch_options_resources(hash_table_lookup(q->options, "batch-options"));
	if(resources)
	{
		work_queue_task_specify_resources(t, resources);
		free(resources);
	}

	work_queue_submit(q->data, t);

	return t->taskid;
}
static int task_consider( int x, int y )
{
	char command[WAVEFRONT_LINE_MAX];
	char tag[WAVEFRONT_LINE_MAX];
	
	struct work_queue_task* t;

	if(x>=xsize) return 1;
	if(y>=ysize) return 1;

	if(text_array_get(array,x,y)) return 0;

	const char *left   = text_array_get(array,x-1,y);
	const char *bottom = text_array_get(array,x,y-1);
	const char *diag   = text_array_get(array,x-1,y-1);

	if(!left || !bottom || !diag) return 1;

	sprintf(command,"./%s %d %d xfile yfile dfile",function,x,y);
	sprintf(tag,"%d %d",x,y);
	
	t = work_queue_task_create(command);
	work_queue_task_specify_tag(t,tag);
	work_queue_task_specify_input_file(t, function, function);
	work_queue_task_specify_input_buf(t, left, strlen(left), "xfile");
	work_queue_task_specify_input_buf(t, bottom, strlen(bottom), "yfile");
	work_queue_task_specify_input_buf(t, diag, strlen(diag), "dfile");
	work_queue_submit(queue,t);

	return 1;
}
Exemplo n.º 5
0
int submit_task(struct work_queue *q, const char *command, const char *executable, const char *infile, off_t infile_offset_start, off_t infile_offset_end, const char *outfile) {
	struct work_queue_task *t;
	int taskid;

	char *infile_dup = strdup(infile); //basename() modifies its arguments. So we need to pass a duplicate.
	char *executable_dup = strdup(executable);
	char *outfile_dup = strdup(outfile);

	t = work_queue_task_create(command);
	if (!work_queue_task_specify_file_piece(t, infile, basename(infile_dup), infile_offset_start, infile_offset_end, WORK_QUEUE_INPUT, WORK_QUEUE_NOCACHE)) {
		fprintf(stderr, "task_specify_file_piece() failed for %s: start offset %ld, end offset %ld.\n", infile, infile_offset_start, infile_offset_end);
		return 0;
	}
	if (!work_queue_task_specify_file(t, executable, basename(executable_dup), WORK_QUEUE_INPUT, WORK_QUEUE_CACHE)) {
		fprintf(stderr, "task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", executable);
		return 0;
	}
	if (!work_queue_task_specify_file(t, outfile, basename(outfile_dup), WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE)) {
		fprintf(stderr, "task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", outfile);
		return 0;
	}

	taskid = work_queue_submit(q, t);
	fprintf(stdout, "submitted task (id# %d): %s\n", taskid, t->command_line);

	free(infile_dup);
	free(executable_dup);
	free(outfile_dup);

	return taskid;
}
int submit_task_series(struct work_queue *q, struct task_series *ts, int series_id) {
	char input_file[128], output_file[128], command[256];
	char gen_input_cmd[256];

	sprintf(input_file, "input-%d", series_id);
	list_push_tail(created_files, xxstrdup(input_file));
	sprintf(gen_input_cmd, "dd if=/dev/zero of=%s bs=1M count=%d", input_file, ts->input_size);
	system(gen_input_cmd);

	// submit tasks to the queue
	int i;
	for(i = 0; i < ts->num_of_tasks; i++) {
		sprintf(output_file, "output-%d-%d", series_id, i);
		list_push_tail(created_files, xxstrdup(output_file));
		sprintf(command, "dd if=/dev/zero of=%s bs=1M count=%d; sleep %d", output_file, ts->output_size, ts->execution_time);

		struct work_queue_task *t = work_queue_task_create(command);
		if (!work_queue_task_specify_file(t, input_file, input_file, WORK_QUEUE_INPUT, WORK_QUEUE_CACHE)) {
			printf("task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", input_file);
			return 0; 	
		}
		if (!work_queue_task_specify_file(t, output_file, output_file, WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE)) {
			printf("task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", output_file);
			return 0; 	
		}	
		int taskid = work_queue_submit(q, t);

		printf("submitted task (id# %d): %s\n", taskid, t->command_line);
	}
	return 1; // success
}
Exemplo n.º 7
0
static void task_submit(struct work_queue *q, int curr_rect_x, int curr_rect_y)
{
    struct work_queue_task *t;

    char rname_x[32];
    char rname_y[32];
    char cmd[255];
    char fname_x[255];
    char fname_y[255];
    char tag[32];

    sprintf(tag, "%03d-%03d", curr_rect_y, curr_rect_x);

    sprintf(rname_x, "rect%03d.cfa", curr_rect_x);

    if(curr_rect_x != curr_rect_y) {
        sprintf(rname_y, "rect%03d.cfa", curr_rect_y);
    } else {
        sprintf(rname_y, "%s", "");
    }

    sprintf(cmd, "./%s %s %s %s", filter_program_name, filter_program_args, rname_x, rname_y);

    // Create the task.
    t = work_queue_task_create(cmd);

    // Specify the tag for this task. Used for identifying which
    // ones are done.
    work_queue_task_specify_tag(t, tag);

    // Send the executable, if it's not already there.
    work_queue_task_specify_file(t, filter_program_path, filter_program_name, WORK_QUEUE_INPUT, WORK_QUEUE_CACHE);

    // Send the repeat file if we need it and it's not already there.
    if(repeat_filename)
        work_queue_task_specify_file(t, repeat_filename, string_basename(repeat_filename), WORK_QUEUE_INPUT, WORK_QUEUE_CACHE);

    // Add the rectangle. Add it as staged, so if the worker
    // already has these sequences, there's no need to send them again.
    sprintf(fname_x, "%s/%s", outdirname, rname_x);
    work_queue_task_specify_file(t, fname_x, rname_x, WORK_QUEUE_INPUT, WORK_QUEUE_CACHE);
    if(curr_rect_x != curr_rect_y) {
        sprintf(fname_y, "%s/%s", outdirname, rname_y);
        work_queue_task_specify_file(t, fname_y, rname_y, WORK_QUEUE_INPUT, WORK_QUEUE_CACHE);
    }

    work_queue_submit(q, t);
    total_submitted++;
    debug(D_DEBUG, "Submitted task for rectangle (%d, %d)\n", curr_rect_y, curr_rect_x);
}
Exemplo n.º 8
0
static void task_complete(struct work_queue_task *t)
{
    checkpoint_task(t);

    if(t->result == 0) {
        debug(D_DEBUG, "task complete: %s: %s", t->tag, t->command_line);
        if(strlen(t->output) > 0) {
            char *out = strdup(t->output);
            char *cand1 = malloc(sizeof(char) * 500);
            char *cand2 = malloc(sizeof(char) * 500);
            int dir, start1, start2;
            char *line = strtok(out, "\n");
            int result = sscanf(line, "%s\t%s\t%d\t%d\t%d", cand1, cand2, &dir, &start1, &start2);
            while(result == 5) {
                cand_count++;
                line = strtok(NULL, "\n");
                if(line == NULL) {
                    break;
                }
                result = sscanf(line, "%s\t%s\t%d\t%d\t%d", cand1, cand2, &dir, &start1, &start2);
            }
            free(out);
            free(cand1);
            free(cand2);
        }
        fputs(t->output, outfile);
        fflush(outfile);
        total_processed++;
        tasks_runtime += (t->time_receive_output_finish - t->time_send_input_start);
        tasks_filetime += t->total_transfer_time;
        work_queue_task_delete(t);
    } else {
        debug(D_DEBUG, "task failed: %s: %s", t->tag, t->command_line);

        if(retry_max > total_retried) {
            debug(D_DEBUG, "retrying task %d/%d", total_retried, retry_max);
            total_retried++;
            work_queue_submit(q, t);
        } else {
            fprintf(stderr, "%s: giving up after retrying %d tasks.\n", progname, retry_max);
            exit(1);
        }
    }
}
int main(int argc, char *argv[])
{
	struct work_queue *q;
	struct work_queue_task *t;
	int port = WORK_QUEUE_DEFAULT_PORT;
	int taskid;
	int i;

	if(argc < 2) {
		printf("work_queue_example <executable> <file1> [file2] [file3] ...\n");
		printf("Each file given on the command line will be compressed using a remote worker.\n");
		return 0;
	}

	debug_flags_set("all");

	q = work_queue_create(port);
	if(!q) {
		printf("couldn't listen on port %d: %s\n", port, strerror(errno));
		return 1;
	}

	printf("listening on port %d...\n", work_queue_port(q));

	for(i = 1; i < argc; i++) {

		char infile[256], outfile[256], command[256];

		sprintf(infile, "%s", argv[i]);
		sprintf(outfile, "%s.gz", argv[i]);
		sprintf(command, "./gzip < %s > %s", infile, outfile);

		t = work_queue_task_create(command);
		if (!work_queue_task_specify_file(t, "/usr/bin/gzip", "gzip", WORK_QUEUE_INPUT, WORK_QUEUE_CACHE)) {
			printf("task_specify_file() failed for /usr/bin/gzip: check if arguments are null or remote name is an absolute path.\n");
			return 1; 	
		}
		if (!work_queue_task_specify_file(t, infile, infile, WORK_QUEUE_INPUT, WORK_QUEUE_NOCACHE)) {
			printf("task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", infile);
			return 1; 	
		}
		if (!work_queue_task_specify_file(t, outfile, outfile, WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE)) {
			printf("task_specify_file() failed for %s: check if arguments are null or remote name is an absolute path.\n", outfile);
			return 1; 	
		}	
		taskid = work_queue_submit(q, t);

		printf("submitted task (id# %d): %s\n", taskid, t->command_line);
	}

	printf("waiting for tasks to complete...\n");

	while(!work_queue_empty(q)) {

		t = work_queue_wait(q, 5);
		if(t) {
			printf("task (id# %d) complete: %s (return code %d)\n", t->taskid, t->command_line, t->return_status);
			work_queue_task_delete(t);
		}
	}

	printf("all tasks complete!\n");

	work_queue_delete(q);

	return 0;
}
Exemplo n.º 10
0
int main(int argc, char **argv)
{
	signed char c;
	struct work_queue *q;
	int port = WORK_QUEUE_DEFAULT_PORT;
	static const char *port_file = NULL;
	int work_queue_master_mode = WORK_QUEUE_MASTER_MODE_STANDALONE;
	char *project = NULL;
	int priority = 0;

	debug_config("allpairs_master");

	extra_files_list = list_create();

	struct option long_options[] = {
		{"debug", required_argument, 0, 'd'},
		{"help",  no_argument, 0, 'h'},
		{"version", no_argument, 0, 'v'},
		{"port", required_argument, 0, 'p'},
		{"random-port", required_argument, 0, 'Z'},
		{"extra-args", required_argument, 0, 'e'},
		{"width", required_argument, 0, 'x'},
		{"height", required_argument, 0, 'y'},
		{"advertise", no_argument, 0, 'a'},    //deprecated, left here for backwards compatibility
		{"project-name", required_argument, 0, 'N'},
		{"debug-file", required_argument, 0, 'o'},
		{"output-file", required_argument, 0, 'O'},
		{"wqstats-file", required_argument, 0, 's'},
		{"input-file", required_argument, 0, 'f'},
		{"estimated-time", required_argument, 0, 't'},
		{"priority", required_argument, 0, 'P'},
        {0,0,0,0}
	};


	while((c = getopt_long(argc, argv, "ad:e:f:hN:p:P:t:vx:y:Z:O:o:s:", long_options, NULL)) >= 0) {
		switch (c) {
	    case 'a':
			work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
			break;
		case 'd':
			debug_flags_set(optarg);
			break;
		case 'e':
			extra_arguments = optarg;
			break;
		case 'f':
			list_push_head(extra_files_list,optarg);
			break;
		case 'o':
			debug_config_file(optarg);
			break;
		case 'O':
			free(output_filename);
			output_filename=xxstrdup(optarg);
			break;
		case 's':
			free(wqstats_filename);
			wqstats_filename=xxstrdup(optarg);
			break;
		case 'h':
			show_help(progname);
			exit(0);
			break;
		case 'N':
			work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
			free(project);
			project = xxstrdup(optarg);
			break;
		case 'p':
			port = atoi(optarg);
			break;
		case 'P':
			priority = atoi(optarg);
			break;
		case 't':
			compare_program_time = atof(optarg);
			break;
		case 'v':
			cctools_version_print(stdout, progname);
			exit(0);
			break;
		case 'x':
			xblock = atoi(optarg);
			break;
		case 'y':
			yblock = atoi(optarg);
			break;
		case 'Z':
			port_file = optarg;
			port = 0;
			break;
		default:
			show_help(progname);
			return 1;
		}
	}

	cctools_version_debug(D_DEBUG, argv[0]);

	if((argc - optind) < 3) {
		show_help(progname);
		exit(1);
	}

	struct text_list *seta = text_list_load(argv[optind]);
	if(!seta) {
		fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
		return 1;
	}

	fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta));

	struct text_list *setb = text_list_load(argv[optind+1]);
	if(!setb) {
		fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
		return 1;
	}

	fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb));

	if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) {
		fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname);
		return 1;
	}

	debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program);
	
	xstop = text_list_size(seta);
	ystop = text_list_size(setb);

	if(allpairs_compare_function_get(argv[optind+2])) {
		strcpy(allpairs_compare_program,argv[optind+2]);
		debug(D_DEBUG,"using internal function %s",allpairs_compare_program);
		use_external_program = 0;
	} else {
		if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) {
			fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program);
			return 1;
		}
		debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program);
		use_external_program = 1;
	}

	if(!xblock || !yblock) {
		estimate_block_size(seta,setb,&xblock,&yblock);
	}

	fprintf(stdout, "%s: using block size of %dx%d\n",progname,xblock,yblock);

	if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) {
		fprintf(stderr, "allpairs: allpairs master running in catalog mode. Please use '-N' option to specify the name of this project.\n");
		fprintf(stderr, "allpairs: Run \"%s -h\" for help with options.\n", argv[0]);
		return 1;
	}

	q = work_queue_create(port);

	//Read the port the queue is actually running, in case we just called
	//work_queue_create(LINK_PORT_ANY)
	port  = work_queue_port(q);

	if(!q) {
		fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno));
		return 1;
	}

	if(port_file)
		opts_write_port_file(port_file, port);

	if(wqstats_filename)
		work_queue_specify_log(q, wqstats_filename);	

	// advanced work queue options
	work_queue_specify_master_mode(q, work_queue_master_mode);
	work_queue_specify_name(q, project);
	work_queue_specify_priority(q, priority);

	fprintf(stdout, "%s: listening for workers on port %d...\n",progname,work_queue_port(q));

	while(1) {
		struct work_queue_task *task = NULL;
		while(work_queue_hungry(q)) {
			task = ap_task_create(seta,setb);
			if(task) {
				work_queue_submit(q, task);
			} else {
				break;
			}
		}

		if(!task && work_queue_empty(q)) break;

		task = work_queue_wait(q,5);
		if(task) task_complete(task);
	}

	work_queue_delete(q);

	return 0;
}
Exemplo n.º 11
0
int main(int argc, char *argv[])
{
	struct work_queue *q;
	struct work_queue_task *t;
	int port = WORK_QUEUE_DEFAULT_PORT;
	int taskid;
	int i;
	char *gzip_path; 

	if(argc < 2) {
		printf("work_queue_example <file1> [file2] [file3] ...\n");
		printf("Each file given on the command line will be compressed using a remote worker.\n");
		return 0;
	}

	/*
	   Usually, we can execute the gzip utility by simply typing its name at a
	   terminal. However, this is not enough for work queue; we have to specify
	   precisely which files need to be transmitted to the workers. We record
	   the location of gzip in 'gzip_path', which is usually found in /bin/gzip
	   or /usr/bin/gzip. We use the 'access' function (from unistd.h standard C
	   library), and test the path for execution (X_OK) and reading (R_OK)
	   permissions. 
	 */
	gzip_path = "/bin/gzip";
	if(access(gzip_path, X_OK | R_OK) != 0) {
		gzip_path = "/usr/bin/gzip";
		if(access(gzip_path, X_OK | R_OK) != 0) {
			fprintf(stderr, "gzip was not found. Please modify the gzip_path variable accordingly. To determine the location of gzip, from the terminal type: which gzip (usual locations are /bin/gzip and /usr/bin/gzip)\n");
			exit(1);
		}
	}

	/* We create the tasks queue using the default port. If this port is
	 * already been used by another program, you can try setting port = 0 to
	 * use an available port.  */
	q = work_queue_create(port);
	if(!q) {
		printf("couldn't listen on port %d: %s\n", port, strerror(errno));
		return 1;
	}
	printf("listening on port %d...\n", work_queue_port(q));

	/* We create and dispatch a task for each filename given in the argument list */
	for(i = 1; i < argc; i++) {

		char infile[256], outfile[256], command[256];

		sprintf(infile, "%s", argv[i]);
		sprintf(outfile, "%s.gz", argv[i]);

		/* Note that we write ./gzip here, to guarantee that the gzip version
		 * we are using is the one being sent to the workers. */
		sprintf(command, "./gzip < %s > %s", infile, outfile);

		t = work_queue_task_create(command);

		/* gzip is the same across all tasks, so we can cache it in the
		 * workers. Note that when specifying a file, we have to name its local
		 * name (e.g. gzip_path), and its remote name (e.g. "gzip"). Unlike the
		 * following line, more often than not these are the same. */
		work_queue_task_specify_file(t, gzip_path, "gzip", WORK_QUEUE_INPUT, WORK_QUEUE_CACHE); 

		/* files to be compressed are different across all tasks, so we do not
		 * cache them. This is, of course, application specific. Sometimes you
		 * may want to cache an output file if is the input of a later task.*/
		work_queue_task_specify_file(t, infile, infile, WORK_QUEUE_INPUT, WORK_QUEUE_NOCACHE);
		work_queue_task_specify_file(t, outfile, outfile, WORK_QUEUE_OUTPUT, WORK_QUEUE_NOCACHE); 

		/* Once all files has been specified, we are ready to submit the task to the queue. */
		taskid = work_queue_submit(q, t);

		printf("submitted task (id# %d): %s\n", taskid, t->command_line);
	}

	printf("waiting for tasks to complete...\n");

	while(!work_queue_empty(q)) {

		/* Application specific code goes here ... */

		/* work_queue_wait waits at most 5 seconds for some task to return. */
		t = work_queue_wait(q, 5);

		if(t) {
			printf("task (id# %d) complete: %s (return code %d)\n", t->taskid, t->command_line, t->return_status);
			if(t->return_status != 0)
			{
				/* The task failed. Error handling (e.g., resubmit with new parameters) here. */
			}

			work_queue_task_delete(t);
		}

		/* Application specific code goes here ... */
	}

	printf("all tasks complete!\n");

	work_queue_delete(q);

	return 0;
}
Exemplo n.º 12
0
int main(int argc, char **argv)
{
	char c;
	struct work_queue *q;
	int port = WORK_QUEUE_DEFAULT_PORT;

	extra_files_list = list_create();

	while((c = getopt(argc, argv, "e:f:t:x:y:p:N:E:d:vh")) != (char) -1) {
		switch (c) {
		case 'e':
			extra_arguments = optarg;
			break;
		case 'f':
			list_push_head(extra_files_list,optarg);
			break;
		case 't':
			compare_program_time = atof(optarg);
			break;
		case 'x':
			xblock = atoi(optarg);
			break;
		case 'y':
			yblock = atoi(optarg);
			break;
		case 'p':
			port = atoi(optarg);
			break;
		case 'N':
			setenv("WORK_QUEUE_NAME", optarg, 1);
			break;
		case 'E':
			setenv("WORK_QUEUE_PRIORITY", optarg, 1);
			break;
		case 'd':
			debug_flags_set(optarg);
			break;
		case 'v':
			show_version(progname);
			exit(0);
			break;
		case 'h':
			show_help(progname);
			exit(0);
			break;
		default:
			show_help(progname);
			return 1;
		}
	}

	if((argc - optind) < 3) {
		show_help(progname);
		exit(1);
	}

	struct text_list *seta = text_list_load(argv[optind]);
	if(!seta) {
		fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
		return 1;
	}

	fprintf(stderr, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta));

	struct text_list *setb = text_list_load(argv[optind+1]);
	if(!setb) {
		fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
		return 1;
	}

	fprintf(stderr, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb));

	if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) {
		fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname);
		return 1;
	}

	debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program);
	
	xstop = text_list_size(seta);
	ystop = text_list_size(setb);

	if(allpairs_compare_function_get(argv[optind+2])) {
		strcpy(allpairs_compare_program,argv[optind+2]);
		debug(D_DEBUG,"using internal function %s",allpairs_compare_program);
		use_external_program = 0;
	} else {
		if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) {
			fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program);
			return 1;
		}
		debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program);
		use_external_program = 1;
	}

	if(!xblock || !yblock) {
		estimate_block_size(seta,setb,&xblock,&yblock);
	}

	fprintf(stderr, "%s: using block size of %dx%d\n",progname,xblock,yblock);

	q = work_queue_create(port);
	if(!q) {
		fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno));
		return 1;
	}

	fprintf(stderr, "%s: listening for workers on port %d...\n",progname,work_queue_port(q));

	while(1) {
		struct work_queue_task *task = NULL;
		while(work_queue_hungry(q)) {
			task = ap_task_create(seta,setb);
			if(task) {
				work_queue_submit(q, task);
			} else {
				break;
			}
		}

		if(!task && work_queue_empty(q)) break;

		task = work_queue_wait(q,5);
		if(task) task_complete(task);
	}

	work_queue_delete(q);

	return 0;
}