Beispiel #1
0
void estimate_block_size( struct text_list *seta, struct text_list *setb, int *xblock, int *yblock )
{
	if(compare_program_time==0) {
		if(use_external_program) {
			compare_program_time = estimate_run_time(seta,setb);
		} else {
			compare_program_time = 0.1;
		}
	}

	fprintf(stdout, "%s: %s estimated at %.02lfs per comparison\n",progname,allpairs_compare_program,compare_program_time);

	int block_limit = 60;
	double block_time;

	*xblock = *yblock = 1;

	while(1) {
		block_time = *xblock * *yblock * compare_program_time;
		if(block_time>block_limit) break;

		if(*xblock < text_list_size(seta)) (*xblock)++;
		if(*yblock < text_list_size(setb)) (*yblock)++;

		if(*xblock==text_list_size(seta) && *yblock==text_list_size(setb)) break;
	}
}
Beispiel #2
0
int block_size_estimate( struct text_list *seta )
{
	int count = MIN(100,text_list_size(seta));
	int i;
	UINT64_T total_data = 0,free_mem,total_mem;
	int block_size;

	host_memory_info_get(&free_mem, &total_mem);

	for(i=0;i<count;i++) {
		total_data += get_file_size(text_list_get(seta,i));
	}

	total_mem = total_mem/2;

	if(total_data>=total_mem) {
		block_size = text_list_size(seta) * total_mem / total_data;
		if(block_size<1) block_size = 1;
		if(block_size>text_list_size(seta)) block_size = text_list_size(seta);
	} else {
		block_size = text_list_size(seta);
	}

	return block_size;
}
Beispiel #3
0
static int main_loop_program( const char *funcpath, struct text_list *seta, struct text_list *setb )
{
	int x,i,j,c;
	char line[1024];
	FILE *proc[num_cores];

	int xstop = text_list_size(seta);

	/* for each block sized vertical stripe... */
	for(x=0;x<xstop;x+=block_size) {

		/* for each row in the stripe ... */
		for(j=0;j<text_list_size(setb);j++) {

			/* for each group of num_cores in the stripe... */
			for(i=x;i<(x+block_size);i+=num_cores) {

				/* make sure we don't run past the block width */
				int n = MIN(num_cores,x+block_size-i);

				/* make sure we don't run off the width of the array */
				n = MIN(n,xstop-i);

				/* start one process for each core */
				/* if nindex = 2, the real xindex = index_array[0] + i; the real yindex = index_array[1] + j. */
				for(c=0;c<n;c++) {
					if((nindex == 2 && (index_array[0] + i + c) <= (index_array[1] + j)) || //calcuate xindex and yindex of the unit in the original matrix of allpairs_master
						(is_symmetric == 0 && nindex == 0) ||
						(is_symmetric && (i+c) <= j)) {
							sprintf(line,"%s %s %s %s\n",funcpath,extra_arguments,text_list_get(seta,i+c),text_list_get(setb,j));
							proc[c] = fast_popen(line);
							if(!proc[c]) {
								fprintf(stderr,"%s: couldn't execute %s: %s\n",progname,line,strerror(errno));
								return 1;
							}
					}
				}

				/* then finish one process for each core */
				for(c=0;c<n;c++) {
					if((nindex == 2 && (index_array[0] + i + c) <= (index_array[1] + j)) ||
						(is_symmetric == 0 && nindex == 0) ||
						(is_symmetric && (i+c) <= j)) {
							printf("%s\t%s\t",text_list_get(seta,i+c),text_list_get(setb,j));
							int lines = 0;
							while(fgets(line,sizeof(line),proc[c])) {
								printf("%s",line);
								lines++;
							}
							if(lines==0) printf("\n");
							fast_pclose(proc[c]);
					}
				}
			}
		}
	}

	return 0;
}
Beispiel #4
0
static int main_loop_threaded( allpairs_compare_t funcptr, struct text_list *seta, struct text_list *setb )
{
	int x,i,j,c;

	char *xname[block_size];
	char *xdata[block_size];
	int xdata_length[block_size];

	char *yname[num_cores];
	char *ydata[num_cores];
	int ydata_length[num_cores];

	struct thread_args args[num_cores];
	pthread_t thread[num_cores];

	/* for each block sized vertical stripe... */
	for(x=0;x<text_list_size(seta);x+=block_size) {

		/* load the horizontal members of the stripe */
		for(i=0;i<block_size;i++) {
			xname[i] = text_list_get(seta,x+i);
			xdata[i] = load_one_file(text_list_get(seta,x+i),&xdata_length[i]);
		}

		/* for each row in the stripe ... */
		for(j=0;j<text_list_size(setb);j+=num_cores) {

			/* don't start more threads than rows remaining. */
			int n = MIN(num_cores,text_list_size(setb)-j);

			/* start one thread working on a whole row. */
			for(c=0;c<n;c++) {
				yname[c] = text_list_get(setb,j+c);
				ydata[c] = load_one_file(text_list_get(setb,j+c),&ydata_length[c]);

				args[c].func         = funcptr;
				args[c].xname        = xname;
				args[c].xdata        = xdata;
				args[c].xdata_length = xdata_length;
				args[c].yname	     = yname[c];
				args[c].ydata        = ydata[c];
				args[c].ydata_length = ydata_length[c];
				pthread_create(&thread[c],0,row_loop_threaded,&args[c]);
			}

			/* wait for each one to finish */
			for(c=0;c<n;c++) {
				pthread_join(thread[c],0);
				free(ydata[c]);
			}
		}

		for(i=0;i<block_size;i++) {
			free(xdata[i]);
		}
	}

	return 0;
}
Beispiel #5
0
int main(int argc, char **argv)
{
	signed char c;
	struct work_queue *q;
	int port = WORK_QUEUE_DEFAULT_PORT;
	static const char *port_file = NULL;
	int work_queue_master_mode = WORK_QUEUE_MASTER_MODE_STANDALONE;
	char *project = NULL;
	int priority = 0;

	debug_config("allpairs_master");

	extra_files_list = list_create();

	struct option long_options[] = {
		{"debug", required_argument, 0, 'd'},
		{"help",  no_argument, 0, 'h'},
		{"version", no_argument, 0, 'v'},
		{"port", required_argument, 0, 'p'},
		{"random-port", required_argument, 0, 'Z'},
		{"extra-args", required_argument, 0, 'e'},
		{"width", required_argument, 0, 'x'},
		{"height", required_argument, 0, 'y'},
		{"advertise", no_argument, 0, 'a'},    //deprecated, left here for backwards compatibility
		{"project-name", required_argument, 0, 'N'},
		{"debug-file", required_argument, 0, 'o'},
		{"output-file", required_argument, 0, 'O'},
		{"wqstats-file", required_argument, 0, 's'},
		{"input-file", required_argument, 0, 'f'},
		{"estimated-time", required_argument, 0, 't'},
		{"priority", required_argument, 0, 'P'},
        {0,0,0,0}
	};


	while((c = getopt_long(argc, argv, "ad:e:f:hN:p:P:t:vx:y:Z:O:o:s:", long_options, NULL)) >= 0) {
		switch (c) {
	    case 'a':
			work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
			break;
		case 'd':
			debug_flags_set(optarg);
			break;
		case 'e':
			extra_arguments = optarg;
			break;
		case 'f':
			list_push_head(extra_files_list,optarg);
			break;
		case 'o':
			debug_config_file(optarg);
			break;
		case 'O':
			free(output_filename);
			output_filename=xxstrdup(optarg);
			break;
		case 's':
			free(wqstats_filename);
			wqstats_filename=xxstrdup(optarg);
			break;
		case 'h':
			show_help(progname);
			exit(0);
			break;
		case 'N':
			work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
			free(project);
			project = xxstrdup(optarg);
			break;
		case 'p':
			port = atoi(optarg);
			break;
		case 'P':
			priority = atoi(optarg);
			break;
		case 't':
			compare_program_time = atof(optarg);
			break;
		case 'v':
			cctools_version_print(stdout, progname);
			exit(0);
			break;
		case 'x':
			xblock = atoi(optarg);
			break;
		case 'y':
			yblock = atoi(optarg);
			break;
		case 'Z':
			port_file = optarg;
			port = 0;
			break;
		default:
			show_help(progname);
			return 1;
		}
	}

	cctools_version_debug(D_DEBUG, argv[0]);

	if((argc - optind) < 3) {
		show_help(progname);
		exit(1);
	}

	struct text_list *seta = text_list_load(argv[optind]);
	if(!seta) {
		fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
		return 1;
	}

	fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta));

	struct text_list *setb = text_list_load(argv[optind+1]);
	if(!setb) {
		fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
		return 1;
	}

	fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb));

	if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) {
		fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname);
		return 1;
	}

	debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program);
	
	xstop = text_list_size(seta);
	ystop = text_list_size(setb);

	if(allpairs_compare_function_get(argv[optind+2])) {
		strcpy(allpairs_compare_program,argv[optind+2]);
		debug(D_DEBUG,"using internal function %s",allpairs_compare_program);
		use_external_program = 0;
	} else {
		if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) {
			fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program);
			return 1;
		}
		debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program);
		use_external_program = 1;
	}

	if(!xblock || !yblock) {
		estimate_block_size(seta,setb,&xblock,&yblock);
	}

	fprintf(stdout, "%s: using block size of %dx%d\n",progname,xblock,yblock);

	if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) {
		fprintf(stderr, "allpairs: allpairs master running in catalog mode. Please use '-N' option to specify the name of this project.\n");
		fprintf(stderr, "allpairs: Run \"%s -h\" for help with options.\n", argv[0]);
		return 1;
	}

	q = work_queue_create(port);

	//Read the port the queue is actually running, in case we just called
	//work_queue_create(LINK_PORT_ANY)
	port  = work_queue_port(q);

	if(!q) {
		fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno));
		return 1;
	}

	if(port_file)
		opts_write_port_file(port_file, port);

	if(wqstats_filename)
		work_queue_specify_log(q, wqstats_filename);	

	// advanced work queue options
	work_queue_specify_master_mode(q, work_queue_master_mode);
	work_queue_specify_name(q, project);
	work_queue_specify_priority(q, priority);

	fprintf(stdout, "%s: listening for workers on port %d...\n",progname,work_queue_port(q));

	while(1) {
		struct work_queue_task *task = NULL;
		while(work_queue_hungry(q)) {
			task = ap_task_create(seta,setb);
			if(task) {
				work_queue_submit(q, task);
			} else {
				break;
			}
		}

		if(!task && work_queue_empty(q)) break;

		task = work_queue_wait(q,5);
		if(task) task_complete(task);
	}

	work_queue_delete(q);

	return 0;
}
Beispiel #6
0
static batch_job_id_t batch_job_mesos_submit (struct batch_queue *q, const char *cmd, 
	const char *extra_input_files, const char *extra_output_files, 
	struct jx *envlist, const struct rmsummary *resources )
{

	// Get the path to mesos python site-packages
	if (!is_mesos_py_path_known) {
		mesos_py_path = batch_queue_get_option(q, "mesos-path");
		if (mesos_py_path != NULL) {
			debug(D_INFO, "Get mesos_path %s from command line\n", mesos_py_path);
		}
		is_mesos_py_path_known = 1;
	}

	// Get the mesos master address
	if (!is_mesos_master_known) {
		mesos_master = batch_queue_get_option(q, "mesos-master");
		if (mesos_master == NULL) {
			fatal("Please specify the hostname of mesos master by using --mesos-master");
		} else {
			debug(D_INFO, "Get mesos_path %s from command line\n", mesos_py_path);
			is_mesos_master_known = 1;
		}
	}

	mesos_preload = batch_queue_get_option(q, "mesos-preload");

	if (is_mesos_py_path_known && 
		is_mesos_master_known && 
		!is_scheduler_running ) {
		// start mesos scheduler if it is not running
		start_mesos_scheduler(q);
		is_scheduler_running = 1;
	}

	int task_id = ++counter;

	debug(D_BATCH, "task %d is ready", task_id);
	struct batch_job_info *info = calloc(1, sizeof(*info));
	info->started = time(0);
	info->submitted = time(0);
	itable_insert(q->job_table, task_id, info);

	// write the ready task information as  
	// "task_id, task_cmd, inputs, outputs" to 
	// mesos_task_info, which will be scanned by 
	// mf_mesos_scheduler later. 

	FILE *task_info_fp;

	if(access(FILE_TASK_INFO, F_OK) != -1) {
		task_info_fp = fopen(FILE_TASK_INFO, "a+");
	} else {
		task_info_fp = fopen(FILE_TASK_INFO, "w+");
	}
	
	struct mesos_task *mt = mesos_task_create(task_id, cmd, extra_input_files, extra_output_files);

	fprintf(task_info_fp, "%d,%s,", mt->task_id, mt->task_cmd);

	if (extra_input_files != NULL && strlen(extra_input_files) != 0) {

		int j = 0;
		int num_input_files = text_list_size(mt->task_input_files);
		for (j = 0; j < (num_input_files-1); j++) {
			fprintf(task_info_fp, "%s ", text_list_get(mt->task_input_files, j));
		}
		fprintf(task_info_fp, "%s,", text_list_get(mt->task_input_files, num_input_files-1));

	} else {
		fprintf(task_info_fp, ",");
	}
    
	if (extra_output_files != NULL && strlen(extra_output_files) != 0) {
		int j = 0;
		int num_output_files = text_list_size(mt->task_output_files);
		for (j = 0; j < (num_output_files-1); j++) {
			fprintf(task_info_fp, "%s ", text_list_get(mt->task_output_files, j));
		}
		fprintf(task_info_fp, "%s,",text_list_get(mt->task_output_files, num_output_files-1));
	} else {
		fprintf(task_info_fp, ",");	
	}

	// The default resource requirements for each task
	int64_t cores = -1;
	int64_t memory = -1;
	int64_t disk = -1;

	if (resources) {
		cores  = resources->cores  > -1 ? resources->cores  : cores;
		memory = resources->memory > -1 ? resources->memory : memory;
		disk   = resources->disk   > -1 ? resources->disk   : disk;
	}

	fprintf(task_info_fp, "%" PRId64 ",%" PRId64 ",%" PRId64 ",", cores, memory, disk);

	fputs("submitted\n", task_info_fp);

	mesos_task_delete(mt);
	fclose(task_info_fp);

	return task_id;
}
int main(int argc, char **argv)
{
	char c;
	struct work_queue *q;
	int port = WORK_QUEUE_DEFAULT_PORT;

	extra_files_list = list_create();

	while((c = getopt(argc, argv, "e:f:t:x:y:p:N:E:d:vh")) != (char) -1) {
		switch (c) {
		case 'e':
			extra_arguments = optarg;
			break;
		case 'f':
			list_push_head(extra_files_list,optarg);
			break;
		case 't':
			compare_program_time = atof(optarg);
			break;
		case 'x':
			xblock = atoi(optarg);
			break;
		case 'y':
			yblock = atoi(optarg);
			break;
		case 'p':
			port = atoi(optarg);
			break;
		case 'N':
			setenv("WORK_QUEUE_NAME", optarg, 1);
			break;
		case 'E':
			setenv("WORK_QUEUE_PRIORITY", optarg, 1);
			break;
		case 'd':
			debug_flags_set(optarg);
			break;
		case 'v':
			show_version(progname);
			exit(0);
			break;
		case 'h':
			show_help(progname);
			exit(0);
			break;
		default:
			show_help(progname);
			return 1;
		}
	}

	if((argc - optind) < 3) {
		show_help(progname);
		exit(1);
	}

	struct text_list *seta = text_list_load(argv[optind]);
	if(!seta) {
		fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
		return 1;
	}

	fprintf(stderr, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta));

	struct text_list *setb = text_list_load(argv[optind+1]);
	if(!setb) {
		fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
		return 1;
	}

	fprintf(stderr, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb));

	if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) {
		fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname);
		return 1;
	}

	debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program);
	
	xstop = text_list_size(seta);
	ystop = text_list_size(setb);

	if(allpairs_compare_function_get(argv[optind+2])) {
		strcpy(allpairs_compare_program,argv[optind+2]);
		debug(D_DEBUG,"using internal function %s",allpairs_compare_program);
		use_external_program = 0;
	} else {
		if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) {
			fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program);
			return 1;
		}
		debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program);
		use_external_program = 1;
	}

	if(!xblock || !yblock) {
		estimate_block_size(seta,setb,&xblock,&yblock);
	}

	fprintf(stderr, "%s: using block size of %dx%d\n",progname,xblock,yblock);

	q = work_queue_create(port);
	if(!q) {
		fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno));
		return 1;
	}

	fprintf(stderr, "%s: listening for workers on port %d...\n",progname,work_queue_port(q));

	while(1) {
		struct work_queue_task *task = NULL;
		while(work_queue_hungry(q)) {
			task = ap_task_create(seta,setb);
			if(task) {
				work_queue_submit(q, task);
			} else {
				break;
			}
		}

		if(!task && work_queue_empty(q)) break;

		task = work_queue_wait(q,5);
		if(task) task_complete(task);
	}

	work_queue_delete(q);

	return 0;
}