Exemple #1
0
static void
aio_check_io(struct ns_worker_ctx *ns_ctx)
{
    int count, i;
    struct timespec timeout;

    timeout.tv_sec = 0;
    timeout.tv_nsec = 0;

    count = io_getevents(ns_ctx->ctx, 1, g_queue_depth, ns_ctx->events, &timeout);
    if (count < 0) {
        fprintf(stderr, "io_getevents error\n");
        exit(1);
    }

    for (i = 0; i < count; i++) {
        task_complete(ns_ctx->events[i].data);
    }
}
static void *insert_thread(void *arg) {
    cuckoo_status st;
    KeyType key;
    ValType val;

    thread_arg_t* th = (thread_arg_t*) arg;
    th->ops         = 0;
    th->failures    = 0;
    th->num_read    = 0;
    th->num_written = 0;
    
    while (keep_writing) {
        size_t i, task;
        task = task_assign();
        if (task >= task_num)
            break;
        for (i = task * task_size + 1; i <= (task + 1) * task_size; i ++) {
            th->ops ++;
            key = (KeyType) i;
            val = (ValType) VALUE(i);
            st = cuckoo_insert(table, (const char*) &key, (const char*) &val);

            if (st == ok) {
                th->num_written ++;

            }
            else {
                printf("[writer%d] unknown error for key %zu (%d)\n", th->id, i, st);
                th->failures ++;
            }
        }
        task_complete(task);        
    }

    pthread_exit(NULL);
}
Exemple #3
0
int main( int argc, char *argv[] )
{
	signed char c;
	int work_queue_master_mode = WORK_QUEUE_MASTER_MODE_STANDALONE;
	char *project = NULL;
	int priority = 0;

	const char *progname = "wavefront";

	debug_config(progname);

	struct option long_options[] = {
		{"help",  no_argument, 0, 'h'},
		{"version", no_argument, 0, 'v'},
		{"debug", required_argument, 0, 'd'},
		{"advertise", no_argument, 0, 'a'},
		{"project-name", required_argument, 0, 'N'},
		{"debug-file", required_argument, 0, 'o'},
		{"port", required_argument, 0, 'p'},
		{"priority", required_argument, 0, 'P'},
		{"estimated-time", required_argument, 0, 't'},
		{"random-port", required_argument, 0, 'Z'},
		{"bitmap", required_argument, 0, 'B'},
		{0,0,0,0}
	};

	while((c=getopt_long(argc,argv,"aB:d:hN:p:P:o:v:Z:", long_options, NULL)) >= 0) {
		switch(c) {
	    	case 'a':
			break;
		case 'd':
			debug_flags_set(optarg);
			break;
		case 'h':
			show_help(progname);
			exit(0);
			break;
		case 'N':
			work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
			free(project);
			project = xxstrdup(optarg);
			break;
		case 'p':
			port = atoi(optarg);
			break;
		case 'P':
			priority = atoi(optarg);
			break;
		case 'o':
			debug_config_file(optarg);
			break;
		case 'v':
			cctools_version_print(stdout, progname);
			exit(0);
			break;
		case 'Z':
			port_file = optarg;
			port = 0;
			break;
		case 'B':
			progress_bitmap_file = optarg;
			break;
		default:
			show_help(progname);
			return 1;
		}
	}

	cctools_version_debug(D_DEBUG, argv[0]);

	if( (argc-optind)!=5 ) {
		show_help(progname);
		exit(1);
	}

	function = argv[optind];
	xsize=atoi(argv[optind+1]);
	ysize=atoi(argv[optind+2]);
	infile=argv[optind+3];
	outfile=argv[optind+4];

	start_time = time(0);
	last_display_time = 0;

	cells_total = xsize*ysize;
	
	xsize++;
	ysize++;

	array = text_array_create(xsize,ysize);
	if(!text_array_load(array,infile)) {
		fprintf(stderr,"couldn't load %s: %s",infile,strerror(errno));
		return 1;
	}

	int count = text_array_load(array,outfile);
	if(count>0) printf("recovered %d results from %s\n",count,outfile);
	
	logfile = fopen(outfile,"a");
	if(!logfile) {
		fprintf(stderr,"couldn't open %s for append: %s\n",outfile,strerror(errno));
		return 1;
	}

	if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) {
		fprintf(stderr, "wavefront: wavefront master running in catalog mode. Please use '-N' option to specify the name of this project.\n");
		fprintf(stderr, "wavefront: Run \"%s -h\" for help with options.\n", argv[0]);
		return 1;
	}

	queue = work_queue_create(port);

	//Read the port the queue is actually running, in case we just called
	//work_queue_create(LINK_PORT_ANY)
	port  = work_queue_port(queue); 

	if(!queue) {
		fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno));
		return 1;
	}

	if(port_file)
		opts_write_port_file(port_file, port);

	// advanced work queue options
	work_queue_specify_master_mode(queue, work_queue_master_mode);
	work_queue_specify_name(queue, project);
	work_queue_specify_priority(queue, priority);

	fprintf(stdout, "%s: listening for workers on port %d...\n",progname,work_queue_port(queue));

	if(progress_bitmap_file)
	{
		bmap = bitmap_create(xsize,ysize);
		wavefront_bitmap_initialize(bmap);
	}
		

	task_prime();

	struct work_queue_task *t;

	while(1) {
		if(time(0)!=last_display_time) display_progress(queue);

		t = work_queue_wait(queue,WORK_QUEUE_WAITFORTASK);
		if(!t) break;
		
		if(t->return_status==0) {
			int x,y;
			if(sscanf(t->tag,"%d %d",&x,&y)==2) {
				text_array_set(array,x,y,t->output);
				task_complete(x,y);
				fprintf(logfile,"%d %d %s\n",x,y,t->output);
				fflush(logfile);
				tasks_done++;
			} else {
				fprintf(stderr,"unexpected output: %s\nfrom command: %s\non host: %s",t->output,t->command_line,t->host);
			}
		} else {
		    fprintf(stderr,"function failed return value (%i) result (%i) on host %s. output:\n%s\n",t->return_status,t->result,t->host,t->output);
		}
		work_queue_task_delete(t);
		if(work_queue_empty(queue))
		    break;
	}

	display_progress(queue);
	return 0;
}
Exemple #4
0
static void
io_complete(void *ctx, const struct nvme_completion *completion)
{
    task_complete((struct perf_task *)ctx);
}
Exemple #5
0
static void
io_complete(void *ctx, const struct spdk_nvme_cpl *completion)
{
	task_complete((struct reset_task *)ctx, completion);
}
int main(int argc, char **argv)
{
    debug_config(progname);

    // By default, turn on fast abort option since we know each job is of very similar size (in terms of runtime).
    // One can also set the fast_abort_multiplier by the '-f' option.
    wq_option_fast_abort_multiplier = 10;

    get_options(argc, argv, progname);

    outfile = fopen(outfilename, "a+");
    if(!outfile) {
        fprintf(stderr, "%s: couldn't open %s: %s\n", progname, outfilename, strerror(errno));
        exit(1);
    }

    if(!find_executable(filter_program_name, "PATH", filter_program_path, sizeof(filter_program_path))) {
        fprintf(stderr, "%s: couldn't find %s in your PATH.\n", progname, filter_program_path);
        exit(1);
    }

    if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) {
        fprintf(stderr, "sand_filter: sand filter master running in catalog mode. Please use '-N' option to specify the name of this project.\n");
        fprintf(stderr, "sand_filter: Run \"%s -h\" for help with options.\n", argv[0]);
        return 1;
    }

    q = work_queue_create(port);
    if(!q) {
        fprintf(stderr, "%s: couldn't listen on port %d: %s\n", progname, port, strerror(errno));
        exit(1);
    }

    port = work_queue_port(q);

    if(port_file) {
        opts_write_port_file(port_file,port);
    }

    // advanced work queue options
    work_queue_specify_master_mode(q, work_queue_master_mode);
    work_queue_specify_name(q, project);
    work_queue_specify_priority(q, priority);

    load_sequences(sequence_filename);
    debug(D_DEBUG, "Sequence loaded.\n", curr_rect_y, curr_rect_x);

    init_checkpoint();

    start_time = time(0);

    int curr_start_x = 0, curr_start_y = 0, curr_rect_x = 0, curr_rect_y = 0;

    while(1) {
        while(work_queue_hungry(q)) {
            if(curr_start_y >= num_seqs)
                break;

            display_progress();

            if(checkpoint[curr_rect_y][curr_rect_x] != CHECKPOINT_STATUS_SUCCESS)
                task_submit(q, curr_rect_x, curr_rect_y);

            // Increment the x rectangle
            curr_rect_x++;
            curr_start_x += rectangle_size;

            // If we've reached the end of a row, move to the
            // next row by incrementing the y rectangle.
            if(curr_start_x >= num_seqs) {
                curr_rect_y++;
                curr_start_y += rectangle_size;
                curr_rect_x = curr_rect_y;
                curr_start_x = curr_rect_x * rectangle_size;
            }
        }

        if(work_queue_empty(q) && curr_start_y >= num_seqs)
            break;

        struct work_queue_task *t = work_queue_wait(q, 5);
        if(t)
            task_complete(t);

        display_progress();
    }

    printf("%s: candidates generated: %lu\n", progname, cand_count);

    if(checkpoint_file) {
        fclose(checkpoint_file);
    }

    fprintf(outfile, "EOF\n");
    fclose(outfile);

    work_queue_delete(q);

    if(!do_not_unlink)
        delete_dir(outdirname);

    return 0;
}
int main( int argc, char *argv[] )
{
	char c;

	const char *progname = "wavefront";

	debug_config(progname);

	while((c=getopt(argc,argv,"p:Pd:o:vh"))!=(char)-1) {
		switch(c) {
			case 'p':
				port = atoi(optarg);
				break;
			case 'd':
				debug_flags_set(optarg);
				break;
			case 'o':
				debug_config_file(optarg);
				break;
			case 'v':
				show_version(progname);
				exit(0);
				break;
			case 'h':
				show_help(progname);
				exit(0);
				break;
		}
	}

	if( (argc-optind)!=5 ) {
		show_help(progname);
		exit(1);
	}

	function = argv[optind];
	xsize=atoi(argv[optind+1]);
	ysize=atoi(argv[optind+2]);
	infile=argv[optind+3];
	outfile=argv[optind+4];

	start_time = time(0);
	last_display_time = 0;

	cells_total = xsize*ysize;
	
	xsize++;
	ysize++;

	array = text_array_create(xsize,ysize);
	if(!text_array_load(array,infile)) {
		fprintf(stderr,"couldn't load %s: %s",infile,strerror(errno));
		return 1;
	}

	int count = text_array_load(array,outfile);
	if(count>0) printf("recovered %d results from %s\n",count,outfile);
	
	logfile = fopen(outfile,"a");
	if(!logfile) {
		fprintf(stderr,"couldn't open %s for append: %s\n",outfile,strerror(errno));
		return 1;
	}

	queue = work_queue_create(port);

	task_prime();

	struct work_queue_task *t;

	while(1) {
		if(time(0)!=last_display_time) display_progress(queue);

		t = work_queue_wait(queue,WORK_QUEUE_WAITFORTASK);
		if(!t) break;
		
		if(t->return_status==0) {
			int x,y;
			if(sscanf(t->tag,"%d %d",&x,&y)==2) {
				text_array_set(array,x,y,t->output);
				task_complete(x,y);
				fprintf(logfile,"%d %d %s\n",x,y,t->output);
				fflush(logfile);
				tasks_done++;
			} else {
				fprintf(stderr,"unexpected output: %s\nfrom command: %s\non host: %s",t->output,t->command_line,t->host);
			}
		} else {
		    fprintf(stderr,"function failed return value (%i) result (%i) on host %s. output:\n%s\n",t->return_status,t->result,t->host,t->output);
		}
		work_queue_task_delete(t);
		if(work_queue_empty(queue))
		    break;
	}

	display_progress(queue);
	return 0;
}
Exemple #8
0
int main(int argc, char **argv)
{
	signed char c;
	struct work_queue *q;
	int port = WORK_QUEUE_DEFAULT_PORT;
	static const char *port_file = NULL;
	int work_queue_master_mode = WORK_QUEUE_MASTER_MODE_STANDALONE;
	char *project = NULL;
	int priority = 0;

	debug_config("allpairs_master");

	extra_files_list = list_create();

	struct option long_options[] = {
		{"debug", required_argument, 0, 'd'},
		{"help",  no_argument, 0, 'h'},
		{"version", no_argument, 0, 'v'},
		{"port", required_argument, 0, 'p'},
		{"random-port", required_argument, 0, 'Z'},
		{"extra-args", required_argument, 0, 'e'},
		{"width", required_argument, 0, 'x'},
		{"height", required_argument, 0, 'y'},
		{"advertise", no_argument, 0, 'a'},    //deprecated, left here for backwards compatibility
		{"project-name", required_argument, 0, 'N'},
		{"debug-file", required_argument, 0, 'o'},
		{"output-file", required_argument, 0, 'O'},
		{"wqstats-file", required_argument, 0, 's'},
		{"input-file", required_argument, 0, 'f'},
		{"estimated-time", required_argument, 0, 't'},
		{"priority", required_argument, 0, 'P'},
        {0,0,0,0}
	};


	while((c = getopt_long(argc, argv, "ad:e:f:hN:p:P:t:vx:y:Z:O:o:s:", long_options, NULL)) >= 0) {
		switch (c) {
	    case 'a':
			work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
			break;
		case 'd':
			debug_flags_set(optarg);
			break;
		case 'e':
			extra_arguments = optarg;
			break;
		case 'f':
			list_push_head(extra_files_list,optarg);
			break;
		case 'o':
			debug_config_file(optarg);
			break;
		case 'O':
			free(output_filename);
			output_filename=xxstrdup(optarg);
			break;
		case 's':
			free(wqstats_filename);
			wqstats_filename=xxstrdup(optarg);
			break;
		case 'h':
			show_help(progname);
			exit(0);
			break;
		case 'N':
			work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
			free(project);
			project = xxstrdup(optarg);
			break;
		case 'p':
			port = atoi(optarg);
			break;
		case 'P':
			priority = atoi(optarg);
			break;
		case 't':
			compare_program_time = atof(optarg);
			break;
		case 'v':
			cctools_version_print(stdout, progname);
			exit(0);
			break;
		case 'x':
			xblock = atoi(optarg);
			break;
		case 'y':
			yblock = atoi(optarg);
			break;
		case 'Z':
			port_file = optarg;
			port = 0;
			break;
		default:
			show_help(progname);
			return 1;
		}
	}

	cctools_version_debug(D_DEBUG, argv[0]);

	if((argc - optind) < 3) {
		show_help(progname);
		exit(1);
	}

	struct text_list *seta = text_list_load(argv[optind]);
	if(!seta) {
		fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
		return 1;
	}

	fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta));

	struct text_list *setb = text_list_load(argv[optind+1]);
	if(!setb) {
		fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
		return 1;
	}

	fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb));

	if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) {
		fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname);
		return 1;
	}

	debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program);
	
	xstop = text_list_size(seta);
	ystop = text_list_size(setb);

	if(allpairs_compare_function_get(argv[optind+2])) {
		strcpy(allpairs_compare_program,argv[optind+2]);
		debug(D_DEBUG,"using internal function %s",allpairs_compare_program);
		use_external_program = 0;
	} else {
		if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) {
			fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program);
			return 1;
		}
		debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program);
		use_external_program = 1;
	}

	if(!xblock || !yblock) {
		estimate_block_size(seta,setb,&xblock,&yblock);
	}

	fprintf(stdout, "%s: using block size of %dx%d\n",progname,xblock,yblock);

	if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) {
		fprintf(stderr, "allpairs: allpairs master running in catalog mode. Please use '-N' option to specify the name of this project.\n");
		fprintf(stderr, "allpairs: Run \"%s -h\" for help with options.\n", argv[0]);
		return 1;
	}

	q = work_queue_create(port);

	//Read the port the queue is actually running, in case we just called
	//work_queue_create(LINK_PORT_ANY)
	port  = work_queue_port(q);

	if(!q) {
		fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno));
		return 1;
	}

	if(port_file)
		opts_write_port_file(port_file, port);

	if(wqstats_filename)
		work_queue_specify_log(q, wqstats_filename);	

	// advanced work queue options
	work_queue_specify_master_mode(q, work_queue_master_mode);
	work_queue_specify_name(q, project);
	work_queue_specify_priority(q, priority);

	fprintf(stdout, "%s: listening for workers on port %d...\n",progname,work_queue_port(q));

	while(1) {
		struct work_queue_task *task = NULL;
		while(work_queue_hungry(q)) {
			task = ap_task_create(seta,setb);
			if(task) {
				work_queue_submit(q, task);
			} else {
				break;
			}
		}

		if(!task && work_queue_empty(q)) break;

		task = work_queue_wait(q,5);
		if(task) task_complete(task);
	}

	work_queue_delete(q);

	return 0;
}
int main(int argc, char **argv)
{
	char c;
	struct work_queue *q;
	int port = WORK_QUEUE_DEFAULT_PORT;

	extra_files_list = list_create();

	while((c = getopt(argc, argv, "e:f:t:x:y:p:N:E:d:vh")) != (char) -1) {
		switch (c) {
		case 'e':
			extra_arguments = optarg;
			break;
		case 'f':
			list_push_head(extra_files_list,optarg);
			break;
		case 't':
			compare_program_time = atof(optarg);
			break;
		case 'x':
			xblock = atoi(optarg);
			break;
		case 'y':
			yblock = atoi(optarg);
			break;
		case 'p':
			port = atoi(optarg);
			break;
		case 'N':
			setenv("WORK_QUEUE_NAME", optarg, 1);
			break;
		case 'E':
			setenv("WORK_QUEUE_PRIORITY", optarg, 1);
			break;
		case 'd':
			debug_flags_set(optarg);
			break;
		case 'v':
			show_version(progname);
			exit(0);
			break;
		case 'h':
			show_help(progname);
			exit(0);
			break;
		default:
			show_help(progname);
			return 1;
		}
	}

	if((argc - optind) < 3) {
		show_help(progname);
		exit(1);
	}

	struct text_list *seta = text_list_load(argv[optind]);
	if(!seta) {
		fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
		return 1;
	}

	fprintf(stderr, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta));

	struct text_list *setb = text_list_load(argv[optind+1]);
	if(!setb) {
		fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
		return 1;
	}

	fprintf(stderr, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb));

	if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) {
		fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname);
		return 1;
	}

	debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program);
	
	xstop = text_list_size(seta);
	ystop = text_list_size(setb);

	if(allpairs_compare_function_get(argv[optind+2])) {
		strcpy(allpairs_compare_program,argv[optind+2]);
		debug(D_DEBUG,"using internal function %s",allpairs_compare_program);
		use_external_program = 0;
	} else {
		if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) {
			fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program);
			return 1;
		}
		debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program);
		use_external_program = 1;
	}

	if(!xblock || !yblock) {
		estimate_block_size(seta,setb,&xblock,&yblock);
	}

	fprintf(stderr, "%s: using block size of %dx%d\n",progname,xblock,yblock);

	q = work_queue_create(port);
	if(!q) {
		fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno));
		return 1;
	}

	fprintf(stderr, "%s: listening for workers on port %d...\n",progname,work_queue_port(q));

	while(1) {
		struct work_queue_task *task = NULL;
		while(work_queue_hungry(q)) {
			task = ap_task_create(seta,setb);
			if(task) {
				work_queue_submit(q, task);
			} else {
				break;
			}
		}

		if(!task && work_queue_empty(q)) break;

		task = work_queue_wait(q,5);
		if(task) task_complete(task);
	}

	work_queue_delete(q);

	return 0;
}