void estimate_block_size( struct text_list *seta, struct text_list *setb, int *xblock, int *yblock ) { if(compare_program_time==0) { if(use_external_program) { compare_program_time = estimate_run_time(seta,setb); } else { compare_program_time = 0.1; } } fprintf(stdout, "%s: %s estimated at %.02lfs per comparison\n",progname,allpairs_compare_program,compare_program_time); int block_limit = 60; double block_time; *xblock = *yblock = 1; while(1) { block_time = *xblock * *yblock * compare_program_time; if(block_time>block_limit) break; if(*xblock < text_list_size(seta)) (*xblock)++; if(*yblock < text_list_size(setb)) (*yblock)++; if(*xblock==text_list_size(seta) && *yblock==text_list_size(setb)) break; } }
int block_size_estimate( struct text_list *seta ) { int count = MIN(100,text_list_size(seta)); int i; UINT64_T total_data = 0,free_mem,total_mem; int block_size; host_memory_info_get(&free_mem, &total_mem); for(i=0;i<count;i++) { total_data += get_file_size(text_list_get(seta,i)); } total_mem = total_mem/2; if(total_data>=total_mem) { block_size = text_list_size(seta) * total_mem / total_data; if(block_size<1) block_size = 1; if(block_size>text_list_size(seta)) block_size = text_list_size(seta); } else { block_size = text_list_size(seta); } return block_size; }
static int main_loop_program( const char *funcpath, struct text_list *seta, struct text_list *setb ) { int x,i,j,c; char line[1024]; FILE *proc[num_cores]; int xstop = text_list_size(seta); /* for each block sized vertical stripe... */ for(x=0;x<xstop;x+=block_size) { /* for each row in the stripe ... */ for(j=0;j<text_list_size(setb);j++) { /* for each group of num_cores in the stripe... */ for(i=x;i<(x+block_size);i+=num_cores) { /* make sure we don't run past the block width */ int n = MIN(num_cores,x+block_size-i); /* make sure we don't run off the width of the array */ n = MIN(n,xstop-i); /* start one process for each core */ /* if nindex = 2, the real xindex = index_array[0] + i; the real yindex = index_array[1] + j. */ for(c=0;c<n;c++) { if((nindex == 2 && (index_array[0] + i + c) <= (index_array[1] + j)) || //calcuate xindex and yindex of the unit in the original matrix of allpairs_master (is_symmetric == 0 && nindex == 0) || (is_symmetric && (i+c) <= j)) { sprintf(line,"%s %s %s %s\n",funcpath,extra_arguments,text_list_get(seta,i+c),text_list_get(setb,j)); proc[c] = fast_popen(line); if(!proc[c]) { fprintf(stderr,"%s: couldn't execute %s: %s\n",progname,line,strerror(errno)); return 1; } } } /* then finish one process for each core */ for(c=0;c<n;c++) { if((nindex == 2 && (index_array[0] + i + c) <= (index_array[1] + j)) || (is_symmetric == 0 && nindex == 0) || (is_symmetric && (i+c) <= j)) { printf("%s\t%s\t",text_list_get(seta,i+c),text_list_get(setb,j)); int lines = 0; while(fgets(line,sizeof(line),proc[c])) { printf("%s",line); lines++; } if(lines==0) printf("\n"); fast_pclose(proc[c]); } } } } } return 0; }
static int main_loop_threaded( allpairs_compare_t funcptr, struct text_list *seta, struct text_list *setb ) { int x,i,j,c; char *xname[block_size]; char *xdata[block_size]; int xdata_length[block_size]; char *yname[num_cores]; char *ydata[num_cores]; int ydata_length[num_cores]; struct thread_args args[num_cores]; pthread_t thread[num_cores]; /* for each block sized vertical stripe... */ for(x=0;x<text_list_size(seta);x+=block_size) { /* load the horizontal members of the stripe */ for(i=0;i<block_size;i++) { xname[i] = text_list_get(seta,x+i); xdata[i] = load_one_file(text_list_get(seta,x+i),&xdata_length[i]); } /* for each row in the stripe ... */ for(j=0;j<text_list_size(setb);j+=num_cores) { /* don't start more threads than rows remaining. */ int n = MIN(num_cores,text_list_size(setb)-j); /* start one thread working on a whole row. */ for(c=0;c<n;c++) { yname[c] = text_list_get(setb,j+c); ydata[c] = load_one_file(text_list_get(setb,j+c),&ydata_length[c]); args[c].func = funcptr; args[c].xname = xname; args[c].xdata = xdata; args[c].xdata_length = xdata_length; args[c].yname = yname[c]; args[c].ydata = ydata[c]; args[c].ydata_length = ydata_length[c]; pthread_create(&thread[c],0,row_loop_threaded,&args[c]); } /* wait for each one to finish */ for(c=0;c<n;c++) { pthread_join(thread[c],0); free(ydata[c]); } } for(i=0;i<block_size;i++) { free(xdata[i]); } } return 0; }
int main(int argc, char **argv) { signed char c; struct work_queue *q; int port = WORK_QUEUE_DEFAULT_PORT; static const char *port_file = NULL; int work_queue_master_mode = WORK_QUEUE_MASTER_MODE_STANDALONE; char *project = NULL; int priority = 0; debug_config("allpairs_master"); extra_files_list = list_create(); struct option long_options[] = { {"debug", required_argument, 0, 'd'}, {"help", no_argument, 0, 'h'}, {"version", no_argument, 0, 'v'}, {"port", required_argument, 0, 'p'}, {"random-port", required_argument, 0, 'Z'}, {"extra-args", required_argument, 0, 'e'}, {"width", required_argument, 0, 'x'}, {"height", required_argument, 0, 'y'}, {"advertise", no_argument, 0, 'a'}, //deprecated, left here for backwards compatibility {"project-name", required_argument, 0, 'N'}, {"debug-file", required_argument, 0, 'o'}, {"output-file", required_argument, 0, 'O'}, {"wqstats-file", required_argument, 0, 's'}, {"input-file", required_argument, 0, 'f'}, {"estimated-time", required_argument, 0, 't'}, {"priority", required_argument, 0, 'P'}, {0,0,0,0} }; while((c = getopt_long(argc, argv, "ad:e:f:hN:p:P:t:vx:y:Z:O:o:s:", long_options, NULL)) >= 0) { switch (c) { case 'a': work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG; break; case 'd': debug_flags_set(optarg); break; case 'e': extra_arguments = optarg; break; case 'f': list_push_head(extra_files_list,optarg); break; case 'o': debug_config_file(optarg); break; case 'O': free(output_filename); output_filename=xxstrdup(optarg); break; case 's': free(wqstats_filename); wqstats_filename=xxstrdup(optarg); break; case 'h': show_help(progname); exit(0); break; case 'N': work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG; free(project); project = xxstrdup(optarg); break; case 'p': port = atoi(optarg); break; case 'P': priority = atoi(optarg); break; case 't': compare_program_time = atof(optarg); break; case 'v': cctools_version_print(stdout, progname); exit(0); break; case 'x': xblock = atoi(optarg); break; case 'y': yblock = atoi(optarg); break; case 'Z': port_file = optarg; port = 0; break; default: show_help(progname); return 1; } } cctools_version_debug(D_DEBUG, argv[0]); if((argc - optind) < 3) { show_help(progname); exit(1); } struct text_list *seta = text_list_load(argv[optind]); if(!seta) { fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno)); return 1; } fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta)); struct text_list *setb = text_list_load(argv[optind+1]); if(!setb) { fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno)); return 1; } fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb)); if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) { fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname); return 1; } debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program); xstop = text_list_size(seta); ystop = text_list_size(setb); if(allpairs_compare_function_get(argv[optind+2])) { strcpy(allpairs_compare_program,argv[optind+2]); debug(D_DEBUG,"using internal function %s",allpairs_compare_program); use_external_program = 0; } else { if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) { fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program); return 1; } debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program); use_external_program = 1; } if(!xblock || !yblock) { estimate_block_size(seta,setb,&xblock,&yblock); } fprintf(stdout, "%s: using block size of %dx%d\n",progname,xblock,yblock); if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) { fprintf(stderr, "allpairs: allpairs master running in catalog mode. Please use '-N' option to specify the name of this project.\n"); fprintf(stderr, "allpairs: Run \"%s -h\" for help with options.\n", argv[0]); return 1; } q = work_queue_create(port); //Read the port the queue is actually running, in case we just called //work_queue_create(LINK_PORT_ANY) port = work_queue_port(q); if(!q) { fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno)); return 1; } if(port_file) opts_write_port_file(port_file, port); if(wqstats_filename) work_queue_specify_log(q, wqstats_filename); // advanced work queue options work_queue_specify_master_mode(q, work_queue_master_mode); work_queue_specify_name(q, project); work_queue_specify_priority(q, priority); fprintf(stdout, "%s: listening for workers on port %d...\n",progname,work_queue_port(q)); while(1) { struct work_queue_task *task = NULL; while(work_queue_hungry(q)) { task = ap_task_create(seta,setb); if(task) { work_queue_submit(q, task); } else { break; } } if(!task && work_queue_empty(q)) break; task = work_queue_wait(q,5); if(task) task_complete(task); } work_queue_delete(q); return 0; }
static batch_job_id_t batch_job_mesos_submit (struct batch_queue *q, const char *cmd, const char *extra_input_files, const char *extra_output_files, struct jx *envlist, const struct rmsummary *resources ) { // Get the path to mesos python site-packages if (!is_mesos_py_path_known) { mesos_py_path = batch_queue_get_option(q, "mesos-path"); if (mesos_py_path != NULL) { debug(D_INFO, "Get mesos_path %s from command line\n", mesos_py_path); } is_mesos_py_path_known = 1; } // Get the mesos master address if (!is_mesos_master_known) { mesos_master = batch_queue_get_option(q, "mesos-master"); if (mesos_master == NULL) { fatal("Please specify the hostname of mesos master by using --mesos-master"); } else { debug(D_INFO, "Get mesos_path %s from command line\n", mesos_py_path); is_mesos_master_known = 1; } } mesos_preload = batch_queue_get_option(q, "mesos-preload"); if (is_mesos_py_path_known && is_mesos_master_known && !is_scheduler_running ) { // start mesos scheduler if it is not running start_mesos_scheduler(q); is_scheduler_running = 1; } int task_id = ++counter; debug(D_BATCH, "task %d is ready", task_id); struct batch_job_info *info = calloc(1, sizeof(*info)); info->started = time(0); info->submitted = time(0); itable_insert(q->job_table, task_id, info); // write the ready task information as // "task_id, task_cmd, inputs, outputs" to // mesos_task_info, which will be scanned by // mf_mesos_scheduler later. FILE *task_info_fp; if(access(FILE_TASK_INFO, F_OK) != -1) { task_info_fp = fopen(FILE_TASK_INFO, "a+"); } else { task_info_fp = fopen(FILE_TASK_INFO, "w+"); } struct mesos_task *mt = mesos_task_create(task_id, cmd, extra_input_files, extra_output_files); fprintf(task_info_fp, "%d,%s,", mt->task_id, mt->task_cmd); if (extra_input_files != NULL && strlen(extra_input_files) != 0) { int j = 0; int num_input_files = text_list_size(mt->task_input_files); for (j = 0; j < (num_input_files-1); j++) { fprintf(task_info_fp, "%s ", text_list_get(mt->task_input_files, j)); } fprintf(task_info_fp, "%s,", text_list_get(mt->task_input_files, num_input_files-1)); } else { fprintf(task_info_fp, ","); } if (extra_output_files != NULL && strlen(extra_output_files) != 0) { int j = 0; int num_output_files = text_list_size(mt->task_output_files); for (j = 0; j < (num_output_files-1); j++) { fprintf(task_info_fp, "%s ", text_list_get(mt->task_output_files, j)); } fprintf(task_info_fp, "%s,",text_list_get(mt->task_output_files, num_output_files-1)); } else { fprintf(task_info_fp, ","); } // The default resource requirements for each task int64_t cores = -1; int64_t memory = -1; int64_t disk = -1; if (resources) { cores = resources->cores > -1 ? resources->cores : cores; memory = resources->memory > -1 ? resources->memory : memory; disk = resources->disk > -1 ? resources->disk : disk; } fprintf(task_info_fp, "%" PRId64 ",%" PRId64 ",%" PRId64 ",", cores, memory, disk); fputs("submitted\n", task_info_fp); mesos_task_delete(mt); fclose(task_info_fp); return task_id; }
int main(int argc, char **argv) { char c; struct work_queue *q; int port = WORK_QUEUE_DEFAULT_PORT; extra_files_list = list_create(); while((c = getopt(argc, argv, "e:f:t:x:y:p:N:E:d:vh")) != (char) -1) { switch (c) { case 'e': extra_arguments = optarg; break; case 'f': list_push_head(extra_files_list,optarg); break; case 't': compare_program_time = atof(optarg); break; case 'x': xblock = atoi(optarg); break; case 'y': yblock = atoi(optarg); break; case 'p': port = atoi(optarg); break; case 'N': setenv("WORK_QUEUE_NAME", optarg, 1); break; case 'E': setenv("WORK_QUEUE_PRIORITY", optarg, 1); break; case 'd': debug_flags_set(optarg); break; case 'v': show_version(progname); exit(0); break; case 'h': show_help(progname); exit(0); break; default: show_help(progname); return 1; } } if((argc - optind) < 3) { show_help(progname); exit(1); } struct text_list *seta = text_list_load(argv[optind]); if(!seta) { fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno)); return 1; } fprintf(stderr, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta)); struct text_list *setb = text_list_load(argv[optind+1]); if(!setb) { fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno)); return 1; } fprintf(stderr, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb)); if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) { fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname); return 1; } debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program); xstop = text_list_size(seta); ystop = text_list_size(setb); if(allpairs_compare_function_get(argv[optind+2])) { strcpy(allpairs_compare_program,argv[optind+2]); debug(D_DEBUG,"using internal function %s",allpairs_compare_program); use_external_program = 0; } else { if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) { fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program); return 1; } debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program); use_external_program = 1; } if(!xblock || !yblock) { estimate_block_size(seta,setb,&xblock,&yblock); } fprintf(stderr, "%s: using block size of %dx%d\n",progname,xblock,yblock); q = work_queue_create(port); if(!q) { fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno)); return 1; } fprintf(stderr, "%s: listening for workers on port %d...\n",progname,work_queue_port(q)); while(1) { struct work_queue_task *task = NULL; while(work_queue_hungry(q)) { task = ap_task_create(seta,setb); if(task) { work_queue_submit(q, task); } else { break; } } if(!task && work_queue_empty(q)) break; task = work_queue_wait(q,5); if(task) task_complete(task); } work_queue_delete(q); return 0; }