int main(int argc, char *argv[])
{
const char *master_host = 0;
int master_port = WORK_QUEUE_DEFAULT_PORT;
debug_config(argv[0]);
work_queue_status_parse_command_line_arguments(argc, argv);
cctools_version_debug(D_DEBUG, argv[0]);
if( optind < argc ) {
master_host = argv[optind];
optind++;
}
if( optind < argc ) {
master_port = atoi(argv[optind]);
optind++;
}
if(optind < argc) {
fprintf(stderr,"work_queue_status: Too many arguments. Try the -h option for help.\n");
exit(EXIT_FAILURE);
}
time_t stoptime = time(0) + work_queue_status_timeout;
if(master_host) {
return do_direct_query(master_host,master_port,stoptime);
} else {
global_catalog = malloc(sizeof(*global_catalog)*CATALOG_SIZE); //only malloc if catalog queries are being done
get_masters(stoptime);
return do_catalog_query(stoptime);
}
}
int main(int argc, char *argv[])
{
struct link *link, *list_port = 0;
char ch;
time_t current;
int is_daemon = 0;
char *pidfile = NULL;
outgoing_host_list = list_create();
debug_config(argv[0]);
while((ch = getopt(argc, argv, "bB:d:hH:l:L:m:M:n:o:O:p:ST:u:U:v")) != (char) -1) {
switch (ch) {
case 'b':
is_daemon = 1;
break;
case 'B':
free(pidfile);
pidfile = strdup(optarg);
break;
case 'd':
debug_flags_set(optarg);
break;
case 'h':
default:
show_help(argv[0]);
return 1;
case 'l':
lifetime = string_time_parse(optarg);
break;
case 'L':
logfilename = strdup(optarg);
break;
case 'H':
history_dir = strdup(optarg);
break;
case 'm':
child_procs_max = atoi(optarg);
break;
case 'M':
max_server_size = string_metric_parse(optarg);
break;
case 'n':
preferred_hostname = optarg;
break;
case 'o':
free(debug_filename);
debug_filename = strdup(optarg);
break;
case 'O':
debug_config_file_size(string_metric_parse(optarg));
break;
case 'p':
port = atoi(optarg);
break;
case 'S':
fork_mode = 0;
break;
case 'T':
child_procs_timeout = string_time_parse(optarg);
break;
case 'u':
list_push_head(outgoing_host_list, xxstrdup(optarg));
break;
case 'U':
outgoing_timeout = string_time_parse(optarg);
break;
case 'v':
cctools_version_print(stdout, argv[0]);
return 0;
}
}
if (is_daemon) daemonize(0, pidfile);
debug_config_file(debug_filename);
cctools_version_debug(D_DEBUG, argv[0]);
if(logfilename) {
logfile = fopen(logfilename,"a");
if(!logfile) fatal("couldn't open %s: %s\n",optarg,strerror(errno));
}
current = time(0);
debug(D_ALL, "*** %s starting at %s", argv[0], ctime(¤t));
if(!list_size(outgoing_host_list)) {
list_push_head(outgoing_host_list, CATALOG_HOST_DEFAULT);
}
install_handler(SIGPIPE, ignore_signal);
install_handler(SIGHUP, ignore_signal);
install_handler(SIGCHLD, ignore_signal);
install_handler(SIGINT, shutdown_clean);
install_handler(SIGTERM, shutdown_clean);
install_handler(SIGQUIT, shutdown_clean);
install_handler(SIGALRM, shutdown_clean);
if(!preferred_hostname) {
domain_name_cache_guess(hostname);
preferred_hostname = hostname;
}
username_get(owner);
starttime = time(0);
table = nvpair_database_create(history_dir);
if(!table)
fatal("couldn't create directory %s: %s\n",history_dir,strerror(errno));
update_dgram = datagram_create(port);
if(!update_dgram)
fatal("couldn't listen on udp port %d", port);
outgoing_dgram = datagram_create(0);
if(!outgoing_dgram)
fatal("couldn't create outgoing udp port");
list_port = link_serve(port);
if(!list_port)
fatal("couldn't listen on tcp port %d", port);
while(1) {
fd_set rfds;
int ufd = datagram_fd(update_dgram);
int lfd = link_fd(list_port);
int result, maxfd;
struct timeval timeout;
remove_expired_records();
if(time(0) > outgoing_alarm) {
update_all_catalogs(outgoing_dgram);
outgoing_alarm = time(0) + outgoing_timeout;
}
while(1) {
int status;
pid_t pid = waitpid(-1, &status, WNOHANG);
if(pid>0) {
child_procs_count--;
continue;
} else {
break;
}
}
FD_ZERO(&rfds);
FD_SET(ufd, &rfds);
if(child_procs_count < child_procs_max) {
FD_SET(lfd, &rfds);
}
maxfd = MAX(ufd, lfd) + 1;
timeout.tv_sec = 5;
timeout.tv_usec = 0;
result = select(maxfd, &rfds, 0, 0, &timeout);
if(result <= 0)
continue;
if(FD_ISSET(ufd, &rfds)) {
handle_updates(update_dgram);
}
if(FD_ISSET(lfd, &rfds)) {
link = link_accept(list_port, time(0) + 5);
if(link) {
if(fork_mode) {
pid_t pid = fork();
if(pid == 0) {
alarm(child_procs_timeout);
handle_query(link);
_exit(0);
} else if (pid>0) {
child_procs_count++;
}
} else {
handle_query(link);
}
link_close(link);
}
}
}
return 1;
}
static void get_options(int argc, char **argv, const char *progname)
{
signed char c;
char tmp[512];
char *catalog_host = NULL;
int catalog_port = 0;
while((c = getopt(argc, argv, "p:P:n:d:F:N:C:s:r:R:k:w:c:o:uxvhaZ:")) > -1) {
switch (c) {
case 'p':
port = atoi(optarg);
break;
case 'r':
repeat_filename = optarg;
break;
case 'R':
retry_max = atoi(optarg);
break;
case 's':
rectangle_size = atoi(optarg);
break;
case 'k':
kmer_size = atoi(optarg);
break;
case 'w':
window_size = atoi(optarg);
break;
case 'c':
checkpoint_filename = optarg;
break;
case 'd':
debug_flags_set(optarg);
break;
case 'F':
wq_option_fast_abort_multiplier = atof(optarg);
break;
case 'a':
work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
break;
case 'N':
free(project);
project = xxstrdup(optarg);
break;
case 'P':
priority = atoi(optarg);
break;
case 'C':
if(!parse_catalog_server_description(optarg, &catalog_host, &catalog_port)) {
fprintf(stderr, "sand_filter: catalog server should be given as HOSTNAME:PORT'.\n");
exit(1);
}
setenv("CATALOG_HOST", catalog_host, 1);
char *value = string_format("%d", catalog_port);
setenv("CATALOG_PORT", value, 1);
free(value);
work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
break;
case 'u':
do_not_unlink = 1;
break;
case 'Z':
port_file = optarg;
port = 0;
break;
case 'o':
debug_config_file(optarg);
break;
case 'v':
cctools_version_print(stdout, progname);
exit(0);
default:
case 'h':
show_help(progname);
exit(0);
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if(argc - optind != 2) {
show_help(progname);
exit(1);
}
sequence_filename = argv[optind++];
outfilename = argv[optind++];
outdirname = malloc(strlen(outfilename) + 15);
sprintf(outdirname, "%s.filter.tmp", outfilename);
if(mkdir(outdirname, S_IRWXU) != 0) {
if(errno == EEXIST) {
fprintf(stderr, "%s: directory %s already exists, you may want to delete or rename before running.\n", progname, outdirname);
} else {
fprintf(stderr, "%s: couldn't create %s: %s\n", progname, outdirname, strerror(errno));
exit(1);
}
}
sprintf(filter_program_args, "-k %d -w %d -s d", kmer_size, window_size);
if(repeat_filename) {
sprintf(tmp, " -r %s", string_basename(repeat_filename));
strcat(filter_program_args, tmp);
}
}
int main(int argc, char *argv[])
{
signed char c;
random_init();
install_handler(SIGINT, handle_signal);
install_handler(SIGTERM, handle_signal);
install_handler(SIGQUIT, handle_signal);
install_handler(SIGCHLD, handle_signal);
install_handler(SIGHUP, handle_signal);
debug_config(argv[0]);
while(((c = getopt(argc, argv, "+d:o:O:m:M:s:S:vh")) > -1)) {
switch (c) {
case 'c':
check_interval = string_time_parse(optarg);
break;
case 'd':
debug_flags_set(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'O':
debug_config_file_size(string_metric_parse(optarg));
break;
case 'm':
min_wait_time = string_time_parse(optarg);
break;
case 'M':
max_wait_time = string_time_parse(optarg);
break;
case 's':
start_interval = string_time_parse(optarg);
break;
case 'S':
stop_interval = string_time_parse(optarg);
break;
case 'v':
cctools_version_print(stdout, argv[0]);
exit(0);
break;
case 'h':
default:
show_help(argv[0]);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if(optind >= argc) {
show_help(argv[0]);
return 1;
}
program_argv = &argv[optind];
state_start = time(0);
state = STATE_READY;
if(program_argv[0][0] != '/') {
fprintf(stderr, "watchdog: please give me the full path to %s\n", program_argv[0]);
return 1;
}
while(1) {
time_t time_in_state = time(0) - state_start;
switch (state) {
case STATE_READY:
if(start_program()) {
change_state(STATE_STARTED);
} else {
change_state(STATE_STOPPED);
start_failures++;
}
break;
case STATE_STARTED:
if(program_exited()) {
change_state(STATE_STOPPED);
start_failures++;
} else if(time_in_state > start_interval) {
change_state(STATE_RUNNING);
} else if(want_to_exit) {
change_state(STATE_STOP_WAIT);
}
break;
case STATE_RUNNING:
start_failures = 0;
if(program_exited()) {
change_state(STATE_STOPPED);
} else if(program_changed()) {
change_state(STATE_STOP_WAIT);
} else if(want_to_exit) {
change_state(STATE_STOP_WAIT);
}
break;
case STATE_STOP_WAIT:
send_stop_signal();
if(program_exited()) {
change_state(STATE_STOPPED);
} else if(time_in_state > stop_interval) {
change_state(STATE_KILL_WAIT);
}
break;
case STATE_KILL_WAIT:
send_kill_signal();
if(program_exited()) {
change_state(STATE_STOPPED);
}
break;
case STATE_STOPPED:
if(want_to_exit) {
debug(D_DEBUG, "all done");
exit(0);
} else {
unsigned int wait_time;
int i;
wait_time = min_wait_time;
for(i = 0; i < start_failures; i++) {
wait_time *= 2;
}
if(wait_time > max_wait_time || wait_time < min_wait_time) {
wait_time = max_wait_time;
}
if(time_in_state >= (int) wait_time) {
change_state(STATE_READY);
}
}
break;
default:
fatal("invalid state %d!\n", state);
break;
}
if(state_changes == 0) {
sleep(5);
} else {
state_changes = 0;
}
}
return 1;
}
int main(int argc, char *argv[])
{
int did_explicit_auth = 0;
int follow_mode = 0;
int whole_file_mode = 1;
const char *hostname, *source_file, *target_file;
time_t stoptime;
FILE *file;
int c;
char *tickets = NULL;
debug_config(argv[0]);
static struct option long_options[] = {
{"auth", required_argument, 0, 'a'},
{"block-size", required_argument, 0, 'b'},
{"debug", required_argument, 0, 'd'},
{"follow", no_argument, 0, 'f'},
{"tickets", required_argument, 0, 'i'},
{"timeout", required_argument, 0, 't'},
{"version", no_argument, 0, 'v'},
{"help", no_argument, 0, 'h'},
{0, 0, 0, 0}
};
while((c = getopt_long(argc, argv, "a:b:d:fi:t:vh", long_options, NULL)) > -1) {
switch (c) {
case 'a':
auth_register_byname(optarg);
did_explicit_auth = 1;
break;
case 'b':
buffer_size = (size_t)strtoul(optarg, NULL, 0);
break;
case 'd':
debug_flags_set(optarg);
break;
case 'f':
follow_mode = 1;
break;
case 'i':
tickets = strdup(optarg);
break;
case 't':
timeout = string_time_parse(optarg);
break;
case 'v':
cctools_version_print(stdout, argv[0]);
exit(0);
break;
case 'h':
show_help(argv[0]);
exit(0);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if(!did_explicit_auth)
auth_register_all();
if(tickets) {
auth_ticket_load(tickets);
free(tickets);
} else if(getenv(CHIRP_CLIENT_TICKETS)) {
auth_ticket_load(getenv(CHIRP_CLIENT_TICKETS));
} else {
auth_ticket_load(NULL);
}
if((argc - optind) < 3) {
show_help(argv[0]);
exit(0);
}
source_file = argv[optind];
hostname = argv[optind + 1];
target_file = argv[optind + 2];
stoptime = time(0) + timeout;
if(!strcmp(source_file, "-")) {
file = stdin;
source_file = "/dev/stdin";
} else {
file = fopen(source_file, "r");
if(!file) {
fprintf(stderr, "chirp_put: couldn't open %s: %s\n", source_file, strerror(errno));
return 1;
}
}
if(follow_mode)
whole_file_mode = 0;
if(whole_file_mode) {
INT64_T result = chirp_recursive_put(hostname, source_file, target_file, stoptime);
if(result < 0) {
fprintf(stderr, "chirp_put: couldn't put %s to host %s: %s\n", source_file, hostname, strerror(errno));
return 1;
} else {
return 0;
}
} else {
struct chirp_stream *stream;
char *buffer = xxmalloc(buffer_size);
INT64_T ractual, wactual;
stream = chirp_stream_open(hostname, target_file, CHIRP_STREAM_WRITE, stoptime);
if(!stream) {
fprintf(stderr, "chirp_put: couldn't open %s for writing: %s\n", target_file, strerror(errno));
return 1;
}
while(1) {
ractual = full_fread(file, buffer, buffer_size);
if(ractual == 0) {
if(follow_mode) {
debug(D_DEBUG, "waiting for more data...");
sleep(1);
continue;
} else {
break;
}
}
wactual = chirp_stream_write(stream, buffer, (int)ractual, stoptime);
if(wactual != ractual) {
fprintf(stderr, "chirp_put: couldn't write to %s: %s\n", target_file, strerror(errno));
return 1;
}
}
chirp_stream_close(stream, stoptime);
return 0;
}
}
int main(int argc, char *argv[])
{
batch_queue_type_t batch_queue_type = BATCH_QUEUE_TYPE_UNKNOWN;
catalog_host = CATALOG_HOST;
catalog_port = CATALOG_PORT;
debug_config(argv[0]);
int c;
while((c = getopt_long(argc, argv, "F:N:M:T:t:w:W:E:P:S:cd:o:O:vh", long_options, NULL)) > -1) {
switch (c) {
case 'F':
foremen_regex = optarg;
break;
case 'N':
case 'M':
project_regex = optarg;
break;
case 'T':
batch_queue_type = batch_queue_type_from_string(optarg);
if(batch_queue_type == BATCH_QUEUE_TYPE_UNKNOWN) {
fprintf(stderr, "unknown batch queue type: %s\n", optarg);
return EXIT_FAILURE;
}
break;
case 't':
worker_timeout = atoi(optarg);
break;
case 'w':
workers_min = atoi(optarg);
break;
case 'W':
workers_max = atoi(optarg);
break;
case LONG_OPT_TASKS_PER_WORKER:
tasks_per_worker = atof(optarg);
break;
case 'E':
extra_worker_args = optarg;
break;
case LONG_OPT_CORES:
num_cores_option = xxstrdup(optarg);
break;
case LONG_OPT_MEMORY:
num_memory_option = xxstrdup(optarg);
break;
case LONG_OPT_DISK:
num_disk_option = xxstrdup(optarg);
break;
case LONG_OPT_GPUS:
num_gpus_option = xxstrdup(optarg);
break;
case 'P':
password_file = optarg;
break;
case 'S':
scratch_dir = optarg;
break;
case 'c':
consider_capacity = 1;
break;
case 'd':
debug_flags_set(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'O':
debug_config_file_size(string_metric_parse(optarg));
break;
case 'v':
cctools_version_print(stdout, argv[0]);
exit(EXIT_SUCCESS);
case 'h':
show_help(argv[0]);
exit(EXIT_SUCCESS);
default:
show_help(argv[0]);
return EXIT_FAILURE;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if(batch_queue_type == BATCH_QUEUE_TYPE_UNKNOWN) {
fprintf(stderr,"work_queue_pool: You must specify a batch type with the -T option.\n");
fprintf(stderr, "valid options:\n");
fprintf(stderr, "%s\n", batch_queue_type_string());
return 1;
}
if(!project_regex) {
fprintf(stderr,"work_queue_pool: You must give a project name with the -M option.\n");
return 1;
}
if(workers_min>workers_max) {
fprintf(stderr,"work_queue_pool: --min-workers (%d) is greater than --max-workers (%d)\n",workers_min,workers_max);
return 1;
}
if(tasks_per_worker < 1)
{
tasks_per_worker = num_cores_option ? atof(num_cores_option) : 1;
}
if(!scratch_dir) {
scratch_dir = string_format("/tmp/wq-pool-%d",getuid());
}
if(!create_dir(scratch_dir,0777)) {
fprintf(stderr,"work_queue_pool: couldn't create %s: %s",scratch_dir,strerror(errno));
return 1;
}
char cmd[1024];
sprintf(cmd,"cp \"$(which work_queue_worker)\" '%s'",scratch_dir);
if (system(cmd)) {
fprintf(stderr, "work_queue_pool: please add work_queue_worker to your PATH.\n");
exit(EXIT_FAILURE);
}
if(password_file) {
sprintf(cmd,"cp %s %s/pwfile",password_file,scratch_dir);
system(cmd);
}
if(chdir(scratch_dir)!=0) {
fprintf(stderr,"work_queue_pool: couldn't chdir to %s: %s",scratch_dir,strerror(errno));
return 1;
}
signal(SIGINT, handle_abort);
signal(SIGQUIT, handle_abort);
signal(SIGTERM, handle_abort);
signal(SIGHUP, ignore_signal);
struct batch_queue * queue = batch_queue_create(batch_queue_type);
if(!queue) {
fprintf(stderr,"work_queue_pool: couldn't establish queue type %s",batch_queue_type_to_string(batch_queue_type));
return 1;
}
set_worker_resources( queue );
mainloop( queue, project_regex, foremen_regex );
batch_queue_delete(queue);
return 0;
}
int main(int argc, char *argv[])
{
int c;
random_init();
debug_config(argv[0]);
int display_mode = 0;
cctools_version_debug(D_MAKEFLOW_RUN, argv[0]);
const char *dagfile;
int condense_display = 0;
int change_size = 0;
int export_as_dax = 0;
int ppm_mode = 0;
char *ppm_option = NULL;
struct option long_options_viz[] = {
{"display-mode", required_argument, 0, 'D'},
{"help", no_argument, 0, 'h'},
{"dot-merge-similar", no_argument, 0, LONG_OPT_DOT_CONDENSE},
{"dot-proportional", no_argument, 0, LONG_OPT_DOT_PROPORTIONAL},
{"ppm-highlight-row", required_argument, 0, LONG_OPT_PPM_ROW},
{"ppm-highlight-exe", required_argument, 0, LONG_OPT_PPM_EXE},
{"ppm-highlight-file", required_argument, 0, LONG_OPT_PPM_FILE},
{"ppm-show-levels", no_argument, 0, LONG_OPT_PPM_LEVELS},
{"export-as-dax", no_argument, 0, 'e'},
{"version", no_argument, 0, 'v'},
{0, 0, 0, 0}
};
const char *option_string_viz = "b:D:ehv";
while((c = getopt_long(argc, argv, option_string_viz, long_options_viz, NULL)) >= 0) {
switch (c) {
case 'D':
if(strcasecmp(optarg, "dot") == 0)
display_mode = SHOW_DAG_DOT;
else if (strcasecmp(optarg, "ppm") == 0)
display_mode = SHOW_DAG_PPM;
else if (strcasecmp(optarg, "cytoscape") == 0)
display_mode = SHOW_DAG_CYTO;
else
fatal("Unknown display option: %s\n", optarg);
break;
case LONG_OPT_DOT_CONDENSE:
display_mode = SHOW_DAG_DOT;
condense_display = 1;
break;
case LONG_OPT_DOT_PROPORTIONAL:
display_mode = SHOW_DAG_DOT;
change_size = 1;
break;
case LONG_OPT_PPM_EXE:
display_mode = SHOW_DAG_PPM;
ppm_option = optarg;
ppm_mode = 2;
break;
case LONG_OPT_PPM_FILE:
display_mode = SHOW_DAG_PPM;
ppm_option = optarg;
ppm_mode = 3;
break;
case LONG_OPT_PPM_ROW:
display_mode = SHOW_DAG_PPM;
ppm_option = optarg;
ppm_mode = 4;
break;
case LONG_OPT_PPM_LEVELS:
display_mode = SHOW_DAG_PPM;
ppm_mode = 5;
break;
case 'e':
export_as_dax = 1;
break;
case 'h':
show_help_viz(argv[0]);
return 0;
case 'v':
cctools_version_print(stdout, argv[0]);
return 0;
default:
show_help_viz(argv[0]);
return 1;
}
}
if((argc - optind) != 1) {
int rv = access("./Makeflow", R_OK);
if(rv < 0) {
fprintf(stderr, "makeflow_viz: No makeflow specified and file \"./Makeflow\" could not be found.\n");
fprintf(stderr, "makeflow_viz: Run \"%s -h\" for help with options.\n", argv[0]);
return 1;
}
dagfile = "./Makeflow";
} else {
dagfile = argv[optind];
}
struct dag *d = dag_from_file(dagfile);
if(!d) {
fatal("makeflow_viz: couldn't load %s: %s\n", dagfile, strerror(errno));
}
if(export_as_dax) {
dag_to_dax(d, path_basename(dagfile));
return 0;
}
if(display_mode)
{
switch(display_mode)
{
case SHOW_DAG_DOT:
dag_to_dot(d, condense_display, change_size);
break;
case SHOW_DAG_PPM:
dag_to_ppm(d, ppm_mode, ppm_option);
break;
case SHOW_DAG_CYTO:
dag_to_cyto(d, condense_display, change_size);
break;
default:
fatal("Unknown display option.");
break;
}
exit(0);
}
return 0;
}
int main(int argc, char *argv[])
{
char c;
int did_explicit_auth = 0;
char *tickets = NULL;
struct fuse_args fa;
fa.argc = 0;
fa.argv = string_array_new();
fa.allocated = 1;
debug_config(argv[0]);
while((c = getopt(argc, argv, "a:b:d:Dfhi:m:o:t:v")) != -1) {
switch (c) {
case 'd':
debug_flags_set(optarg);
break;
case 'D':
enable_small_file_optimizations = 0;
break;
case 'b':
chirp_reli_blocksize_set(atoi(optarg));
break;
case 'i':
tickets = xxstrdup(optarg);
break;
case 'm':
fa.argc += 1;
fa.argv = string_array_append(fa.argv, optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'a':
auth_register_byname(optarg);
did_explicit_auth = 1;
break;
case 't':
chirp_fuse_timeout = string_time_parse(optarg);
break;
case 'f':
run_in_foreground = 1;
break;
case 'v':
cctools_version_print(stdout, argv[0]);
return 0;
break;
case 'h':
default:
show_help(argv[0]);
return 1;
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if((argc - optind) != 1) {
show_help(argv[0]);
return 1;
}
fuse_mountpoint = argv[optind];
if(!did_explicit_auth)
auth_register_all();
if(tickets) {
auth_ticket_load(tickets);
free(tickets);
} else if(getenv(CHIRP_CLIENT_TICKETS)) {
auth_ticket_load(getenv(CHIRP_CLIENT_TICKETS));
} else {
auth_ticket_load(NULL);
}
file_table = itable_create(0);
signal(SIGHUP, exit_handler);
signal(SIGINT, exit_handler);
signal(SIGTERM, exit_handler);
fuse_chan = fuse_mount(fuse_mountpoint, &fa);
if(!fuse_chan) {
fprintf(stderr, "chirp_fuse: couldn't access %s\n", fuse_mountpoint);
return 1;
}
fuse_instance = fuse_new(fuse_chan, &fa, &chirp_fuse_operations, sizeof(chirp_fuse_operations), 0);
if(!fuse_instance) {
fuse_unmount(fuse_mountpoint, fuse_chan);
fprintf(stderr, "chirp_fuse: couldn't access %s\n", fuse_mountpoint);
return 1;
}
printf("chirp_fuse: mounted chirp on %s\n", fuse_mountpoint);
#ifdef CCTOOLS_OPSYS_DARWIN
printf("chirp_fuse: to unmount: umount %s\n", fuse_mountpoint);
#else
printf("chirp_fuse: to unmount: fusermount -u %s\n", fuse_mountpoint);
#endif
fflush(0);
if(!run_in_foreground)
daemon(0, 0);
fuse_loop(fuse_instance);
fuse_unmount(fuse_mountpoint, fuse_chan);
fuse_destroy(fuse_instance);
free(fa.argv);
return 0;
}
int main( int argc, char *argv[] )
{
char c;
const char *progname = "wavefront";
debug_config(progname);
while((c=getopt(argc,argv,"p:Pd:o:vh"))!=(char)-1) {
switch(c) {
case 'p':
port = atoi(optarg);
break;
case 'd':
debug_flags_set(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'v':
cctools_version_print(stdout, progname);
exit(0);
break;
case 'h':
show_help(progname);
exit(0);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if( (argc-optind)!=5 ) {
show_help(progname);
exit(1);
}
function = argv[optind];
xsize=atoi(argv[optind+1]);
ysize=atoi(argv[optind+2]);
infile=argv[optind+3];
outfile=argv[optind+4];
start_time = time(0);
last_display_time = 0;
cells_total = xsize*ysize;
xsize++;
ysize++;
array = text_array_create(xsize,ysize);
if(!text_array_load(array,infile)) {
fprintf(stderr,"couldn't load %s: %s",infile,strerror(errno));
return 1;
}
int count = text_array_load(array,outfile);
if(count>0) printf("recovered %d results from %s\n",count,outfile);
logfile = fopen(outfile,"a");
if(!logfile) {
fprintf(stderr,"couldn't open %s for append: %s\n",outfile,strerror(errno));
return 1;
}
queue = work_queue_create(port);
task_prime();
struct work_queue_task *t;
while(1) {
if(time(0)!=last_display_time) display_progress(queue);
t = work_queue_wait(queue,WORK_QUEUE_WAITFORTASK);
if(!t) break;
if(t->return_status==0) {
int x,y;
if(sscanf(t->tag,"%d %d",&x,&y)==2) {
text_array_set(array,x,y,t->output);
task_complete(x,y);
fprintf(logfile,"%d %d %s\n",x,y,t->output);
fflush(logfile);
tasks_done++;
} else {
fprintf(stderr,"unexpected output: %s\nfrom command: %s\non host: %s",t->output,t->command_line,t->host);
}
} else {
fprintf(stderr,"function failed return value (%i) result (%i) on host %s. output:\n%s\n",t->return_status,t->result,t->host,t->output);
}
work_queue_task_delete(t);
if(work_queue_empty(queue))
break;
}
display_progress(queue);
return 0;
}
int main(int argc, char ** argv)
{
FILE * input;
struct seq *s1=0, *s2=0;
char ori;
char c;
int fileindex;
int del_input=0;
while((c = getopt(argc, argv, "a:o:k:m:q:xd:vh")) != (char) -1) {
switch (c) {
case 'a':
align_type = optarg;
break;
case 'o':
output_format = optarg;
break;
case 'm':
min_align = atoi(optarg);
break;
case 'q':
min_qual = atof(optarg);
break;
case 'x':
del_input = 1;
break;
case 'd':
debug_flags_set(optarg);
break;
case 'v':
cctools_version_print(stdout, argv[0]);
exit(0);
break;
default:
case 'h':
show_help(argv[0]);
exit(0);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
fileindex = optind;
if ((argc - optind) == 1) {
input = fopen(argv[fileindex], "r");
if (!input) {
fprintf(stderr, "sand_align_kernel: couldn't open %s: %s\n",argv[fileindex],strerror(errno));
exit(1);
}
} else {
input = stdin;
}
struct cseq *c1, *c2;
if(!strcmp(output_format,"ovl") || !strcmp(output_format, "ovl_new")) {
overlap_write_begin(stdout);
}
// outer loop: read first sequence in comparison list
while((c1=cseq_read(input))) {
s1 = cseq_uncompress(c1);
cseq_free(c1);
// inner loop: read sequences until null (indicating end of list)
// then continue again with outer loop. (two nulls to halt.)
while((c2=cseq_read(input))) {
s2 = cseq_uncompress(c2);
cseq_free(c2);
int dir = 0;
int start1 = 0;
int start2 = 0;
char* tmp = strdup(s2->metadata);
int metadata_valid = 0;
char* token = strtok(tmp, " ");
start2 = atoi(token);
metadata_valid++;
while((token = strtok(NULL, " ")))
{
dir = start1;
start1 = start2;
start2 = atoi(token);
metadata_valid++;
}
if(metadata_valid>=1 && dir==-1) {
seq_reverse_complement(s2);
ori = 'I';
} else {
ori = 'N';
}
struct matrix *m = matrix_create(s1->num_bases,s2->num_bases);
if(!m) {
fprintf(stderr,"sand_align_kernel: out of memory when creating alignment matrix.\n");
exit(1);
}
struct alignment *aln;
if(!strcmp(align_type,"sw")) {
aln = align_smith_waterman(m,s1->data,s2->data);
} else if(!strcmp(align_type,"ps")) {
aln = align_prefix_suffix(m,s1->data,s2->data, min_align);
} else if(!strcmp(align_type,"banded")) {
if(metadata_valid<3) {
fprintf(stderr,"sand_align_kernel: sequence %s did not indicate start positions for the banded alignment.\n",s2->name);
exit(1);
}
/* The width of the band is proportional to the desired quality of the match. */
int k = 2 + min_qual * MIN(s1->num_bases,s2->num_bases) / 2.0;
if(k<5) k = 5;
aln = align_banded(m,s1->data, s2->data, start1, start2, k);
} else {
fprintf(stderr,"unknown alignment type: %s\n",align_type);
exit(1);
}
aln->ori = ori;
if(aln->quality <= min_qual) {
if(!strcmp(output_format,"ovl")) {
overlap_write_v5(stdout, aln, s1->name, s2->name);
} else if(!strcmp(output_format, "ovl_new")) {
overlap_write_v7(stdout, aln, s1->name, s2->name);
} else if(!strcmp(output_format,"matrix")) {
printf("*** %s alignment of sequences %s and %s (quality %lf):\n\n",align_type,s1->name,s2->name,aln->quality);
matrix_print(m,s1->data,s2->data);
} else if(!strcmp(output_format,"align")) {
printf("*** %s alignment of sequences %s and %s (quality %lf):\n\n",align_type,s1->name,s2->name,aln->quality);
alignment_print(stdout,s1->data,s2->data,aln);
} else {
printf("unknown output format '%s'\n",output_format);
exit(1);
}
}
matrix_delete(m);
seq_free(s2);
alignment_delete(aln);
}
seq_free(s1);
}
fclose(input);
if(!strcmp(output_format,"ovl") || !strcmp(output_format, "ovl_new")) {
overlap_write_end(stdout);
}
if ((argc - optind) == 1 && del_input == 1)
{
remove(argv[fileindex]);
}
return 0;
}
int main(int argc, char *argv[])
{
char *temp;
int did_explicit_auth = 0;
char *tickets = NULL;
char prompt[CHIRP_LINE_MAX];
char line[CHIRP_LINE_MAX];
char **user_argv = 0;
int user_argc;
signed char c;
int result = 0;
debug_config(argv[0]);
static struct option long_options[] = {
{"auth", required_argument, 0, 'a'},
{"debug", required_argument, 0, 'd'},
{"tickets", required_argument, 0, 'i'},
{"verbose", no_argument, 0, 'l'},
{"timeout", required_argument, 0, 't'},
{"version", no_argument, 0, 'v'},
{"help", no_argument, 0, 'h'},
{0, 0, 0, 0}
};
while((c = getopt_long(argc, argv, "+a:d:hi:lt:v", long_options, NULL)) > -1) {
switch (c) {
case 'a':
auth_register_byname(optarg);
did_explicit_auth = 1;
break;
case 'd':
debug_flags_set(optarg);
break;
case 'h':
show_help(argv[0]);
exit(0);
break;
case 'i':
tickets = strdup(optarg);
break;
case 'l':
long_information = 1;
break;
case 't':
timeout = string_time_parse(optarg);
break;
case 'v':
cctools_version_print(stdout, argv[0]);
exit(0);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if(!did_explicit_auth)
auth_register_all();
if(tickets) {
auth_ticket_load(tickets);
free(tickets);
} else if(getenv(CHIRP_CLIENT_TICKETS)) {
auth_ticket_load(getenv(CHIRP_CLIENT_TICKETS));
} else {
auth_ticket_load(NULL);
}
getcwd(current_local_dir, CHIRP_PATH_MAX);
/* interactive mode if input is a TTY but we are not simply executing a
* command from argv */
interactive_mode = isatty(0) && !((argc - optind) > 1);
if(optind < argc) {
stoptime = time(0) + timeout;
if(do_open(1, &argv[optind - 1])) {
fprintf(stderr, "couldn't open %s: %s\n", argv[optind], strerror(errno));
return 1;
}
}
if((argc - optind) > 1) {
return !process_command(argc - optind - 1, &argv[optind + 1]);
}
while(1) {
if(interactive_mode) {
sprintf(prompt, " chirp:%s:%s> ", current_host, current_remote_dir);
} else {
prompt[0] = 0;
}
#ifdef HAS_LIBREADLINE
temp = readline(prompt);
if(!temp)
break;
strcpy(line, temp);
free(temp);
#else
printf("%s", prompt);
fflush(stdout);
if(!fgets(line, CHIRP_LINE_MAX, stdin))
break;
#endif
if(!line[0])
continue;
if(!interactive_mode && (temp = strchr(line, '#'))) { /* comment? */
for(temp--; temp > line && isspace((int) *temp); temp--); /* preceding space? */
if(temp <= line)
continue; /* else not comment */
}
#ifdef HAS_LIBREADLINE
add_history(line);
#endif
{
char *start = line, *last = strlen(line) + line;
while(*start != '\0') { /* process compound commands */
char *end = strchr(start, ';');
while(end != NULL && end != start && *(end - 1) == '\\')
end = strchr(end + 1, ';');
if(end == NULL)
end = start + strlen(start);
*end = '\0';
if(user_argv)
free(user_argv);
string_split(start, &user_argc, &user_argv);
if(user_argc == 0) {
start++;
continue;
}
result = process_command(user_argc, user_argv);
start = end == last ? last : end + 1;
}
}
if(!interactive_mode && !result)
break;
}
if(result) {
return 0;
} else {
return 1;
}
}
int main(int argc, char **argv)
{
FILE *input;
FILE *repeats = 0;
FILE *output;
int start_x, end_x, start_y, end_y;
debug_config(progname);
get_options(argc, argv, progname);
cctools_version_debug(D_DEBUG, argv[0]);
unsigned long start_mem, cand_mem, table_mem;
input = fopen(sequence_filename, "r");
if(!input) fatal("couldn't open %s: %s\n",sequence_filename,strerror(errno));
if(repeat_filename) {
repeats = fopen(repeat_filename, "r");
if(!repeats) fatal("couldn't open %s: %s\n",repeat_filename,strerror(errno));
}
if(output_filename) {
output = fopen(output_filename, "w");
} else {
output = stdout;
}
// Data is in the form:
// >id metadata
// data
// >id metadata
// data
// >>
// ...
set_k(kmer_size);
set_window_size(window_size);
// If we only give one file, do an all vs. all
// on them.
if(!second_sequence_filename) {
num_seqs = load_seqs(input);
start_x = 0;
end_x = num_seqs;
start_y = 0;
end_y = num_seqs;
}
// If we had two files, do not compare ones from
// the same file to each other.
else {
FILE *input2 = fopen(second_sequence_filename, "r");
if(!input2) {
fprintf(stderr, "Could not open file %s for reading.\n", second_sequence_filename);
exit(1);
}
num_seqs = load_seqs_two_files(input, &end_x, input2, &end_y);
start_x = 0;
start_y = end_x;
debug(D_DEBUG,"First file contains %d sequences, stored from (%d,%d].\n", end_x, start_x, end_x);
debug(D_DEBUG,"Second file contains %d sequences, stored from (%d,%d].\n", end_y-end_x, start_y, end_y);
}
fclose(input);
debug(D_DEBUG,"Loaded %d sequences\n",num_seqs);
init_cand_table(num_seqs * 5);
init_mer_table(num_seqs * 5);
if(repeats) {
int repeat_count = init_repeat_mer_table(repeats, 2000000, 0);
fclose(repeats);
debug(D_DEBUG,"Loaded %d repeated mers\n", repeat_count);
}
if(rectangle_size == -1) {
// Do get_mem_avail*0.95 to leave some memory for overhead
rectangle_size = DYNAMIC_RECTANGLE_SIZE(max_mem_kb);
debug(D_DEBUG,"Mem avail: %lu, rectangle size: %d\n",(unsigned long)MEMORY_FOR_MERS(max_mem_kb), rectangle_size);
}
int curr_start_x = start_x;
int curr_start_y = start_y;
candidate_t *output_list = 0;
int num_in_list;
while(curr_start_y < end_y) {
while(curr_start_x < end_x) {
if(start_x == start_y) {
debug(D_DEBUG,"Loading mer table (%d,%d)\n", curr_rect_x, curr_rect_y);
} else {
debug(D_DEBUG,"Loading mer table for [%d,%d) and [%d,%d)\n",curr_start_x, MIN(curr_start_x + rectangle_size, end_x), curr_start_y, MIN(curr_start_y + rectangle_size, end_y));
}
start_mem = get_mem_usage();
load_mer_table_subset(curr_start_x, MIN(curr_start_x + rectangle_size, end_x), curr_start_y, MIN(curr_start_y + rectangle_size, end_y), (curr_start_x == curr_start_y));
table_mem = get_mem_usage();
debug(D_DEBUG,"Finished loading, now generating candidates\n");
debug(D_DEBUG,"Memory used: %lu\n", table_mem - start_mem);
generate_candidates();
cand_mem = get_mem_usage();
debug(D_DEBUG,"Total candidates generated: %llu\n", (long long unsigned int) total_cand);
debug(D_DEBUG,"Candidate memory used: %lu\n", cand_mem - table_mem);
output_list = retrieve_candidates(&num_in_list);
output_candidate_list(output, output_list, num_in_list);
free(output_list);
fflush(output);
debug(D_DEBUG,"Now freeing\n");
free_cand_table();
free_mer_table();
debug(D_DEBUG,"Successfully output and freed!\n");
curr_rect_x++;
curr_start_x += rectangle_size;
}
curr_rect_y++;
curr_start_y += rectangle_size;
curr_rect_x = curr_rect_y;
if(start_y == 0) {
curr_start_x = curr_start_y;
} else {
curr_start_x = start_x;
}
}
fclose(output);
return 0;
}
int main(int argc, char ** argv)
{
const char *progname = "sand_compress_reads";
FILE * infile;
FILE * outfile;
int quiet_mode = 0;
struct seq *s;
struct cseq *c;
signed char d;
int clip = 0;
int internal = 0;
char tmp_id[128];
int count = 0;
while((d=getopt(argc,argv,"cvqhi")) > -1) {
switch(d) {
case 'c':
clip = 1;
break;
case 'i':
internal = 1;
break;
case 'q':
quiet_mode = 1;
break;
case 'v':
cctools_version_print(stdout, progname);
exit(0);
break;
case 'h':
default:
show_help(progname);
exit(0);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if( optind<argc ) {
infile = fopen(argv[optind], "r");
if(!infile) {
fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind],strerror(errno));
return 1;
}
optind++;
} else {
infile = stdin;
}
if( optind<argc ) {
outfile = fopen(argv[optind],"w");
if(!outfile) {
fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind],strerror(errno));
return 1;
}
optind++;
} else {
outfile = stdout;
}
while((s = seq_read(infile))) {
if(clip != 0 || internal != 0){
strcpy(tmp_id, s->name);
strcpy(s->name, strtok(tmp_id,","));
if(internal != 0){
strcpy(s->name, strtok(NULL,","));
}
}
c = seq_compress(s);
cseq_write(outfile,c);
cseq_free(c);
seq_free(s);
count++;
}
if(!quiet_mode) {
fprintf(stderr,"%d sequences compressed.\n",count);
}
fclose(infile);
fclose(outfile);
return 0;
}
int main(int argc, char *argv[])
{
int did_explicit_auth = 0;
int follow_mode = 0;
time_t stoptime;
signed char c;
int setAindex, setBindex; // funcindex;
FILE *setA = NULL;
FILE *setB = NULL;
char setApath[CHIRP_PATH_MAX];
char setBpath[CHIRP_PATH_MAX];
char *LIST_FILE_NAME = "set.list";
char setAfilename[CHIRP_PATH_MAX];
char setBfilename[CHIRP_PATH_MAX];
int len = CHIRP_PATH_MAX;
struct chirp_matrix *mat = NULL;
int mathost, matpath;
double *resbuff = NULL;
int numels;
int cntr;
// Variables defined by Li
int i; // for multiprocess calculation
int numOfMovingElements, numOfStableElements;
//int setACount, setBCount;
int setAPos, setBPos;
long setAStartPos;
//long setBStartPos;
int x1, y1, x2, y2, topLeftX, topLeftY; // [x1,y1]-start position, [x2,y2]-end position, the sub matrix we compute in a round
int x_rel, y_rel;
double threshold;
double threshold_min = 0;
double threshold_max = 1;
double threshold_interval = 0.2;
int count_thresholds;
int count_genuine = 0;
int count_impostar = 0;
//int count_fa = 0;
//int count_fr = 0;
double (*roc_data)[3];
int subject_equal;
// ~Variables defined by Li
int w, h, e, n;
w = 10;
h = 10;
e = 8;
n = 1;
x1 = y1 = x2 = y2 = -1;
topLeftX = topLeftY = 0;
debug_config(argv[0]);
while((c = getopt(argc, argv, "a:b:d:ft:vhw:i:e:n:x:y:p:q:r:s:X:Y:c:")) > -1) {
switch (c) {
case 'a':
auth_register_byname(optarg);
did_explicit_auth = 1;
break;
case 'b':
buffer_size = atoi(optarg);
break;
case 'd':
debug_flags_set(optarg);
break;
case 'f':
follow_mode = 1;
break;
case 't':
timeout = string_time_parse(optarg);
break;
case 'w':
w = atoi(optarg);
break;
case 'i':
h = atoi(optarg);
break;
case 'e':
e = atoi(optarg);
break;
case 'n':
n = atoi(optarg);
break;
case 'v':
cctools_version_print(stdout, argv[0]);
exit(0);
break;
case 'h':
show_help(argv[0]);
exit(0);
break;
case 'x':
numOfStableElements = atoi(optarg);
break;
case 'y':
numOfMovingElements = atoi(optarg);
break;
case 'p':
x1 = atoi(optarg);
break;
case 'q':
y1 = atoi(optarg);
break;
case 'r':
x2 = atoi(optarg);
break;
case 's':
y2 = atoi(optarg);
break;
case 'X':
topLeftX = atoi(optarg);
break;
case 'Y':
topLeftY = atoi(optarg);
break;
case 'c':
//numOfCores = atoi(optarg);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if(!did_explicit_auth)
auth_register_all();
if((argc - optind) < 4) {
fprintf(stderr, "after all options, you must have: setA setB function mathost matpath\n");
exit(0);
}
stoptime = time(0) + timeout;
setAindex = optind;
setBindex = optind + 1;
mathost = optind + 2;
matpath = optind + 3;
// Set threshhold min, max and interval
threshold_min = 0;
threshold_max = 1;
threshold_interval = 0.01;
// Initialize result array - roc_data
count_thresholds = (threshold_max - threshold_min) / threshold_interval + 1;
roc_data = malloc(count_thresholds * 3 * sizeof(double));
if(!roc_data) {
fprintf(stderr, "Cannot initialize result buffer!\n");
exit(1);
}
for(i = 0, threshold = threshold_min; i < count_thresholds; i++, threshold += threshold_interval) {
roc_data[i][0] = threshold;
}
// Load matrix to be verified
printf("X1,X2,Y1,Y2: %i,%i,%i,%i\n", x1, x2, y1, y2);
mat = chirp_matrix_open(argv[mathost], argv[matpath], stoptime);
if(mat == NULL) {
fprintf(stderr, "No such matrix. Fail.\n");
exit(1);
}
printf("Start loading matrix ... \n");
printf("Width, height: %d, %d\n\n", chirp_matrix_width(mat), chirp_matrix_height(mat));
numels = (x2 - x1 + 1) * (y2 - y1 + 1);
resbuff = (double *) malloc(numels * sizeof(double));
double *pilot_resbuff;
pilot_resbuff = resbuff;
// TODO get_range function can get at most 10*10 matrix a time (actually it can get more)
/**
for(j=0; j<y2-y1+1; j++) {
for(i=0; i<x2-x1+1; i++) {
int matrtn = chirp_matrix_get_range( mat, x1+i, y1+j, 1, 1, pilot_resbuff, stoptime); //(x2-x1+1), (y2-y1+1), resbuff, stoptime );
printf("%.2f\t", (*pilot_resbuff));
if(matrtn == -1) printf("return mat error @ [%d, %d]!!\n", x1+i, y1+j);
pilot_resbuff++;
}
printf("\n");
}*/
int matrtn = chirp_matrix_get_range(mat, x1, y1, x2 - x1 + 1, y2 - y1 + 1, resbuff, stoptime); //(x2-x1+1), (y2-y1+1), resbuff, stoptime );
if(matrtn == -1) {
fprintf(stderr, "return mat error @ [%d, %d], width: %d; height: %d!\n", x1, y1, x2 - x1 + 1, y2 - y1 + 1);
exit(1);
}
printf("*******end of loading matrix********\n\n");
// Get local path for data sets directories
if(get_local_path(setApath, argv[setAindex], stoptime) != 0 || get_local_path(setBpath, argv[setBindex], stoptime) != 0) {
fprintf(stderr, "Paths to data sets are invalid!\n");
exit(1);
}
// setA and setB each contains a list of file names that points to the data files
char setAlistfile[CHIRP_PATH_MAX];
char setBlistfile[CHIRP_PATH_MAX];
strcpy(setAlistfile, setApath);
strcat(setAlistfile, LIST_FILE_NAME);
if((setA = fopen(setAlistfile, "r")) == NULL) {
fprintf(stderr, "Cannot open data set A list file - %s!\n", setAlistfile);
exit(1);
}
strcpy(setBlistfile, setBpath);
strcat(setBlistfile, LIST_FILE_NAME);
if((setB = fopen(setBlistfile, "r")) == NULL) {
fprintf(stderr, "Cannot open data set B list file - %s!\n", setBlistfile);
exit(1);
}
// Initialize position parameters and allocate memory for storing results of a block (sub matrix)
x_rel = y_rel = 0; // relative to the sub-matrix we are actually working on
// Go forward until line x1 in Set A list file
for(i = 0; i < x1 && !feof(setA); i++) {
fgets(setAfilename, len, setA);
}
if(i < x1) {
fprintf(stderr, "Set A has less then x1 elements!\n");
exit(1);
}
setAStartPos = ftell(setA);
// Go forward until line y1 in Set B list file
for(i = 0; i < y1 && !feof(setB); i++) {
fgets(setBfilename, len, setB);
}
if(i < y1) {
fprintf(stderr, "Set B has less then x1 elements!\n");
exit(1);
}
//setBStartPos = ftell(setB);
debug(D_CHIRP, "Matrix data:\n");
// start loop
fgets(setBfilename, len, setB);
if(setBfilename != NULL) {
size_t last = strlen(setBfilename) - 1;
if(setBfilename[last] == '\n')
setBfilename[last] = '\0';
}
for(setBPos = y1; !feof(setB) && setBPos <= y2; setBPos++) { // Set B - column of matrix
// Go directly to line y1 in Set B list file
fseek(setA, setAStartPos, SEEK_SET);
fgets(setAfilename, len, setA);
if(setAfilename != NULL) {
size_t last = strlen(setAfilename) - 1;
if(setAfilename[last] == '\n')
setAfilename[last] = '\0';
}
setAPos = x1;
for(setAPos = x1; !feof(setA) && setAPos <= x2; setAPos++) { // Set A- row of matrix
// Threshhold comparison
cntr = ((setBPos - y1) * (x2 - x1 + 1)) + (setAPos - x1);
subject_equal = isSubjectIdEqual(setAfilename, setBfilename);
if(subject_equal == 1) { // A genuine match
for(threshold = threshold_max, i = count_thresholds - 1; 1 - resbuff[cntr] < threshold; threshold -= threshold_interval, i--) {
// False reject
roc_data[i][1] += 1;
}
count_genuine++;
} else if(subject_equal == 0) { // A impostar match
for(threshold = threshold_min, i = 0; 1 - resbuff[cntr] >= threshold; threshold += threshold_interval, i++) {
// False accept
roc_data[i][2] += 1;
}
count_impostar++;
} else {
fprintf(stderr, "Cannot resolve filename in either %s or %s!\n", setAfilename, setBfilename);
exit(1);
}
debug(D_CHIRP, "%.2f\t", resbuff[cntr]);
fgets(setAfilename, len, setA);
if(setAfilename != NULL) {
size_t last = strlen(setAfilename) - 1;
if(setAfilename[last] == '\n')
setAfilename[last] = '\0';
}
}
debug(D_CHIRP, "\n");
fgets(setBfilename, len, setB);
if(setBfilename != NULL) {
size_t last = strlen(setBfilename) - 1;
if(setBfilename[last] == '\n')
setBfilename[last] = '\0';
}
}
printf("\n**********************************************************************\n");
// Printf roc_data
debug(D_CHIRP, "ROC raw data format: Threshold | False reject count | False accept count\n");
for(i = 0; i < count_thresholds; i++) {
debug(D_CHIRP, "%.2f\t%.2f\t%.2f;\t", roc_data[i][0], roc_data[i][1], roc_data[i][2]);
}
debug(D_CHIRP, "\n");
// Transform roc_data to ROC curve data
for(i = 0; i < count_thresholds; i++) {
roc_data[i][1] = 1 - (roc_data[i][1] / count_genuine); // 1 - FRR
roc_data[i][2] = roc_data[i][2] / count_impostar; // FAR
}
// Printf roc_data
debug(D_CHIRP, "ROC curve data format: Threshold | 1 - False reject rate | False accept rate\n");
for(i = 0; i < count_thresholds; i++) {
debug(D_CHIRP, "%.2f\t%.2f\t%.2f;\t", roc_data[i][0], roc_data[i][1], roc_data[i][2]);
}
debug(D_CHIRP, "\n");
// Write to dat file for gnuplot's use
FILE *roc_data_fp;
//char roc_line[20];
roc_data_fp = fopen("roc.dat", "w");
for(i = 0; i < count_thresholds; i++) {
fprintf(roc_data_fp, "%.2f\t%.2f\n", roc_data[i][1], roc_data[i][2]); // 1 - FRR, FAR
}
fclose(roc_data_fp);
free(resbuff);
debug(D_CHIRP, "%d comparisons in the matrix are tested! Genuine matches: %d\t Impostar matches: %d\n\n", cntr + 1, count_genuine, count_impostar);
printf("\nROC curve data generation completed successfully!\n%d comparisons in the matrix are tested!\n", cntr + 1);
return 0;
}
int main(int argc, char **argv)
{
signed char c;
struct work_queue *q;
int port = WORK_QUEUE_DEFAULT_PORT;
static const char *port_file = NULL;
int work_queue_master_mode = WORK_QUEUE_MASTER_MODE_STANDALONE;
char *project = NULL;
int priority = 0;
debug_config("allpairs_master");
extra_files_list = list_create();
struct option long_options[] = {
{"debug", required_argument, 0, 'd'},
{"help", no_argument, 0, 'h'},
{"version", no_argument, 0, 'v'},
{"port", required_argument, 0, 'p'},
{"random-port", required_argument, 0, 'Z'},
{"extra-args", required_argument, 0, 'e'},
{"width", required_argument, 0, 'x'},
{"height", required_argument, 0, 'y'},
{"advertise", no_argument, 0, 'a'}, //deprecated, left here for backwards compatibility
{"project-name", required_argument, 0, 'N'},
{"debug-file", required_argument, 0, 'o'},
{"output-file", required_argument, 0, 'O'},
{"wqstats-file", required_argument, 0, 's'},
{"input-file", required_argument, 0, 'f'},
{"estimated-time", required_argument, 0, 't'},
{"priority", required_argument, 0, 'P'},
{0,0,0,0}
};
while((c = getopt_long(argc, argv, "ad:e:f:hN:p:P:t:vx:y:Z:O:o:s:", long_options, NULL)) >= 0) {
switch (c) {
case 'a':
work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
break;
case 'd':
debug_flags_set(optarg);
break;
case 'e':
extra_arguments = optarg;
break;
case 'f':
list_push_head(extra_files_list,optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'O':
free(output_filename);
output_filename=xxstrdup(optarg);
break;
case 's':
free(wqstats_filename);
wqstats_filename=xxstrdup(optarg);
break;
case 'h':
show_help(progname);
exit(0);
break;
case 'N':
work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
free(project);
project = xxstrdup(optarg);
break;
case 'p':
port = atoi(optarg);
break;
case 'P':
priority = atoi(optarg);
break;
case 't':
compare_program_time = atof(optarg);
break;
case 'v':
cctools_version_print(stdout, progname);
exit(0);
break;
case 'x':
xblock = atoi(optarg);
break;
case 'y':
yblock = atoi(optarg);
break;
case 'Z':
port_file = optarg;
port = 0;
break;
default:
show_help(progname);
return 1;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if((argc - optind) < 3) {
show_help(progname);
exit(1);
}
struct text_list *seta = text_list_load(argv[optind]);
if(!seta) {
fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
return 1;
}
fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind],text_list_size(seta));
struct text_list *setb = text_list_load(argv[optind+1]);
if(!setb) {
fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind+1],strerror(errno));
return 1;
}
fprintf(stdout, "%s: %s has %d elements\n",progname,argv[optind+1],text_list_size(setb));
if (!find_executable("allpairs_multicore","PATH",allpairs_multicore_program,sizeof(allpairs_multicore_program))) {
fprintf(stderr,"%s: couldn't find allpairs_multicore in path\n",progname);
return 1;
}
debug(D_DEBUG,"using multicore executable %s",allpairs_multicore_program);
xstop = text_list_size(seta);
ystop = text_list_size(setb);
if(allpairs_compare_function_get(argv[optind+2])) {
strcpy(allpairs_compare_program,argv[optind+2]);
debug(D_DEBUG,"using internal function %s",allpairs_compare_program);
use_external_program = 0;
} else {
if(!find_executable(argv[optind+2],"PATH",allpairs_compare_program,sizeof(allpairs_compare_program))) {
fprintf(stderr,"%s: %s is neither an executable nor an internal comparison function.\n",progname,allpairs_compare_program);
return 1;
}
debug(D_DEBUG,"using comparison executable %s",allpairs_compare_program);
use_external_program = 1;
}
if(!xblock || !yblock) {
estimate_block_size(seta,setb,&xblock,&yblock);
}
fprintf(stdout, "%s: using block size of %dx%d\n",progname,xblock,yblock);
if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) {
fprintf(stderr, "allpairs: allpairs master running in catalog mode. Please use '-N' option to specify the name of this project.\n");
fprintf(stderr, "allpairs: Run \"%s -h\" for help with options.\n", argv[0]);
return 1;
}
q = work_queue_create(port);
//Read the port the queue is actually running, in case we just called
//work_queue_create(LINK_PORT_ANY)
port = work_queue_port(q);
if(!q) {
fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno));
return 1;
}
if(port_file)
opts_write_port_file(port_file, port);
if(wqstats_filename)
work_queue_specify_log(q, wqstats_filename);
// advanced work queue options
work_queue_specify_master_mode(q, work_queue_master_mode);
work_queue_specify_name(q, project);
work_queue_specify_priority(q, priority);
fprintf(stdout, "%s: listening for workers on port %d...\n",progname,work_queue_port(q));
while(1) {
struct work_queue_task *task = NULL;
while(work_queue_hungry(q)) {
task = ap_task_create(seta,setb);
if(task) {
work_queue_submit(q, task);
} else {
break;
}
}
if(!task && work_queue_empty(q)) break;
task = work_queue_wait(q,5);
if(task) task_complete(task);
}
work_queue_delete(q);
return 0;
}
int main( int argc, char *argv[] )
{
signed char c;
int work_queue_master_mode = WORK_QUEUE_MASTER_MODE_STANDALONE;
char *project = NULL;
int priority = 0;
const char *progname = "wavefront";
debug_config(progname);
struct option long_options[] = {
{"help", no_argument, 0, 'h'},
{"version", no_argument, 0, 'v'},
{"debug", required_argument, 0, 'd'},
{"advertise", no_argument, 0, 'a'},
{"project-name", required_argument, 0, 'N'},
{"debug-file", required_argument, 0, 'o'},
{"port", required_argument, 0, 'p'},
{"priority", required_argument, 0, 'P'},
{"estimated-time", required_argument, 0, 't'},
{"random-port", required_argument, 0, 'Z'},
{"bitmap", required_argument, 0, 'B'},
{0,0,0,0}
};
while((c=getopt_long(argc,argv,"aB:d:hN:p:P:o:v:Z:", long_options, NULL)) >= 0) {
switch(c) {
case 'a':
break;
case 'd':
debug_flags_set(optarg);
break;
case 'h':
show_help(progname);
exit(0);
break;
case 'N':
work_queue_master_mode = WORK_QUEUE_MASTER_MODE_CATALOG;
free(project);
project = xxstrdup(optarg);
break;
case 'p':
port = atoi(optarg);
break;
case 'P':
priority = atoi(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'v':
cctools_version_print(stdout, progname);
exit(0);
break;
case 'Z':
port_file = optarg;
port = 0;
break;
case 'B':
progress_bitmap_file = optarg;
break;
default:
show_help(progname);
return 1;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if( (argc-optind)!=5 ) {
show_help(progname);
exit(1);
}
function = argv[optind];
xsize=atoi(argv[optind+1]);
ysize=atoi(argv[optind+2]);
infile=argv[optind+3];
outfile=argv[optind+4];
start_time = time(0);
last_display_time = 0;
cells_total = xsize*ysize;
xsize++;
ysize++;
array = text_array_create(xsize,ysize);
if(!text_array_load(array,infile)) {
fprintf(stderr,"couldn't load %s: %s",infile,strerror(errno));
return 1;
}
int count = text_array_load(array,outfile);
if(count>0) printf("recovered %d results from %s\n",count,outfile);
logfile = fopen(outfile,"a");
if(!logfile) {
fprintf(stderr,"couldn't open %s for append: %s\n",outfile,strerror(errno));
return 1;
}
if(work_queue_master_mode == WORK_QUEUE_MASTER_MODE_CATALOG && !project) {
fprintf(stderr, "wavefront: wavefront master running in catalog mode. Please use '-N' option to specify the name of this project.\n");
fprintf(stderr, "wavefront: Run \"%s -h\" for help with options.\n", argv[0]);
return 1;
}
queue = work_queue_create(port);
//Read the port the queue is actually running, in case we just called
//work_queue_create(LINK_PORT_ANY)
port = work_queue_port(queue);
if(!queue) {
fprintf(stderr,"%s: could not create work queue on port %d: %s\n",progname,port,strerror(errno));
return 1;
}
if(port_file)
opts_write_port_file(port_file, port);
// advanced work queue options
work_queue_specify_master_mode(queue, work_queue_master_mode);
work_queue_specify_name(queue, project);
work_queue_specify_priority(queue, priority);
fprintf(stdout, "%s: listening for workers on port %d...\n",progname,work_queue_port(queue));
if(progress_bitmap_file)
{
bmap = bitmap_create(xsize,ysize);
wavefront_bitmap_initialize(bmap);
}
task_prime();
struct work_queue_task *t;
while(1) {
if(time(0)!=last_display_time) display_progress(queue);
t = work_queue_wait(queue,WORK_QUEUE_WAITFORTASK);
if(!t) break;
if(t->return_status==0) {
int x,y;
if(sscanf(t->tag,"%d %d",&x,&y)==2) {
text_array_set(array,x,y,t->output);
task_complete(x,y);
fprintf(logfile,"%d %d %s\n",x,y,t->output);
fflush(logfile);
tasks_done++;
} else {
fprintf(stderr,"unexpected output: %s\nfrom command: %s\non host: %s",t->output,t->command_line,t->host);
}
} else {
fprintf(stderr,"function failed return value (%i) result (%i) on host %s. output:\n%s\n",t->return_status,t->result,t->host,t->output);
}
work_queue_task_delete(t);
if(work_queue_empty(queue))
break;
}
display_progress(queue);
return 0;
}
int main(int argc, char *argv[])
{
struct link *link, *list_port = 0;
signed char ch;
time_t current;
int is_daemon = 0;
char *pidfile = NULL;
char *interface = NULL;
outgoing_host_list = list_create();
debug_config(argv[0]);
static const struct option long_options[] = {
{"background", no_argument, 0, 'b'},
{"pid-file", required_argument, 0, 'B'},
{"debug", required_argument, 0, 'd'},
{"help", no_argument, 0, 'h'},
{"history", required_argument, 0, 'H'},
{"lifetime", required_argument, 0, 'l'},
{"update-log", required_argument, 0, 'L'},
{"max-jobs", required_argument, 0, 'm'},
{"server-size", required_argument, 0, 'M'},
{"name", required_argument, 0, 'n'},
{"interface", required_argument, 0, 'I'},
{"debug-file", required_argument, 0, 'o'},
{"debug-rotate-max", required_argument, 0, 'O'},
{"port", required_argument, 0, 'p'},
{"single", no_argument, 0, 'S'},
{"timeout", required_argument, 0, 'T'},
{"update-host", required_argument, 0, 'u'},
{"update-interval", required_argument, 0, 'U'},
{"version", no_argument, 0, 'v'},
{"port-file", required_argument, 0, 'Z'},
{0,0,0,0}};
while((ch = getopt_long(argc, argv, "bB:d:hH:I:l:L:m:M:n:o:O:p:ST:u:U:vZ:", long_options, NULL)) > -1) {
switch (ch) {
case 'b':
is_daemon = 1;
break;
case 'B':
free(pidfile);
pidfile = strdup(optarg);
break;
case 'd':
debug_flags_set(optarg);
break;
case 'h':
default:
show_help(argv[0]);
return 1;
case 'l':
lifetime = string_time_parse(optarg);
break;
case 'L':
logfilename = strdup(optarg);
break;
case 'H':
history_dir = strdup(optarg);
break;
case 'I':
free(interface);
interface = strdup(optarg);
break;
case 'm':
child_procs_max = atoi(optarg);
break;
case 'M':
max_server_size = string_metric_parse(optarg);
break;
case 'n':
preferred_hostname = optarg;
break;
case 'o':
debug_config_file(optarg);
break;
case 'O':
debug_config_file_size(string_metric_parse(optarg));
break;
case 'p':
port = atoi(optarg);
break;
case 'S':
fork_mode = 0;
break;
case 'T':
child_procs_timeout = string_time_parse(optarg);
break;
case 'u':
list_push_head(outgoing_host_list, xxstrdup(optarg));
break;
case 'U':
outgoing_timeout = string_time_parse(optarg);
break;
case 'v':
cctools_version_print(stdout, argv[0]);
return 0;
case 'Z':
port_file = optarg;
port = 0;
break;
}
}
if (is_daemon) daemonize(0, pidfile);
cctools_version_debug(D_DEBUG, argv[0]);
if(logfilename) {
logfile = fopen(logfilename,"a");
if(!logfile) fatal("couldn't open %s: %s\n",optarg,strerror(errno));
}
current = time(0);
debug(D_NOTICE, "*** %s starting at %s", argv[0], ctime(¤t));
if(!list_size(outgoing_host_list)) {
list_push_head(outgoing_host_list, CATALOG_HOST_DEFAULT);
}
install_handler(SIGPIPE, ignore_signal);
install_handler(SIGHUP, ignore_signal);
install_handler(SIGCHLD, ignore_signal);
install_handler(SIGINT, shutdown_clean);
install_handler(SIGTERM, shutdown_clean);
install_handler(SIGQUIT, shutdown_clean);
install_handler(SIGALRM, shutdown_clean);
if(!preferred_hostname) {
domain_name_cache_guess(hostname);
preferred_hostname = hostname;
}
username_get(owner);
starttime = time(0);
table = nvpair_database_create(history_dir);
if(!table)
fatal("couldn't create directory %s: %s\n",history_dir,strerror(errno));
list_port = link_serve_address(interface, port);
if(list_port) {
/*
If a port was chosen automatically, read it back
so that the same one can be used for the update port.
There is the possibility that the UDP listen will
fail because that port is in use.
*/
if(port==0) {
char addr[LINK_ADDRESS_MAX];
link_address_local(list_port,addr,&port);
}
} else {
if(interface)
fatal("couldn't listen on TCP address %s port %d", interface, port);
else
fatal("couldn't listen on TCP port %d", port);
}
outgoing_dgram = datagram_create(0);
if(!outgoing_dgram)
fatal("couldn't create outgoing udp port");
update_dgram = datagram_create_address(interface, port);
if(!update_dgram) {
if(interface)
fatal("couldn't listen on UDP address %s port %d", interface, port);
else
fatal("couldn't listen on UDP port %d", port);
}
opts_write_port_file(port_file,port);
while(1) {
fd_set rfds;
int ufd = datagram_fd(update_dgram);
int lfd = link_fd(list_port);
int result, maxfd;
struct timeval timeout;
remove_expired_records();
if(time(0) > outgoing_alarm) {
update_all_catalogs(outgoing_dgram);
outgoing_alarm = time(0) + outgoing_timeout;
}
while(1) {
int status;
pid_t pid = waitpid(-1, &status, WNOHANG);
if(pid>0) {
child_procs_count--;
continue;
} else {
break;
}
}
FD_ZERO(&rfds);
FD_SET(ufd, &rfds);
if(child_procs_count < child_procs_max) {
FD_SET(lfd, &rfds);
}
maxfd = MAX(ufd, lfd) + 1;
timeout.tv_sec = 5;
timeout.tv_usec = 0;
result = select(maxfd, &rfds, 0, 0, &timeout);
if(result <= 0)
continue;
if(FD_ISSET(ufd, &rfds)) {
handle_updates(update_dgram);
}
if(FD_ISSET(lfd, &rfds)) {
link = link_accept(list_port, time(0) + 5);
if(link) {
if(fork_mode) {
pid_t pid = fork();
if(pid == 0) {
alarm(child_procs_timeout);
handle_query(link);
_exit(0);
} else if (pid>0) {
child_procs_count++;
}
} else {
handle_query(link);
}
link_close(link);
}
}
}
return 1;
}
int main( int argc, char *argv[] )
{
signed char c;
const char *progname = "wavefront";
debug_config(progname);
progress_log_file = stdout;
struct option long_options[] = {
{"help", no_argument, 0, 'h'},
{"version", no_argument, 0, 'v'},
{"debug", required_argument, 0, 'd'},
{"jobs", required_argument, 0, 'n'},
{"block-size", required_argument, 0, 'b'},
{"debug-file", required_argument, 0, 'o'},
{"log-file", required_argument, 0, 'l'},
{"bitmap", required_argument, 0, 'B'},
{"bitmap-interval", required_argument, 0, 'i'},
{"auto", no_argument, 0, 'A'},
{"local", no_argument, 0, 'L'},
{"batch-type", required_argument, 0, 'T'},
{"verify", no_argument, 0, 'V'},
{0,0,0,0}
};
while((c=getopt_long(argc,argv,"n:b:d:o:l:B:i:qALDT:VX:Y:vh", long_options, NULL)) > -1) {
switch(c) {
case 'n':
manual_max_jobs_running = atoi(optarg);
break;
case 'b':
manual_block_size = atoi(optarg);
break;
case 'd':
debug_flags_set(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'B':
progress_bitmap_file = optarg;
break;
case 'i':
progress_bitmap_interval = atoi(optarg);
break;
case 'l':
progress_log_file = fopen(optarg,"w");
if(!progress_log_file) {
fprintf(stderr,"couldn't open %s: %s\n",optarg,strerror(errno));
return 1;
}
break;
case 'A':
wavefront_mode = WAVEFRONT_MODE_AUTO;
break;
case 'L':
wavefront_mode = WAVEFRONT_MODE_MULTICORE;
break;
case 'T':
wavefront_mode = WAVEFRONT_MODE_DISTRIBUTED;
batch_system_type = batch_queue_type_from_string(optarg);
if(batch_system_type==BATCH_QUEUE_TYPE_UNKNOWN) {
fprintf(stderr,"unknown batch system type: %s\n",optarg);
exit(1);
}
break;
case 'V':
verify_mode = 1;
break;
case 'X':
xstart = atoi(optarg);
break;
case 'Y':
ystart = atoi(optarg);
break;
case 'v':
cctools_version_print(stdout, progname);
exit(0);
break;
case 'h':
show_help(progname);
exit(0);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if( (argc-optind<3) ) {
show_help(progname);
exit(1);
}
function = argv[optind];
xsize=atoi(argv[optind+1]);
ysize=atoi(argv[optind+2]);
total_cells = xsize*ysize;
if(!verify_mode && !check_configuration(function,xsize,ysize)) exit(1);
int ncpus = load_average_get_cpus();
if(wavefront_mode!=WAVEFRONT_MODE_MULTICORE) {
double task_time = measure_task_time();
printf("Each function takes %.02lfs to run.\n",task_time);
block_size = find_best_block_size(xsize,1000,2,task_time,average_dispatch_time);
double distributed_time = wavefront_distributed_model(xsize,1000,2,task_time,block_size,average_dispatch_time);
double multicore_time = wavefront_multicore_model(xsize,ncpus,task_time);
double ideal_multicore_time = wavefront_multicore_model(xsize,xsize,task_time);
double sequential_time = wavefront_multicore_model(xsize,1,task_time);
printf("---------------------------------\n");
printf("This workload would take:\n");
printf("%.02lfs sequentially\n",sequential_time);
printf("%.02lfs on this %d-core machine\n",multicore_time,ncpus);
printf("%.02lfs on a %d-core machine\n",ideal_multicore_time,xsize);
printf("%.02lfs on a 1000-node distributed system with block size %d\n",distributed_time,block_size);
printf("---------------------------------\n");
if(wavefront_mode==WAVEFRONT_MODE_AUTO) {
if(multicore_time < distributed_time*2) {
wavefront_mode = WAVEFRONT_MODE_MULTICORE;
} else {
wavefront_mode = WAVEFRONT_MODE_DISTRIBUTED;
}
}
}
if(wavefront_mode==WAVEFRONT_MODE_MULTICORE) {
batch_system_type = BATCH_QUEUE_TYPE_LOCAL;
max_jobs_running = ncpus;
} else {
max_jobs_running = 1000;
}
if(manual_block_size!=0) {
block_size = manual_block_size;
}
if(manual_max_jobs_running!=0) {
max_jobs_running = manual_max_jobs_running;
}
if(wavefront_mode==WAVEFRONT_MODE_MULTICORE) {
printf("Running in multicore mode with %d CPUs.\n",max_jobs_running);
} else {
printf("Running in distributed mode with block size %d on up to %d CPUs\n",block_size,max_jobs_running);
}
batch_q = batch_queue_create(batch_system_type);
if(verify_mode) exit(0);
struct bitmap * b = bitmap_create(xsize+1,ysize+1);
struct list *ready_list = list_create();
struct itable *running_table = itable_create(0);
struct batch_job_info info;
UINT64_T jobid;
struct wavefront_task *task;
wavefront_task_initialize(b,ready_list);
printf("Starting workload...\n");
fprintf(progress_log_file,"# elapsed time : waiting jobs / running jobs / cells complete (percent complete)\n");
while(1) {
if(abort_mode) {
while((task=list_pop_tail(ready_list))) {
wavefront_task_delete(task);
}
itable_firstkey(running_table);
while(itable_nextkey(running_table,&jobid,(void**)&task)) {
batch_job_remove(batch_q,jobid);
}
}
if(list_size(ready_list)==0 && itable_size(running_table)==0) break;
while(1) {
if(itable_size(running_table)>=max_jobs_running) break;
task = list_pop_tail(ready_list);
if(!task) break;
jobid = wavefront_task_submit(task);
if(jobid>0) {
itable_insert(running_table,jobid,task);
wavefront_task_mark_range(task,b,WAVEFRONT_TASK_STATE_RUNNING);
} else {
abort();
sleep(1);
list_push_head(ready_list,task);
}
}
save_status(b,ready_list,running_table);
jobid = batch_job_wait(batch_q,&info);
if(jobid>0) {
task = itable_remove(running_table,jobid);
if(task) {
if(info.exited_normally && info.exit_code==0) {
total_dispatch_time += info.started-info.submitted;
total_execute_time += MAX(info.finished-info.started,1);
total_cells_complete+=task->width*task->height;
total_jobs_complete++;
average_dispatch_time = 1.0*total_dispatch_time / total_jobs_complete;
average_task_time = 1.0*total_execute_time / total_cells_complete;
wavefront_task_complete(b,ready_list,task);
} else {
printf("job %" PRIu64 " failed, aborting this workload\n",jobid);
abort_mode = 1;
}
}
}
}
save_status(b,ready_list,running_table);
if(abort_mode) {
printf("Workload was aborted.\n");
} else {
printf("Workload complete.\n");
}
return 0;
}
int main(int argc, char *argv[])
{
int c;
int result;
debug_config(progname);
static const struct option long_options[] = {
{"debug", required_argument, 0, 'd'},
{"help", no_argument, 0, 'h'},
{"version", no_argument, 0, 'v'},
{"block-size", required_argument, 0, 'b'},
{"cores", required_argument, 0, 'c'},
{"extra-args", required_argument, 0, 'e'},
{"symmetric", no_argument, 0, LONG_OPT_SYMMETRIC},
{"index", required_argument, 0, LONG_OPT_INDEX},
{0,0,0,0}
};
while((c = getopt_long(argc, argv, "b:c:e:d:vh", long_options, NULL)) > -1) {
switch (c) {
case 'b':
block_size = atoi(optarg);
break;
case 'c':
num_cores = atoi(optarg);
break;
case 'e':
extra_arguments = optarg;
break;
case 'd':
debug_flags_set(optarg);
break;
case LONG_OPT_INDEX:
nindex = sscanf(optarg, "%d %d", &index_array[0], &index_array[1]);
if(nindex != 2) {
fprintf(stderr, "You must provide two indexes: xstart and ystart.\n");
show_help(progname);
exit(0);
}
break;
case LONG_OPT_SYMMETRIC:
is_symmetric = 1;
break;
case 'v':
cctools_version_print(stdout, progname);
exit(0);
break;
default:
case 'h':
show_help(progname);
exit(0);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if((argc - optind) < 3) {
show_help(progname);
exit(1);
}
const char * setapath = argv[optind];
const char * setbpath = argv[optind+1];
const char * funcpath = argv[optind+2];
struct text_list *seta = text_list_load(setapath);
if(!seta) {
fprintf(stderr, "allpairs_multicore: cannot open %s: %s\n",setapath,strerror(errno));
exit(1);
}
struct text_list *setb = text_list_load(setbpath);
if(!setb) {
fprintf(stderr, "allpairs_multicore: cannot open %s: %s\n",setbpath,strerror(errno));
exit(1);
}
if(num_cores==0) num_cores = load_average_get_cpus();
debug(D_DEBUG,"num_cores: %d\n",num_cores);
if(block_size==0) block_size = block_size_estimate(seta);
debug(D_DEBUG,"block_size: %d elements",block_size);
allpairs_compare_t funcptr = allpairs_compare_function_get(funcpath);
if(funcptr) {
result = main_loop_threaded(funcptr,seta,setb);
} else {
if(access(funcpath,X_OK)!=0) {
fprintf(stderr, "%s: %s is neither an executable program nor an internal function.\n",progname,funcpath);
return 1;
}
result = main_loop_program(funcpath,seta,setb);
}
return result;
}
int main(int argc, char *argv[])
{
struct catalog_query *q;
struct nvpair *n;
time_t timeout = 60, stoptime;
const char *catalog_host = 0;
signed char c;
int i;
int count = 0;
int mode = MODE_TABLE;
INT64_T total = 0, avail = 0;
const char *filter_name = 0;
const char *filter_value = 0;
debug_config(argv[0]);
static struct option long_options[] = {
{"catalog", required_argument, 0, 'c'},
{"debug", required_argument, 0, 'd'},
{"debug-file", required_argument, 0, 'o'},
{"debug-rotate-max", required_argument, 0, 'O'},
{"server-space", required_argument, 0, 'A'},
{"all", no_argument, 0, 'a'},
{"timeout", required_argument, 0, 't'},
{"brief", no_argument, 0, 's'},
{"verbose", no_argument, 0, 'l'},
{"totals", no_argument, 0, 'T'},
{"version", no_argument, 0, 'v'},
{"help", no_argument, 0, 'h'},
{0, 0, 0, 0}
};
while((c = getopt_long(argc, argv, "aA:c:d:t:o:O:sTlvh", long_options, NULL)) > -1) {
switch (c) {
case 'a':
show_all_types = 1;
break;
case 'c':
catalog_host = optarg;
break;
case 'd':
debug_flags_set(optarg);
break;
case 't':
timeout = string_time_parse(optarg);
break;
case 'A':
minavail = string_metric_parse(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'O':
debug_config_file_size(string_metric_parse(optarg));
break;
case 'v':
cctools_version_print(stdout, argv[0]);
return 1;
case 's':
mode = MODE_SHORT;
break;
case 'l':
mode = MODE_LONG;
break;
case 'T':
mode = MODE_TOTAL;
break;
case 'h':
default:
show_help(argv[0]);
return 1;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if(argc - optind == 0) {
// fine, keep going
} else if((argc - optind) == 1) {
filter_name = "name";
filter_value = argv[optind];
} else if((argc - optind) == 2) {
filter_name = argv[optind];
filter_value = argv[optind + 1];
} else {
show_help(argv[0]);
return 1;
}
stoptime = time(0) + timeout;
q = catalog_query_create(catalog_host, 0, stoptime);
if(!q) {
fprintf(stderr, "couldn't query catalog: %s\n", strerror(errno));
return 1;
}
if(mode == MODE_TABLE) {
nvpair_print_table_header(stdout, headers);
}
while((n = catalog_query_read(q, stoptime))) {
table[count++] = n;
}
qsort(table, count, sizeof(*table), (int (*)(const void *, const void *)) compare_entries);
for(i = 0; i < count; i++) {
const char *etype = nvpair_lookup_string(table[i], "type");
if(!show_all_types) {
if(etype) {
if(!strcmp(etype, "chirp") || !strcmp(etype, "catalog")) {
/* ok, keep going */
} else {
continue;
}
} else {
continue;
}
}
if(minavail != 0) {
if(minavail > nvpair_lookup_integer(table[i], "avail")) {
continue;
}
}
if(filter_name) {
const char *v = nvpair_lookup_string(table[i], filter_name);
if(!v || strcmp(filter_value, v))
continue;
}
if(mode == MODE_SHORT) {
const char *t = nvpair_lookup_string(table[i], "type");
if(t && !strcmp(t, "chirp")) {
printf("%s:%d\n", nvpair_lookup_string(table[i], "name"), (int) nvpair_lookup_integer(table[i], "port"));
}
} else if(mode == MODE_LONG) {
nvpair_print_text(table[i], stdout);
} else if(mode == MODE_TABLE) {
nvpair_print_table(table[i], stdout, headers);
} else if(mode == MODE_TOTAL) {
avail += nvpair_lookup_integer(table[i], "avail");
total += nvpair_lookup_integer(table[i], "total");
}
}
if(mode == MODE_TOTAL) {
printf("NODES: %4d\n", count);
printf("TOTAL: %6sB\n", string_metric(total, -1, 0));
printf("AVAIL: %6sB\n", string_metric(avail, -1, 0));
printf("INUSE: %6sB\n", string_metric(total - avail, -1, 0));
}
if(mode == MODE_TABLE) {
nvpair_print_table_footer(stdout, headers);
}
return 0;
}
int main(int argc, char *argv[])
{
enum {
LONGOPT_SERVER_LASTHEARDFROM = INT_MAX-0,
LONGOPT_SERVER_PROJECT = INT_MAX-1,
LONGOPT_WHERE = INT_MAX-2,
};
static const struct option long_options[] = {
{"all", no_argument, 0, 'a'},
{"brief", no_argument, 0, 's'},
{"catalog", required_argument, 0, 'c'},
{"debug", required_argument, 0, 'd'},
{"debug-file", required_argument, 0, 'o'},
{"debug-rotate-max", required_argument, 0, 'O'},
{"help", no_argument, 0, 'h'},
{"server-lastheardfrom", required_argument, 0, LONGOPT_SERVER_LASTHEARDFROM},
{"server-project", required_argument, 0, LONGOPT_SERVER_PROJECT},
{"server-space", required_argument, 0, 'A'},
{"timeout", required_argument, 0, 't'},
{"totals", no_argument, 0, 'T'},
{"verbose", no_argument, 0, 'l'},
{"version", no_argument, 0, 'v'},
{"where", required_argument, 0, LONGOPT_WHERE },
{0, 0, 0, 0}
};
struct catalog_query *q;
struct jx *j;
time_t timeout = 60, stoptime;
const char *catalog_host = 0;
int i;
int c;
int count = 0;
int mode = MODE_TABLE;
INT64_T sum_total = 0, sum_avail = 0;
const char *filter_name = 0;
const char *filter_value = 0;
const char *where_expr = "true";
int show_all_types = 0;
const char *server_project = NULL;
time_t server_lastheardfrom = 0;
uint64_t server_avail = 0;
debug_config(argv[0]);
while((c = getopt_long(argc, argv, "aA:c:d:t:o:O:sTlvh", long_options, NULL)) > -1) {
switch (c) {
case 'a':
show_all_types = 1;
break;
case 'c':
catalog_host = optarg;
break;
case 'd':
debug_flags_set(optarg);
break;
case 't':
timeout = string_time_parse(optarg);
break;
case 'A':
server_avail = string_metric_parse(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'O':
debug_config_file_size(string_metric_parse(optarg));
break;
case 'v':
cctools_version_print(stdout, argv[0]);
return 1;
case 's':
mode = MODE_SHORT;
break;
case 'l':
mode = MODE_LONG;
break;
case 'T':
mode = MODE_TOTAL;
break;
case LONGOPT_SERVER_LASTHEARDFROM:
server_lastheardfrom = time(0)-string_time_parse(optarg);
break;
case LONGOPT_SERVER_PROJECT:
server_project = xxstrdup(optarg);
break;
case LONGOPT_WHERE:
where_expr = optarg;
break;
case 'h':
default:
show_help(argv[0]);
return 1;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if(argc - optind == 0) {
// fine, keep going
} else if((argc - optind) == 1) {
filter_name = "name";
filter_value = argv[optind];
} else if((argc - optind) == 2) {
filter_name = argv[optind];
filter_value = argv[optind + 1];
} else {
show_help(argv[0]);
return 1;
}
stoptime = time(0) + timeout;
const char *query_expr;
if(show_all_types) {
query_expr = where_expr;
} else {
query_expr = string_format("%s && (type==\"chirp\" || type==\"catalog\")",where_expr);
}
struct jx *jexpr = jx_parse_string(query_expr);
if(!jexpr) {
fprintf(stderr,"invalid expression: %s\n",query_expr);
return 1;
}
q = catalog_query_create(catalog_host, 0, jexpr, stoptime);
if(!q) {
fprintf(stderr, "couldn't query catalog: %s\n", strerror(errno));
return 1;
}
if(mode == MODE_TABLE) {
jx_table_print_header(headers,stdout);
} else if(mode==MODE_LONG) {
printf("[\n");
}
while((j = catalog_query_read(q, stoptime))) {
table[count++] = j;
}
catalog_query_delete(q);
qsort(table, count, sizeof(*table), (int (*)(const void *, const void *)) compare_entries);
for(i = 0; i < count; i++) {
const char *lastheardfrom = jx_lookup_string(table[i], "lastheardfrom");
if (lastheardfrom && (time_t)strtoul(lastheardfrom, NULL, 10) < server_lastheardfrom)
continue;
const char *avail = jx_lookup_string(table[i], "avail");
if (avail && strtoul(avail, NULL, 10) < server_avail)
continue;
const char *project = jx_lookup_string(table[i], "project");
if (server_project && (project == NULL || !(strcmp(project, server_project) == 0)))
continue;
if(filter_name) {
const char *v = jx_lookup_string(table[i], filter_name);
if(!v || strcmp(filter_value, v))
continue;
}
if(mode == MODE_SHORT) {
const char *t = jx_lookup_string(table[i], "type");
if(t && !strcmp(t, "chirp")) {
printf("%s:%d\n", jx_lookup_string(table[i], "name"), (int) jx_lookup_integer(table[i], "port"));
}
} else if(mode == MODE_LONG) {
if(i!=0) printf(",\n");
jx_print_stream(table[i],stdout);
} else if(mode == MODE_TABLE) {
jx_table_print(headers, table[i], stdout);
} else if(mode == MODE_TOTAL) {
sum_avail += jx_lookup_integer(table[i], "avail");
sum_total += jx_lookup_integer(table[i], "total");
}
}
for(i=0;i<count;i++) {
jx_delete(table[i]);
}
if(mode == MODE_TOTAL) {
printf("NODES: %4d\n", count);
printf("TOTAL: %6sB\n", string_metric(sum_total, -1, 0));
printf("AVAIL: %6sB\n", string_metric(sum_avail, -1, 0));
printf("INUSE: %6sB\n", string_metric(sum_total - sum_avail, -1, 0));
} else if(mode == MODE_TABLE) {
jx_table_print_footer(headers,stdout);
} else if(mode==MODE_LONG) {
printf("\n]\n");
}
return 0;
}
int main(int argc, char *argv[])
{
batch_queue_type_t batch_queue_type = BATCH_QUEUE_TYPE_UNKNOWN;
catalog_host = CATALOG_HOST;
catalog_port = CATALOG_PORT;
batch_submit_options = getenv("BATCH_OPTIONS");
debug_config(argv[0]);
resources = rmsummary_create(-1);
int c;
while((c = getopt_long(argc, argv, "B:C:F:N:M:T:t:w:W:E:P:S:cd:o:O:vh", long_options, NULL)) > -1) {
switch (c) {
case 'B':
batch_submit_options = xxstrdup(optarg);
break;
case 'C':
config_file = xxstrdup(optarg);
break;
case 'F':
foremen_regex = xxstrdup(optarg);
break;
case 'N':
case 'M':
project_regex = xxstrdup(optarg);
break;
case 'T':
batch_queue_type = batch_queue_type_from_string(optarg);
if(batch_queue_type == BATCH_QUEUE_TYPE_UNKNOWN) {
fprintf(stderr, "unknown batch queue type: %s\n", optarg);
return EXIT_FAILURE;
}
break;
case 't':
worker_timeout = atoi(optarg);
break;
case 'w':
workers_min = atoi(optarg);
break;
case 'W':
workers_max = atoi(optarg);
break;
case LONG_OPT_WORKERS_PER_CYCLE:
workers_per_cycle = atoi(optarg);
break;
case LONG_OPT_TASKS_PER_WORKER:
tasks_per_worker = atof(optarg);
break;
case 'E':
extra_worker_args = xxstrdup(optarg);
break;
case LONG_OPT_CORES:
resources->cores = atoi(optarg);
break;
case LONG_OPT_AMAZON_CREDENTIALS:
amazon_credentials = xxstrdup(optarg);
break;
case LONG_OPT_AMAZON_AMI:
amazon_ami = xxstrdup(optarg);
break;
case LONG_OPT_MEMORY:
resources->memory = atoi(optarg);
break;
case LONG_OPT_DISK:
resources->disk = atoi(optarg);
break;
case LONG_OPT_GPUS:
resources->gpus = atoi(optarg);
break;
case LONG_OPT_AUTOSIZE:
autosize = 1;
break;
case LONG_OPT_FACTORY_TIMEOUT:
factory_timeout = MAX(0, atoi(optarg));
break;
case LONG_OPT_CONDOR_REQUIREMENTS:
if(condor_requirements) {
char *tmp = condor_requirements;
condor_requirements = string_format("(%s && (%s))", tmp, optarg);
free(tmp);
} else {
condor_requirements = string_format("(%s)", optarg);
}
break;
case LONG_OPT_WRAPPER:
wrapper_command = optarg;
break;
case LONG_OPT_WRAPPER_INPUT:
if(!wrapper_input) {
wrapper_input = strdup(optarg);
} else {
wrapper_input = string_format("%s,%s",wrapper_input,optarg);
}
break;
case 'P':
password_file = optarg;
break;
case 'S':
scratch_dir = optarg;
break;
case 'c':
consider_capacity = 1;
break;
case 'd':
debug_flags_set(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'O':
debug_config_file_size(string_metric_parse(optarg));
break;
case 'v':
cctools_version_print(stdout, argv[0]);
exit(EXIT_SUCCESS);
case 'h':
show_help(argv[0]);
exit(EXIT_SUCCESS);
default:
show_help(argv[0]);
return EXIT_FAILURE;
}
}
if(project_regex) {
using_catalog = 1;
}
else if((argc - optind) == 2) {
master_host = argv[optind];
master_port = atoi(argv[optind+1]);
}
else {
fprintf(stderr,"work_queue_factory: You must either give a project name with the -M option or master-name option with a configuration file, or give the master's host and port.\n");
show_help(argv[0]);
exit(1);
}
cctools_version_debug(D_DEBUG, argv[0]);
if(batch_queue_type == BATCH_QUEUE_TYPE_UNKNOWN) {
fprintf(stderr,"work_queue_factory: You must specify a batch type with the -T option.\n");
fprintf(stderr, "valid options:\n");
fprintf(stderr, "%s\n", batch_queue_type_string());
return 1;
}
if(config_file) {
char abs_path_name[PATH_MAX];
if(!realpath(config_file, abs_path_name)) {
fprintf(stderr, "work_queue_factory: could not resolve configuration file path: '%s'.\n", config_file);
exit(EXIT_FAILURE);
}
free(config_file);
/* From now on, read config_file from absolute path */
config_file = xxstrdup(abs_path_name);
if(!read_config_file(config_file)) {
fprintf(stderr,"work_queue_factory: There were errors in the configuration file: %s\n", config_file);
return 1;
}
}
if(workers_min>workers_max) {
fprintf(stderr,"work_queue_factory: min workers (%d) is greater than max workers (%d)\n",workers_min, workers_max);
return 1;
}
/*
Careful here: most of the supported batch systems expect
that jobs are submitting from a single shared filesystem.
Changing to /tmp only works in the case of Condor.
*/
if(!scratch_dir) {
if(batch_queue_type==BATCH_QUEUE_TYPE_CONDOR) {
scratch_dir = string_format("/tmp/wq-pool-%d",getuid());
} else {
scratch_dir = string_format("wq-pool-%d",getuid());
}
}
if(!create_dir(scratch_dir,0777)) {
fprintf(stderr,"work_queue_factory: couldn't create %s: %s",scratch_dir,strerror(errno));
return 1;
}
char cmd[1024];
sprintf(cmd,"cp \"$(which work_queue_worker)\" '%s'",scratch_dir);
if (system(cmd)) {
fprintf(stderr, "work_queue_factory: please add work_queue_worker to your PATH.\n");
exit(EXIT_FAILURE);
}
if(password_file) {
sprintf(cmd,"cp %s %s/pwfile",password_file,scratch_dir);
system(cmd);
}
if(chdir(scratch_dir)!=0) {
fprintf(stderr,"work_queue_factory: couldn't chdir to %s: %s",scratch_dir,strerror(errno));
return 1;
}
signal(SIGINT, handle_abort);
signal(SIGQUIT, handle_abort);
signal(SIGTERM, handle_abort);
signal(SIGHUP, ignore_signal);
queue = batch_queue_create(batch_queue_type);
if(!queue) {
fprintf(stderr,"work_queue_factory: couldn't establish queue type %s",batch_queue_type_to_string(batch_queue_type));
return 1;
}
batch_queue_set_option(queue, "batch-options", batch_submit_options);
batch_queue_set_option(queue, "autosize", autosize ? "yes" : NULL);
set_worker_resources_options( queue );
if (amazon_credentials != NULL) {
batch_queue_set_option(queue, "amazon-credentials", amazon_credentials);
}
if (amazon_ami != NULL) {
batch_queue_set_option(queue, "amazon-ami", amazon_ami);
}
if(condor_requirements != NULL && batch_queue_type != BATCH_QUEUE_TYPE_CONDOR) {
debug(D_NOTICE, "condor_requirements will be ignored as workers will not be running in condor.");
} else {
batch_queue_set_option(queue, "condor-requirements", condor_requirements);
}
mainloop( queue );
batch_queue_delete(queue);
return 0;
}
int main(int argc, char *argv[])
{
int did_explicit_auth = 0;
time_t stoptime;
signed char c;
int i, srcindex, nstreams;
FILE *localfile;
struct chirp_stream *stream[argc - 2];
const char *localmode;
int remotemode;
char *tickets = NULL;
debug_config(argv[0]);
static struct option long_options[] = {
{"auth", required_argument, 0, 'a'},
{"block-size", required_argument, 0, 'b'},
{"debug", required_argument, 0, 'd'},
{"tickets", required_argument, 0, 'i'},
{"timeout", required_argument, 0, 't'},
{"version", no_argument, 0, 'v'},
{"help", no_argument, 0, 'h'},
{0, 0, 0, 0}
};
while((c = getopt_long(argc, argv, "a:b:d:i:t:vh", long_options, NULL)) > -1) {
switch (c) {
case 'a':
auth_register_byname(optarg);
did_explicit_auth = 1;
break;
case 'b':
buffer_size = atoi(optarg);
break;
case 'd':
debug_flags_set(optarg);
break;
case 'i':
tickets = strdup(optarg);
break;
case 't':
timeout = string_time_parse(optarg);
break;
case 'v':
cctools_version_print(stdout, argv[0]);
exit(0);
break;
case 'h':
show_help(argv[0]);
exit(0);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if(!did_explicit_auth)
auth_register_all();
if(tickets) {
auth_ticket_load(tickets);
free(tickets);
} else if(getenv(CHIRP_CLIENT_TICKETS)) {
auth_ticket_load(getenv(CHIRP_CLIENT_TICKETS));
} else {
auth_ticket_load(NULL);
}
if((argc - optind) < 4) {
show_help(argv[0]);
return 1;
}
if(!strcmp(argv[optind], "split")) {
stream_mode = MODE_SPLIT;
localmode = "r";
remotemode = CHIRP_STREAM_WRITE;
} else if(!strcmp(argv[optind], "copy")) {
stream_mode = MODE_COPY;
localmode = "r";
remotemode = CHIRP_STREAM_WRITE;
} else if(!strcmp(argv[optind], "join")) {
stream_mode = MODE_JOIN;
localmode = "w";
remotemode = CHIRP_STREAM_READ;
} else {
fprintf(stderr, "unknown operation: %s\n", argv[0]);
show_help(argv[0]);
return 1;
}
char *buffer = malloc(buffer_size);
srcindex = optind + 1;
nstreams = (argc - optind - 2) / 2;
stoptime = time(0) + timeout;
localfile = fopen(argv[srcindex], localmode);
if(!localfile) {
fprintf(stderr, "couldn't open %s: %s\n", argv[srcindex], strerror(errno));
return 1;
}
char **hostname = malloc(sizeof(*hostname) * nstreams);
char **filename = malloc(sizeof(*filename) * nstreams);
for(i = 0; i < nstreams; i++) {
hostname[i] = argv[srcindex + (2 * i) + 1];
filename[i] = argv[srcindex + (2 * i) + 2];
stream[i] = chirp_stream_open(hostname[i], filename[i], remotemode, stoptime);
if(!stream[i]) {
fprintf(stderr, "couldn't open %s:%s: %s\n", hostname[i], filename[i], strerror(errno));
return 1;
}
}
if(stream_mode == MODE_SPLIT) {
i = 0;
while(fgets(buffer, buffer_size, localfile)) {
int length = strlen(buffer);
int actual = chirp_stream_write(stream[i], buffer, length, stoptime);
if(actual != length) {
fprintf(stderr, "couldn't write to %s:%s: %s\n", hostname[i], filename[i], strerror(errno));
return 1;
}
i = (i + 1) % nstreams;
}
} else if(stream_mode == MODE_COPY) {
while(fgets(buffer, buffer_size, localfile)) {
int length = strlen(buffer);
for(i = 0; i < nstreams; i++) {
int actual = chirp_stream_write(stream[i], buffer, length, stoptime);
if(actual != length) {
fprintf(stderr, "couldn't write to %s:%s: %s\n", hostname[i], filename[i], strerror(errno));
return 1;
}
}
}
} else {
int streams_left = nstreams;
while(streams_left > 0) {
for(i = 0; i < nstreams; i++) {
if(!stream[i])
continue;
int length = chirp_stream_readline(stream[i], buffer, buffer_size, stoptime);
if(length > 0) {
length = strlen(buffer);
fprintf(localfile, "%s\n", buffer);
} else {
streams_left--;
}
}
}
}
for(i = 0; i < nstreams; i++) {
chirp_stream_flush(stream[i], stoptime);
chirp_stream_close(stream[i], stoptime);
}
return 0;
}
