// if one of the files is reading from stdin, sum_kmers_ptr is set to 0
// `max_cols_ptr` is used to return the most colours being loaded from a single file
// returns the number of colours being loaded in total
size_t graph_files_open(char **graph_paths,
GraphFileReader *gfiles, size_t num_gfiles,
size_t *max_kmers_ptr, size_t *sum_kmers_ptr)
{
size_t i, ctx_max_kmers = 0, ctx_sum_kmers = 0;
bool ctx_uses_stdin = false;
size_t ncols = 0;
for(i = 0; i < num_gfiles; i++)
{
memset(&gfiles[i], 0, sizeof(GraphFileReader));
graph_file_open2(&gfiles[i], graph_paths[i], "r", ncols);
if(gfiles[0].hdr.kmer_size != gfiles[i].hdr.kmer_size) {
cmd_print_usage("Kmer sizes don't match [%u vs %u]",
gfiles[0].hdr.kmer_size, gfiles[i].hdr.kmer_size);
}
ncols = MAX2(ncols, file_filter_into_ncols(&gfiles[i].fltr));
ctx_max_kmers = MAX2(ctx_max_kmers, graph_file_nkmers(&gfiles[i]));
ctx_sum_kmers += graph_file_nkmers(&gfiles[i]);
ctx_uses_stdin |= file_filter_isstdin(&gfiles[i].fltr);
}
if(ctx_uses_stdin) ctx_sum_kmers = SIZE_MAX;
*max_kmers_ptr = ctx_max_kmers;
*sum_kmers_ptr = ctx_sum_kmers;
return ncols;
}
int ctx_sort(int argc, char **argv)
{
const char *out_path = NULL;
struct MemArgs memargs = MEM_ARGS_INIT;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" sort -h` for help. Bad option: %s", argv[optind-1]);
default: die("Bad option: [%c]: %s", c, cmd);
}
}
if(optind+1 != argc)
cmd_print_usage("Require exactly one input graph file (.ctx)");
const char *ctx_path = argv[optind];
//
// Open Graph file
//
GraphFileReader gfile;
memset(&gfile, 0, sizeof(GraphFileReader));
graph_file_open2(&gfile, ctx_path, out_path ? "r" : "r+", true, 0);
if(!file_filter_is_direct(&gfile.fltr))
die("Cannot open graph file with a filter ('in.ctx:blah' syntax)");
size_t num_kmers, memory;
// Reading from a stream
if(gfile.num_of_kmers < 0) {
if(!memargs.num_kmers_set)
die("If reading from a stream, must give -n <num_kmers>");
num_kmers = memargs.num_kmers;
}
else num_kmers = gfile.num_of_kmers;
// Open output path (if given)
FILE *fout = out_path ? futil_fopen_create(out_path, "w") : NULL;
size_t i;
size_t ncols = gfile.hdr.num_of_cols;
size_t kmer_mem = sizeof(BinaryKmer) + (sizeof(Edges)+sizeof(Covg))*ncols;
memory = (sizeof(char*) + kmer_mem) * num_kmers;
char mem_str[50];
bytes_to_str(memory, 1, mem_str);
if(memory > memargs.mem_to_use)
die("Require at least %s memory", mem_str);
status("[memory] Total: %s", mem_str);
char *mem = ctx_malloc(kmer_mem * num_kmers);
char **kmers = ctx_malloc(num_kmers*sizeof(char*));
// Read in whole file
// if(graph_file_fseek(gfile, gfile.hdr_size, SEEK_SET) != 0) die("fseek failed");
size_t nkread = gfr_fread_bytes(&gfile, mem, num_kmers*kmer_mem);
if(nkread != num_kmers*kmer_mem)
die("Could only read %zu bytes [<%zu]", nkread, num_kmers*kmer_mem);
// check we are at the end of the file
char tmpc;
if(gfr_fread_bytes(&gfile, &tmpc, 1) != 0) {
die("More kmers in file than believed (kmers: %zu ncols: %zu).",
num_kmers, ncols);
}
status("Read %zu kmers with %zu colour%s", num_kmers,
ncols, util_plural_str(ncols));
for(i = 0; i < num_kmers; i++)
kmers[i] = mem + kmer_mem*i;
sort_block(kmers, num_kmers);
// Print
if(out_path != NULL) {
// saving to a different destination - write header
graph_write_header(fout, &gfile.hdr);
}
else {
// Directly manipulating gfile.fh here, using it to write later
// Not doing any more reading
if(fseek(gfile.fh, gfile.hdr_size, SEEK_SET) != 0) die("fseek failed");
fout = gfile.fh;
}
for(i = 0; i < num_kmers; i++)
if(fwrite(kmers[i], 1, kmer_mem, fout) != kmer_mem)
die("Cannot write to file");
if(out_path) fclose(fout);
graph_file_close(&gfile);
ctx_free(kmers);
ctx_free(mem);
return EXIT_SUCCESS;
}
int ctx_index(int argc, char **argv)
{
const char *out_path = NULL;
size_t block_size = 0, block_kmers = 0;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case 'b':
cmd_check(!block_kmers, cmd);
block_kmers = cmd_size_nonzero(cmd, optarg);
break;
case 's':
cmd_check(!block_size, cmd);
block_size = cmd_size_nonzero(cmd, optarg);
break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" index -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
if(optind+1 != argc)
cmd_print_usage("Require exactly one input graph file (.ctx)");
if(block_size && block_kmers)
cmd_print_usage("Cannot use --block-kmers and --block-size together");
const char *ctx_path = argv[optind];
//
// Open Graph file
//
GraphFileReader gfile;
memset(&gfile, 0, sizeof(GraphFileReader));
graph_file_open2(&gfile, ctx_path, "r+", true, 0);
if(!file_filter_is_direct(&gfile.fltr))
die("Cannot open graph file with a filter ('in.ctx:blah' syntax)");
// Open output file
FILE *fout = out_path ? futil_fopen_create(out_path, "w") : stdout;
// Start
size_t filencols = gfile.hdr.num_of_cols;
size_t kmer_size = gfile.hdr.kmer_size;
const char *path = file_filter_path(&gfile.fltr);
size_t ncols = file_filter_into_ncols(&gfile.fltr);
size_t kmer_mem = sizeof(BinaryKmer) + (sizeof(Edges)+sizeof(Covg))*filencols;
if(block_size) {
block_kmers = block_size / kmer_mem;
} else if(!block_size && !block_kmers) {
block_size = 4 * ONE_MEGABYTE;
block_kmers = block_size / kmer_mem;
}
// Update block-size
block_size = block_kmers * kmer_mem;
status("[index] block bytes: %zu kmers: %zu; kmer bytes: %zu, hdr: %zu",
block_size, block_kmers, kmer_mem, (size_t)gfile.hdr_size);
if(block_kmers == 0) die("Cannot set block_kmers to zero");
// Print header
fputs("#block_start\tnext_block\tfirst_kmer\tkmer_idx\tnext_kmer_idx\n", fout);
BinaryKmer bkmer = BINARY_KMER_ZERO_MACRO;
BinaryKmer prev_bkmer = BINARY_KMER_ZERO_MACRO;
Covg *covgs = ctx_malloc(ncols * sizeof(Covg));
Edges *edges = ctx_malloc(ncols * sizeof(Edges));
char bkmerstr[MAX_KMER_SIZE+1];
size_t rem_block = block_size - kmer_mem; // block after first kmer
char *tmp_mem = ctx_malloc(rem_block);
// Read in file, print index
size_t nblocks = 0;
size_t bl_bytes = 0, bl_kmers = 0;
size_t bl_byte_offset = gfile.hdr_size, bl_kmer_offset = 0;
while(1)
{
if(!graph_file_read(&gfile, &bkmer, covgs, edges)) {
status("Read kmer failed"); break; }
binary_kmer_to_str(bkmer, kmer_size, bkmerstr);
if(nblocks > 0 && !binary_kmer_less_than(prev_bkmer,bkmer))
die("File is not sorted: %s [%s]", bkmerstr, path);
// We've already read one kmer entry, read rest of block
bl_bytes = kmer_mem + gfr_fread_bytes(&gfile, tmp_mem, rem_block);
bl_kmers = 1 + bl_bytes / kmer_mem;
fprintf(fout, "%zu\t%zu\t%s\t%zu\t%zu\n",
bl_byte_offset, bl_byte_offset+bl_bytes, bkmerstr,
bl_kmer_offset, bl_kmer_offset+bl_kmers);
bl_byte_offset += bl_bytes;
bl_kmer_offset += bl_kmers;
nblocks++;
if(bl_kmers < block_kmers) {
status("last block %zu < %zu; %zu vs %zu",
bl_kmers, block_kmers, bl_bytes, block_size);
break;
}
prev_bkmer = bkmer;
}
ctx_free(covgs);
ctx_free(edges);
ctx_free(tmp_mem);
// done
char num_kmers_str[50], num_blocks_str[50];
char block_mem_str[50], block_kmers_str[50];
ulong_to_str(bl_kmer_offset, num_kmers_str);
ulong_to_str(nblocks, num_blocks_str);
bytes_to_str(block_size, 1, block_mem_str);
ulong_to_str(block_kmers, block_kmers_str);
status("Read %s kmers in %s block%s (block size %s / %s kmers)",
num_kmers_str, num_blocks_str, util_plural_str(nblocks),
block_mem_str, block_kmers_str);
if(fout != stdout) status("Saved to %s", out_path);
graph_file_close(&gfile);
fclose(fout);
return EXIT_SUCCESS;
}
int ctx_join(int argc, char **argv)
{
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_path = NULL;
size_t use_ncols = 0;
GraphFileReader tmp_gfile;
GraphFileBuffer isec_gfiles_buf;
gfile_buf_alloc(&isec_gfiles_buf, 8);
// Arg parsing
char cmd[100], shortopts[100];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'N': cmd_check(!use_ncols, cmd); use_ncols = cmd_uint32_nonzero(cmd, optarg); break;
case 'i':
graph_file_reset(&tmp_gfile);
graph_file_open(&tmp_gfile, optarg);
if(file_filter_into_ncols(&tmp_gfile.fltr) > 1)
warn("Flattening intersection graph into colour 0: %s", optarg);
file_filter_flatten(&tmp_gfile.fltr, 0);
gfile_buf_push(&isec_gfiles_buf, &tmp_gfile, 1);
break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" join -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
GraphFileReader *igfiles = isec_gfiles_buf.b;
size_t num_igfiles = isec_gfiles_buf.len;
if(!out_path) cmd_print_usage("--out <out.ctx> required");
if(optind >= argc)
cmd_print_usage("Please specify at least one input graph file");
// optind .. argend-1 are graphs to load
size_t num_gfiles = (size_t)(argc - optind);
char **gfile_paths = argv + optind;
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
status("Probing %zu graph files and %zu intersect files", num_gfiles, num_igfiles);
// Check all binaries are valid binaries with matching kmer size
size_t i;
size_t ctx_max_cols = 0;
uint64_t min_intersect_num_kmers = 0, ctx_max_kmers = 0, ctx_sum_kmers = 0;
for(i = 0; i < num_gfiles; i++)
{
graph_file_open2(&gfiles[i], gfile_paths[i], "r", true, ctx_max_cols);
if(gfiles[0].hdr.kmer_size != gfiles[i].hdr.kmer_size) {
cmd_print_usage("Kmer sizes don't match [%u vs %u]",
gfiles[0].hdr.kmer_size, gfiles[i].hdr.kmer_size);
}
ctx_max_cols = MAX2(ctx_max_cols, file_filter_into_ncols(&gfiles[i].fltr));
ctx_max_kmers = MAX2(ctx_max_kmers, graph_file_nkmers(&gfiles[i]));
ctx_sum_kmers += graph_file_nkmers(&gfiles[i]);
}
// Probe intersection graph files
for(i = 0; i < num_igfiles; i++)
{
if(gfiles[0].hdr.kmer_size != igfiles[i].hdr.kmer_size) {
cmd_print_usage("Kmer sizes don't match [%u vs %u]",
gfiles[0].hdr.kmer_size, igfiles[i].hdr.kmer_size);
}
uint64_t nkmers = graph_file_nkmers(&igfiles[i]);
if(i == 0) min_intersect_num_kmers = nkmers;
else if(nkmers < min_intersect_num_kmers)
{
// Put smallest intersection binary first
SWAP(igfiles[i], igfiles[0]);
min_intersect_num_kmers = nkmers;
}
}
bool take_intersect = (num_igfiles > 0);
// If we are taking an intersection,
// all kmers intersection kmers will need to be loaded
if(take_intersect)
ctx_max_kmers = ctx_sum_kmers = min_intersect_num_kmers;
bool use_ncols_set = (use_ncols > 0);
bool output_to_stdout = (strcmp(out_path,"-") == 0);
// if(use_ncols == 0) use_ncols = 1;
if(use_ncols_set) {
if(use_ncols < ctx_max_cols && output_to_stdout)
die("I need %zu colours if outputting to STDOUT (--ncols)", ctx_max_cols);
if(use_ncols > ctx_max_cols) {
warn("I only need %zu colour%s ('--ncols %zu' ignored)",
ctx_max_cols, util_plural_str(ctx_max_cols), use_ncols);
use_ncols = ctx_max_cols;
}
}
else {
use_ncols = output_to_stdout ? ctx_max_cols : 1;
}
// Check out_path is writable
futil_create_output(out_path);
status("Output %zu cols; from %zu files; intersecting %zu graphs; ",
ctx_max_cols, num_gfiles, num_igfiles);
if(num_gfiles == 1 && num_igfiles == 0)
{
// Loading only one file with no intersection files
// Don't need to store a graph in memory, can filter as stream
// Don't actually store anything in the de Bruijn graph, but we need to
// pass it, so mock one up
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size,
file_filter_into_ncols(&gfiles[0].fltr), 0, 1024, 0);
graph_writer_stream_mkhdr(out_path, &gfiles[0], &db_graph, NULL, NULL);
graph_file_close(&gfiles[0]);
gfile_buf_dealloc(&isec_gfiles_buf);
ctx_free(gfiles);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem;
bits_per_kmer = sizeof(BinaryKmer)*8 +
(sizeof(Covg) + sizeof(Edges)) * 8 * use_ncols;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
true, &graph_mem);
if(!use_ncols_set)
{
// Maximise use_ncols
size_t max_usencols = (memargs.mem_to_use*8) / bits_per_kmer;
use_ncols = MIN2(max_usencols, ctx_max_cols);
bits_per_kmer = sizeof(BinaryKmer)*8 +
(sizeof(Covg) + sizeof(Edges)) * 8 * use_ncols;
// Re-check memory used
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
true, &graph_mem);
}
status("Using %zu colour%s in memory", use_ncols, util_plural_str(use_ncols));
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
// Create db_graph
dBGraph db_graph;
Edges *intersect_edges = NULL;
size_t edge_cols = (use_ncols + take_intersect);
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, use_ncols, use_ncols,
kmers_in_hash, DBG_ALLOC_COVGS);
// We allocate edges ourself since it's a special case
db_graph.col_edges = ctx_calloc(db_graph.ht.capacity*edge_cols, sizeof(Edges));
// Load intersection binaries
char *intsct_gname_ptr = NULL;
StrBuf intersect_gname;
strbuf_alloc(&intersect_gname, 1024);
if(take_intersect)
{
GraphLoadingPrefs gprefs = graph_loading_prefs(&db_graph);
gprefs.boolean_covgs = true; // covg++ only
for(i = 0; i < num_igfiles; i++)
{
graph_load(&igfiles[i], gprefs, NULL);
// Update intersect header
// note: intersection graphs all load exactly one colour into colour 0
graph_info_make_intersect(&igfiles[i].hdr.ginfo[0], &intersect_gname);
gprefs.must_exist_in_graph = true;
gprefs.must_exist_in_edges = db_graph.col_edges;
}
if(num_igfiles > 1)
{
// Remove nodes where covg != num_igfiles
HASH_ITERATE_SAFE(&db_graph.ht, remove_non_intersect_nodes,
db_graph.col_covgs, (Covg)num_igfiles, &db_graph.ht);
}
status("Loaded intersection set\n");
intsct_gname_ptr = intersect_gname.b;
for(i = 0; i < num_igfiles; i++) graph_file_close(&igfiles[i]);
// Reset graph info
for(i = 0; i < db_graph.num_of_cols; i++)
graph_info_init(&db_graph.ginfo[i]);
// Zero covgs
memset(db_graph.col_covgs, 0, db_graph.ht.capacity * sizeof(Covg));
// Use union edges we loaded to intersect new edges
intersect_edges = db_graph.col_edges;
db_graph.col_edges += db_graph.ht.capacity;
}
bool kmers_loaded = take_intersect, colours_loaded = false;
graph_writer_merge_mkhdr(out_path, gfiles, num_gfiles,
kmers_loaded, colours_loaded, intersect_edges,
intsct_gname_ptr, &db_graph);
if(take_intersect)
db_graph.col_edges -= db_graph.ht.capacity;
for(i = 0; i < num_gfiles; i++) graph_file_close(&gfiles[i]);
strbuf_dealloc(&intersect_gname);
gfile_buf_dealloc(&isec_gfiles_buf);
ctx_free(gfiles);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int graph_file_open(GraphFileReader *file, const char *path)
{
return graph_file_open2(file, path, "r", 0);
}
static void parse_args(int argc, char **argv)
{
BuildGraphTask task;
memset(&task, 0, sizeof(task));
task.prefs = SEQ_LOADING_PREFS_INIT;
task.stats = SEQ_LOADING_STATS_INIT;
uint8_t fq_offset = 0;
int intocolour = -1;
GraphFileReader tmp_gfile;
// Arg parsing
char cmd[100], shortopts[100];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
bool sample_named = false, pref_unused = false;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 't': cmd_check(!nthreads,cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'k': cmd_check(!kmer_size,cmd); kmer_size = cmd_kmer_size(cmd, optarg); break;
case 's':
intocolour++;
check_sample_name(optarg);
sample_name_buf_add(&snamebuf, (SampleName){.colour = intocolour,
.name = optarg});
sample_named = true;
break;
case '1':
case '2':
case 'i':
pref_unused = false;
if(!sample_named)
cmd_print_usage("Please give sample name first [-s,--sample <name>]");
asyncio_task_parse(&task.files, c, optarg, fq_offset, NULL);
task.prefs.colour = intocolour;
add_task(&task);
break;
case 'M':
if(!strcmp(optarg,"FF")) task.prefs.matedir = READPAIR_FF;
else if(!strcmp(optarg,"FR")) task.prefs.matedir = READPAIR_FR;
else if(!strcmp(optarg,"RF")) task.prefs.matedir = READPAIR_RF;
else if(!strcmp(optarg,"RR")) task.prefs.matedir = READPAIR_RR;
else die("-M,--matepair <orient> must be one of: FF,FR,RF,RR");
pref_unused = true; break;
case 'O': fq_offset = cmd_uint8(cmd, optarg); pref_unused = true; break;
case 'Q': task.prefs.fq_cutoff = cmd_uint8(cmd, optarg); pref_unused = true; break;
case 'H': task.prefs.hp_cutoff = cmd_uint8(cmd, optarg); pref_unused = true; break;
case 'p': task.prefs.remove_pcr_dups = true; pref_unused = true; break;
case 'P': task.prefs.remove_pcr_dups = false; pref_unused = true; break;
case 'g':
if(intocolour == -1) intocolour = 0;
graph_file_reset(&tmp_gfile);
graph_file_open2(&tmp_gfile, optarg, "r", true, intocolour);
intocolour = MAX2((size_t)intocolour, file_filter_into_ncols(&tmp_gfile.fltr)-1);
gfile_buf_push(&gfilebuf, &tmp_gfile, 1);
sample_named = false;
break;
case 'I':
graph_file_reset(&tmp_gfile);
graph_file_open(&tmp_gfile, optarg);
if(file_filter_into_ncols(&tmp_gfile.fltr) > 1)
warn("Flattening intersection graph into colour 0: %s", optarg);
file_filter_flatten(&tmp_gfile.fltr, 0);
gfile_buf_push(&gisecbuf, &tmp_gfile, 1);
break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" build -h` for help. Bad option: %s", argv[optind-1]);
default: die("Bad option: %s", cmd);
}
}
int ctx_infer_edges(int argc, char **argv)
{
size_t num_of_threads = DEFAULT_NTHREADS;
struct MemArgs memargs = MEM_ARGS_INIT;
char *out_ctx_path = NULL;
bool add_pop_edges = false, add_all_edges = false;
// Arg parsing
char cmd[100];
char shortopts[100];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'o': cmd_check(!out_ctx_path,cmd); out_ctx_path = optarg; break;
case 't': num_of_threads = cmd_uint32_nonzero(cmd, optarg); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'A': add_all_edges = true; break;
case 'P': add_pop_edges = true; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" inferedges -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
// Default to adding all edges
if(!add_pop_edges && !add_all_edges) add_all_edges = true;
// Can only specify one of --pop --all
if(add_pop_edges && add_all_edges)
cmd_print_usage("Please specify only one of --all --pop");
// Check that optind+1 == argc
if(optind+1 > argc)
cmd_print_usage("Expected exactly one graph file");
else if(optind+1 < argc)
cmd_print_usage("Expected only one graph file. What is this: '%s'", argv[optind]);
//
// Open graph file
//
char *graph_path = argv[optind];
status("Reading graph: %s", graph_path);
if(strchr(graph_path,':') != NULL)
cmd_print_usage("Cannot use ':' in input graph for `"CMD" inferedges`");
GraphFileReader file;
memset(&file, 0, sizeof(file));
file_filter_open(&file.fltr, graph_path);
// Use stat to detect if we are reading from a stream
struct stat st;
bool reading_stream = (stat(file.fltr.path.b, &st) != 0);
// Mode r+ means open (not create) for update (read & write)
graph_file_open2(&file, graph_path, reading_stream ? "r" : "r+", 0);
if(!file_filter_is_direct(&file.fltr))
cmd_print_usage("Inferedges with filter not implemented - sorry");
bool editing_file = !(out_ctx_path || reading_stream);
FILE *fout = NULL;
// Editing input file or writing a new file
if(!editing_file)
fout = futil_fopen_create(out_ctx_path ? out_ctx_path : "-", "w");
// Print output status
if(fout == stdout) status("Writing to STDOUT");
else if(fout != NULL) status("Writing to: %s", out_ctx_path);
else status("Editing file in place: %s", graph_path);
status("Inferring all missing %sedges", add_pop_edges ? "population " : "");
//
// Decide on memory
//
const size_t ncols = file.hdr.num_of_cols;
size_t kmers_in_hash, graph_mem, bits_per_kmer;
// reading stream: all covgs + edges
// reading file: one bit per kmer per colour for 'in colour'
bits_per_kmer = sizeof(BinaryKmer)*8;
if(reading_stream) {
bits_per_kmer += ncols * 8 * (sizeof(Edges) + sizeof(Covg));
} else {
bits_per_kmer += ncols; // in colour
}
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
file.num_of_kmers, file.num_of_kmers,
memargs.mem_to_use_set, &graph_mem);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
//
// Allocate memory
//
int alloc_flags = reading_stream ? DBG_ALLOC_EDGES | DBG_ALLOC_COVGS
: DBG_ALLOC_NODE_IN_COL;
dBGraph db_graph;
db_graph_alloc(&db_graph, file.hdr.kmer_size,
ncols, reading_stream ? ncols : 1,
kmers_in_hash, alloc_flags);
LoadingStats stats = LOAD_STATS_INIT_MACRO;
GraphLoadingPrefs gprefs = {.db_graph = &db_graph,
.boolean_covgs = false,
.must_exist_in_graph = false,
.must_exist_in_edges = NULL,
.empty_colours = false};
// We need to load the graph for both --pop and --all since we need to check
// if the next kmer is in each of the colours
graph_load(&file, gprefs, &stats);
if(add_pop_edges) status("Inferring edges from population...\n");
else status("Inferring all missing edges...\n");
size_t num_kmers_edited;
if(reading_stream)
{
ctx_assert(fout != NULL);
num_kmers_edited = infer_edges(num_of_threads, add_all_edges, &db_graph);
graph_write_header(fout, &file.hdr);
graph_write_all_kmers(fout, &db_graph);
}
else if(fout == NULL) {
num_kmers_edited = inferedges_on_mmap(&db_graph, add_all_edges, &file);
} else {
num_kmers_edited = inferedges_on_file(&db_graph, add_all_edges, &file, fout);
}
if(fout != NULL && fout != stdout) fclose(fout);
char modified_str[100], kmers_str[100];
ulong_to_str(num_kmers_edited, modified_str);
ulong_to_str(db_graph.ht.num_kmers, kmers_str);
double modified_rate = 0;
if(db_graph.ht.num_kmers)
modified_rate = (100.0 * num_kmers_edited) / db_graph.ht.num_kmers;
status("%s of %s (%.2f%%) nodes modified\n",
modified_str, kmers_str, modified_rate);
if(editing_file)
{
// Close and re-open
fclose(file.fh);
file.fh = NULL;
futil_update_timestamp(file.fltr.path.b);
}
graph_file_close(&file);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
