// Using file so can call fseek and don't need to load whole graph
static size_t inferedges_on_file(const dBGraph *db_graph, bool add_all_edges,
GraphFileReader *file, FILE *fout)
{
ctx_assert(db_graph->num_of_cols == file->hdr.num_of_cols);
ctx_assert(file_filter_is_direct(&file->fltr));
ctx_assert2(!isatty(fileno(file->fh)), "Use inferedges_on_stream() instead");
ctx_assert(fout != NULL);
ctx_assert(fileno(file->fh) != fileno(fout));
status("[inferedges] Processing file: %s", file_filter_path(&file->fltr));
// Print header
graph_write_header(fout, &file->hdr);
// Read the input file again
if(fseek(file->fh, file->hdr_size, SEEK_SET) != 0)
die("fseek failed: %s", strerror(errno));
const size_t ncols = file->hdr.num_of_cols;
BinaryKmer bkmer;
Edges edges[ncols];
Covg covgs[ncols];
size_t num_kmers_edited = 0;
bool updated;
while(graph_file_read_reset(file, ncols, &bkmer, covgs, edges))
{
updated = (add_all_edges ? infer_all_edges(bkmer, edges, covgs, db_graph)
: infer_pop_edges(bkmer, edges, covgs, db_graph));
graph_write_kmer(fout, file->hdr.num_of_bitfields, file->hdr.num_of_cols,
bkmer, covgs, edges);
num_kmers_edited += updated;
}
return num_kmers_edited;
}
int ctx_sort(int argc, char **argv)
{
const char *out_path = NULL;
struct MemArgs memargs = MEM_ARGS_INIT;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" sort -h` for help. Bad option: %s", argv[optind-1]);
default: die("Bad option: [%c]: %s", c, cmd);
}
}
if(optind+1 != argc)
cmd_print_usage("Require exactly one input graph file (.ctx)");
const char *ctx_path = argv[optind];
//
// Open Graph file
//
GraphFileReader gfile;
memset(&gfile, 0, sizeof(GraphFileReader));
graph_file_open2(&gfile, ctx_path, out_path ? "r" : "r+", true, 0);
if(!file_filter_is_direct(&gfile.fltr))
die("Cannot open graph file with a filter ('in.ctx:blah' syntax)");
size_t num_kmers, memory;
// Reading from a stream
if(gfile.num_of_kmers < 0) {
if(!memargs.num_kmers_set)
die("If reading from a stream, must give -n <num_kmers>");
num_kmers = memargs.num_kmers;
}
else num_kmers = gfile.num_of_kmers;
// Open output path (if given)
FILE *fout = out_path ? futil_fopen_create(out_path, "w") : NULL;
size_t i;
size_t ncols = gfile.hdr.num_of_cols;
size_t kmer_mem = sizeof(BinaryKmer) + (sizeof(Edges)+sizeof(Covg))*ncols;
memory = (sizeof(char*) + kmer_mem) * num_kmers;
char mem_str[50];
bytes_to_str(memory, 1, mem_str);
if(memory > memargs.mem_to_use)
die("Require at least %s memory", mem_str);
status("[memory] Total: %s", mem_str);
char *mem = ctx_malloc(kmer_mem * num_kmers);
char **kmers = ctx_malloc(num_kmers*sizeof(char*));
// Read in whole file
// if(graph_file_fseek(gfile, gfile.hdr_size, SEEK_SET) != 0) die("fseek failed");
size_t nkread = gfr_fread_bytes(&gfile, mem, num_kmers*kmer_mem);
if(nkread != num_kmers*kmer_mem)
die("Could only read %zu bytes [<%zu]", nkread, num_kmers*kmer_mem);
// check we are at the end of the file
char tmpc;
if(gfr_fread_bytes(&gfile, &tmpc, 1) != 0) {
die("More kmers in file than believed (kmers: %zu ncols: %zu).",
num_kmers, ncols);
}
status("Read %zu kmers with %zu colour%s", num_kmers,
ncols, util_plural_str(ncols));
for(i = 0; i < num_kmers; i++)
kmers[i] = mem + kmer_mem*i;
sort_block(kmers, num_kmers);
// Print
if(out_path != NULL) {
// saving to a different destination - write header
graph_write_header(fout, &gfile.hdr);
}
else {
// Directly manipulating gfile.fh here, using it to write later
// Not doing any more reading
if(fseek(gfile.fh, gfile.hdr_size, SEEK_SET) != 0) die("fseek failed");
fout = gfile.fh;
}
for(i = 0; i < num_kmers; i++)
if(fwrite(kmers[i], 1, kmer_mem, fout) != kmer_mem)
die("Cannot write to file");
if(out_path) fclose(fout);
graph_file_close(&gfile);
ctx_free(kmers);
ctx_free(mem);
return EXIT_SUCCESS;
}
int ctx_infer_edges(int argc, char **argv)
{
size_t num_of_threads = DEFAULT_NTHREADS;
struct MemArgs memargs = MEM_ARGS_INIT;
char *out_ctx_path = NULL;
bool add_pop_edges = false, add_all_edges = false;
// Arg parsing
char cmd[100];
char shortopts[100];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'o': cmd_check(!out_ctx_path,cmd); out_ctx_path = optarg; break;
case 't': num_of_threads = cmd_uint32_nonzero(cmd, optarg); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'A': add_all_edges = true; break;
case 'P': add_pop_edges = true; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" inferedges -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
// Default to adding all edges
if(!add_pop_edges && !add_all_edges) add_all_edges = true;
// Can only specify one of --pop --all
if(add_pop_edges && add_all_edges)
cmd_print_usage("Please specify only one of --all --pop");
// Check that optind+1 == argc
if(optind+1 > argc)
cmd_print_usage("Expected exactly one graph file");
else if(optind+1 < argc)
cmd_print_usage("Expected only one graph file. What is this: '%s'", argv[optind]);
//
// Open graph file
//
char *graph_path = argv[optind];
status("Reading graph: %s", graph_path);
if(strchr(graph_path,':') != NULL)
cmd_print_usage("Cannot use ':' in input graph for `"CMD" inferedges`");
GraphFileReader file;
memset(&file, 0, sizeof(file));
file_filter_open(&file.fltr, graph_path);
// Use stat to detect if we are reading from a stream
struct stat st;
bool reading_stream = (stat(file.fltr.path.b, &st) != 0);
// Mode r+ means open (not create) for update (read & write)
graph_file_open2(&file, graph_path, reading_stream ? "r" : "r+", 0);
if(!file_filter_is_direct(&file.fltr))
cmd_print_usage("Inferedges with filter not implemented - sorry");
bool editing_file = !(out_ctx_path || reading_stream);
FILE *fout = NULL;
// Editing input file or writing a new file
if(!editing_file)
fout = futil_fopen_create(out_ctx_path ? out_ctx_path : "-", "w");
// Print output status
if(fout == stdout) status("Writing to STDOUT");
else if(fout != NULL) status("Writing to: %s", out_ctx_path);
else status("Editing file in place: %s", graph_path);
status("Inferring all missing %sedges", add_pop_edges ? "population " : "");
//
// Decide on memory
//
const size_t ncols = file.hdr.num_of_cols;
size_t kmers_in_hash, graph_mem, bits_per_kmer;
// reading stream: all covgs + edges
// reading file: one bit per kmer per colour for 'in colour'
bits_per_kmer = sizeof(BinaryKmer)*8;
if(reading_stream) {
bits_per_kmer += ncols * 8 * (sizeof(Edges) + sizeof(Covg));
} else {
bits_per_kmer += ncols; // in colour
}
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
file.num_of_kmers, file.num_of_kmers,
memargs.mem_to_use_set, &graph_mem);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
//
// Allocate memory
//
int alloc_flags = reading_stream ? DBG_ALLOC_EDGES | DBG_ALLOC_COVGS
: DBG_ALLOC_NODE_IN_COL;
dBGraph db_graph;
db_graph_alloc(&db_graph, file.hdr.kmer_size,
ncols, reading_stream ? ncols : 1,
kmers_in_hash, alloc_flags);
LoadingStats stats = LOAD_STATS_INIT_MACRO;
GraphLoadingPrefs gprefs = {.db_graph = &db_graph,
.boolean_covgs = false,
.must_exist_in_graph = false,
.must_exist_in_edges = NULL,
.empty_colours = false};
// We need to load the graph for both --pop and --all since we need to check
// if the next kmer is in each of the colours
graph_load(&file, gprefs, &stats);
if(add_pop_edges) status("Inferring edges from population...\n");
else status("Inferring all missing edges...\n");
size_t num_kmers_edited;
if(reading_stream)
{
ctx_assert(fout != NULL);
num_kmers_edited = infer_edges(num_of_threads, add_all_edges, &db_graph);
graph_write_header(fout, &file.hdr);
graph_write_all_kmers(fout, &db_graph);
}
else if(fout == NULL) {
num_kmers_edited = inferedges_on_mmap(&db_graph, add_all_edges, &file);
} else {
num_kmers_edited = inferedges_on_file(&db_graph, add_all_edges, &file, fout);
}
if(fout != NULL && fout != stdout) fclose(fout);
char modified_str[100], kmers_str[100];
ulong_to_str(num_kmers_edited, modified_str);
ulong_to_str(db_graph.ht.num_kmers, kmers_str);
double modified_rate = 0;
if(db_graph.ht.num_kmers)
modified_rate = (100.0 * num_kmers_edited) / db_graph.ht.num_kmers;
status("%s of %s (%.2f%%) nodes modified\n",
modified_str, kmers_str, modified_rate);
if(editing_file)
{
// Close and re-open
fclose(file.fh);
file.fh = NULL;
futil_update_timestamp(file.fltr.path.b);
}
graph_file_close(&file);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
