// Close file
void graph_file_close(GraphFileReader *file)
{
if(file->fh) fclose(file->fh);
file_filter_close(&file->fltr);
graph_header_dealloc(&file->hdr);
memset(file, 0, sizeof(*file));
}
int ctx_clean(int argc, char **argv)
{
size_t nthreads = 0, use_ncols = 0;
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_ctx_path = NULL;
bool tip_cleaning = false, supernode_cleaning = false;
size_t min_keep_tip = 0;
Covg threshold = 0, fallback_thresh = 0;
const char *len_before_path = NULL, *len_after_path = NULL;
const char *covg_before_path = NULL, *covg_after_path = NULL;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'o':
if(out_ctx_path != NULL) cmd_print_usage(NULL);
out_ctx_path = optarg;
break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'N': use_ncols = cmd_uint32_nonzero(cmd, optarg); break;
case 't': cmd_check(!nthreads, cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'T':
cmd_check(!tip_cleaning, cmd);
min_keep_tip = cmd_uint32_nonzero(cmd, optarg);
tip_cleaning = true;
break;
case 'S':
cmd_check(!supernode_cleaning, cmd);
if(optarg != NULL) threshold = cmd_uint32_nonzero(cmd, optarg);
supernode_cleaning = true;
break;
case 'B': cmd_check(!fallback_thresh, cmd); fallback_thresh = cmd_uint32_nonzero(cmd, optarg); break;
case 'l': cmd_check(!len_before_path, cmd); len_before_path = optarg; break;
case 'L': cmd_check(!len_after_path, cmd); len_after_path = optarg; break;
case 'c': cmd_check(!covg_before_path, cmd); covg_before_path = optarg; break;
case 'C': cmd_check(!covg_after_path, cmd); covg_after_path = optarg; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" clean -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
if(nthreads == 0) nthreads = DEFAULT_NTHREADS;
if(optind >= argc) cmd_print_usage("Please give input graph files");
// Default behaviour
if(!tip_cleaning && !supernode_cleaning) {
if(out_ctx_path != NULL)
supernode_cleaning = tip_cleaning = true; // do both
else
warn("No cleaning being done: you did not specify --out <out.ctx>");
}
bool doing_cleaning = (supernode_cleaning || tip_cleaning);
if(doing_cleaning && out_ctx_path == NULL) {
cmd_print_usage("Please specify --out <out.ctx> for cleaned graph");
}
if(!doing_cleaning && (covg_after_path || len_after_path)) {
cmd_print_usage("You gave --len-after <out> / --covg-after <out> without "
"any cleaning (set -s, --supernodes or -t, --tips)");
}
if(doing_cleaning && strcmp(out_ctx_path,"-") != 0 &&
!futil_get_force() && futil_file_exists(out_ctx_path))
{
cmd_print_usage("Output file already exists: %s", out_ctx_path);
}
if(fallback_thresh && !supernode_cleaning)
cmd_print_usage("-B, --fallback <T> without --supernodes");
// Use remaining args as graph files
char **gfile_paths = argv + optind;
size_t i, j, num_gfiles = (size_t)(argc - optind);
// Open graph files
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
size_t ncols, ctx_max_kmers = 0, ctx_sum_kmers = 0;
ncols = graph_files_open(gfile_paths, gfiles, num_gfiles,
&ctx_max_kmers, &ctx_sum_kmers);
size_t kmer_size = gfiles[0].hdr.kmer_size;
// default to one colour for now
if(use_ncols == 0) use_ncols = 1;
// Flatten if we don't have to remember colours / output a graph
if(!doing_cleaning)
{
ncols = use_ncols = 1;
for(i = 0; i < num_gfiles; i++)
file_filter_flatten(&gfiles[i].fltr, 0);
}
if(ncols < use_ncols) {
warn("I only need %zu colour%s ('--ncols %zu' ignored)",
ncols, util_plural_str(ncols), use_ncols);
use_ncols = ncols;
}
char max_kmers_str[100];
ulong_to_str(ctx_max_kmers, max_kmers_str);
status("%zu input graph%s, max kmers: %s, using %zu colours",
num_gfiles, util_plural_str(num_gfiles), max_kmers_str, use_ncols);
// If no arguments given we default to removing tips < 2*kmer_size
if(tip_cleaning && min_keep_tip == 0)
min_keep_tip = 2 * kmer_size;
// Warn if any graph files already cleaned
size_t fromcol, intocol;
ErrorCleaning *cleaning;
for(i = 0; i < num_gfiles; i++) {
for(j = 0; j < file_filter_num(&gfiles[i].fltr); j++) {
fromcol = file_filter_fromcol(&gfiles[i].fltr, j);
cleaning = &gfiles[i].hdr.ginfo[fromcol].cleaning;
if(cleaning->cleaned_snodes && supernode_cleaning) {
warn("%s:%zu already has supernode cleaning with threshold: <%zu",
file_filter_path(&gfiles[i].fltr), fromcol,
(size_t)cleaning->clean_snodes_thresh);
}
if(cleaning->cleaned_tips && tip_cleaning) {
warn("%s:%zu already has had tip cleaned",
file_filter_path(&gfiles[i].fltr), fromcol);
}
}
}
// Print steps
size_t step = 0;
status("Actions:\n");
if(covg_before_path != NULL)
status("%zu. Saving kmer coverage distribution to: %s", step++, covg_before_path);
if(len_before_path != NULL)
status("%zu. Saving supernode length distribution to: %s", step++, len_before_path);
if(tip_cleaning)
status("%zu. Cleaning tips shorter than %zu nodes", step++, min_keep_tip);
if(supernode_cleaning && threshold > 0)
status("%zu. Cleaning supernodes with coverage < %u", step++, threshold);
if(supernode_cleaning && threshold <= 0)
status("%zu. Cleaning supernodes with auto-detected threshold", step++);
if(covg_after_path != NULL)
status("%zu. Saving kmer coverage distribution to: %s", step++, covg_after_path);
if(len_after_path != NULL)
status("%zu. Saving supernode length distribution to: %s", step++, len_after_path);
//
// Decide memory usage
//
bool all_colours_loaded = (ncols <= use_ncols);
bool use_mem_limit = (memargs.mem_to_use_set && num_gfiles > 1) || !ctx_max_kmers;
size_t kmers_in_hash, bits_per_kmer, graph_mem;
size_t per_kmer_per_col_bits = (sizeof(BinaryKmer)+sizeof(Covg)+sizeof(Edges)) * 8;
size_t pop_edges_per_kmer_bits = (all_colours_loaded ? 0 : sizeof(Edges) * 8);
bits_per_kmer = per_kmer_per_col_bits * use_ncols + pop_edges_per_kmer_bits;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
use_mem_limit, &graph_mem);
// Maximise the number of colours we load to fill the mem
size_t max_usencols = (memargs.mem_to_use*8 - pop_edges_per_kmer_bits * kmers_in_hash) /
(per_kmer_per_col_bits * kmers_in_hash);
use_ncols = MIN2(max_usencols, ncols);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
//
// Check output files are writable
//
futil_create_output(out_ctx_path);
// Does nothing if arg is NULL
futil_create_output(covg_before_path);
futil_create_output(covg_after_path);
futil_create_output(len_before_path);
futil_create_output(len_after_path);
// Create db_graph
// Load as many colours as possible
// Use an extra set of edge to take intersections
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, use_ncols, use_ncols,
kmers_in_hash, DBG_ALLOC_COVGS);
// Edges is a special case
size_t num_edges = db_graph.ht.capacity * (use_ncols + !all_colours_loaded);
db_graph.col_edges = ctx_calloc(num_edges, sizeof(Edges));
// Load graph into a single colour
LoadingStats stats = LOAD_STATS_INIT_MACRO;
GraphLoadingPrefs gprefs = {.db_graph = &db_graph,
.boolean_covgs = false,
.must_exist_in_graph = false,
.must_exist_in_edges = NULL,
.empty_colours = false};
// Construct cleaned graph header
GraphFileHeader outhdr;
memset(&outhdr, 0, sizeof(GraphFileHeader));
outhdr.version = CTX_GRAPH_FILEFORMAT;
outhdr.kmer_size = db_graph.kmer_size;
outhdr.num_of_cols = ncols;
outhdr.num_of_bitfields = (db_graph.kmer_size*2+63)/64;
graph_header_alloc(&outhdr, ncols);
// Merge info into header
size_t gcol = 0;
for(i = 0; i < num_gfiles; i++) {
for(j = 0; j < file_filter_num(&gfiles[i].fltr); j++, gcol++) {
fromcol = file_filter_fromcol(&gfiles[i].fltr, j);
intocol = file_filter_intocol(&gfiles[i].fltr, j);
graph_info_merge(&outhdr.ginfo[intocol], &gfiles[i].hdr.ginfo[fromcol]);
}
}
if(ncols > use_ncols) {
graph_files_load_flat(gfiles, num_gfiles, gprefs, &stats);
} else {
for(i = 0; i < num_gfiles; i++)
graph_load(&gfiles[i], gprefs, &stats);
}
char num_kmers_str[100];
ulong_to_str(db_graph.ht.num_kmers, num_kmers_str);
status("Total kmers loaded: %s\n", num_kmers_str);
size_t initial_nkmers = db_graph.ht.num_kmers;
hash_table_print_stats(&db_graph.ht);
uint8_t *visited = ctx_calloc(roundup_bits2bytes(db_graph.ht.capacity), 1);
uint8_t *keep = ctx_calloc(roundup_bits2bytes(db_graph.ht.capacity), 1);
if((supernode_cleaning && threshold <= 0) || covg_before_path || len_before_path)
{
// Get coverage distribution and estimate cleaning threshold
int est_threshold = cleaning_get_threshold(nthreads,
covg_before_path,
len_before_path,
visited, &db_graph);
if(est_threshold < 0) status("Cannot find recommended cleaning threshold");
else status("Recommended cleaning threshold is: %i", est_threshold);
// Use estimated threshold if threshold not set
if(threshold <= 0) {
if(fallback_thresh > 0 && est_threshold < (int)fallback_thresh) {
status("Using fallback threshold: %i", fallback_thresh);
threshold = fallback_thresh;
}
else if(est_threshold >= 0) threshold = est_threshold;
}
}
// Die if we failed to find suitable cleaning threshold
if(supernode_cleaning && threshold <= 0)
die("Need cleaning threshold (--supernodes=<D> or --fallback <D>)");
if(doing_cleaning) {
// Clean graph of tips (if min_keep_tip > 0) and supernodes (if threshold > 0)
clean_graph(nthreads, threshold, min_keep_tip,
covg_after_path, len_after_path,
visited, keep, &db_graph);
}
ctx_free(visited);
ctx_free(keep);
if(doing_cleaning)
{
// Output graph file
Edges *intersect_edges = NULL;
bool kmers_loaded = true;
size_t col, thresh;
// Set output header ginfo cleaned
for(col = 0; col < ncols; col++)
{
cleaning = &outhdr.ginfo[col].cleaning;
cleaning->cleaned_snodes |= supernode_cleaning;
cleaning->cleaned_tips |= tip_cleaning;
// if(tip_cleaning) {
// strbuf_append_str(&outhdr.ginfo[col].sample_name, ".tipclean");
// }
if(supernode_cleaning) {
thresh = cleaning->clean_snodes_thresh;
thresh = cleaning->cleaned_snodes ? MAX2(thresh, (uint32_t)threshold)
: (uint32_t)threshold;
cleaning->clean_snodes_thresh = thresh;
// char name_append[200];
// sprintf(name_append, ".supclean%zu", thresh);
// strbuf_append_str(&outhdr.ginfo[col].sample_name, name_append);
}
}
if(!all_colours_loaded)
{
// We haven't loaded all the colours
// intersect_edges are edges to mask with
// resets graph edges
intersect_edges = db_graph.col_edges;
db_graph.col_edges += db_graph.ht.capacity;
}
// Print stats on removed kmers
size_t removed_nkmers = initial_nkmers - db_graph.ht.num_kmers;
double removed_pct = (100.0 * removed_nkmers) / initial_nkmers;
char removed_str[100], init_str[100];
ulong_to_str(removed_nkmers, removed_str);
ulong_to_str(initial_nkmers, init_str);
status("Removed %s of %s (%.2f%%) kmers", removed_str, init_str, removed_pct);
graph_files_merge(out_ctx_path, gfiles, num_gfiles,
kmers_loaded, all_colours_loaded,
intersect_edges, &outhdr, &db_graph);
// Swap back
if(!all_colours_loaded)
db_graph.col_edges = intersect_edges;
}
ctx_check(db_graph.ht.num_kmers == hash_table_count_kmers(&db_graph.ht));
graph_header_dealloc(&outhdr);
for(i = 0; i < num_gfiles; i++) graph_file_close(&gfiles[i]);
ctx_free(gfiles);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_clean(int argc, char **argv)
{
size_t nthreads = 0, use_ncols = 0;
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_ctx_path = NULL;
int min_keep_tip = -1, unitig_min = -1; // <0 => default, 0 => noclean
uint32_t fallback_thresh = 0;
const char *len_before_path = NULL, *len_after_path = NULL;
const char *covg_before_path = NULL, *covg_after_path = NULL;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'o':
if(out_ctx_path != NULL) cmd_print_usage(NULL);
out_ctx_path = optarg;
break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'N': use_ncols = cmd_uint32_nonzero(cmd, optarg); break;
case 't': cmd_check(!nthreads, cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'T':
cmd_check(min_keep_tip<0, cmd);
min_keep_tip = (optarg != NULL ? (int)cmd_uint32(cmd, optarg) : -1);
break;
case 'S':
case 'U':
cmd_check(unitig_min<0, cmd);
unitig_min = (optarg != NULL ? cmd_uint32(cmd, optarg) : -1);
break;
case 'B': cmd_check(!fallback_thresh, cmd); fallback_thresh = cmd_uint32_nonzero(cmd, optarg); break;
case 'l': cmd_check(!len_before_path, cmd); len_before_path = optarg; break;
case 'L': cmd_check(!len_after_path, cmd); len_after_path = optarg; break;
case 'c': cmd_check(!covg_before_path, cmd); covg_before_path = optarg; break;
case 'C': cmd_check(!covg_after_path, cmd); covg_after_path = optarg; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" clean -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
if(nthreads == 0) nthreads = DEFAULT_NTHREADS;
if(optind >= argc) cmd_print_usage("Please give input graph files");
bool unitig_cleaning = (unitig_min != 0);
bool tip_cleaning = (min_keep_tip != 0);
bool doing_cleaning = (unitig_cleaning || tip_cleaning);
// If you ever want to estimate cleaning threshold without outputting
// a graph, change this to a warning
if(doing_cleaning && out_ctx_path == NULL) {
cmd_print_usage("Please specify --out <out.ctx> for cleaned graph");
// warn("No cleaning being done: you did not specify --out <out.ctx>");
}
if(!doing_cleaning && (covg_after_path || len_after_path)) {
warn("You gave --len-after <out> / --covg-after <out> without "
"any cleaning (set -U, --unitigs or -t, --tips)");
}
if(doing_cleaning && strcmp(out_ctx_path,"-") != 0 &&
!futil_get_force() && futil_file_exists(out_ctx_path))
{
cmd_print_usage("Output file already exists: %s", out_ctx_path);
}
if(fallback_thresh && !unitig_cleaning)
warn("-B, --fallback <T> without --unitigs");
// Use remaining args as graph files
char **gfile_paths = argv + optind;
size_t i, j, num_gfiles = (size_t)(argc - optind);
// Open graph files
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
size_t col, ncols, ctx_max_kmers = 0, ctx_sum_kmers = 0;
ncols = graph_files_open(gfile_paths, gfiles, num_gfiles,
&ctx_max_kmers, &ctx_sum_kmers);
size_t kmer_size = gfiles[0].hdr.kmer_size;
// default to one colour for now
if(use_ncols == 0) use_ncols = 1;
// Flatten if we don't have to remember colours / output a graph
if(out_ctx_path == NULL)
{
ncols = use_ncols = 1;
for(i = 0; i < num_gfiles; i++)
file_filter_flatten(&gfiles[i].fltr, 0);
}
if(ncols < use_ncols) {
warn("I only need %zu colour%s ('--ncols %zu' ignored)",
ncols, util_plural_str(ncols), use_ncols);
use_ncols = ncols;
}
char max_kmers_str[100];
ulong_to_str(ctx_max_kmers, max_kmers_str);
status("%zu input graph%s, max kmers: %s, using %zu colours",
num_gfiles, util_plural_str(num_gfiles), max_kmers_str, use_ncols);
// If no arguments given we default to removing tips < 2*kmer_size
if(min_keep_tip < 0)
min_keep_tip = 2 * kmer_size;
// Warn if any graph files already cleaned
size_t fromcol;
ErrorCleaning *cleaning;
for(i = 0; i < num_gfiles; i++) {
for(j = 0; j < file_filter_num(&gfiles[i].fltr); j++) {
fromcol = file_filter_fromcol(&gfiles[i].fltr, j);
cleaning = &gfiles[i].hdr.ginfo[fromcol].cleaning;
if(cleaning->cleaned_snodes && unitig_cleaning) {
warn("%s:%zu already has unitig cleaning with threshold: <%zu",
file_filter_path(&gfiles[i].fltr), fromcol,
(size_t)cleaning->clean_snodes_thresh);
}
if(cleaning->cleaned_tips && tip_cleaning) {
warn("%s:%zu already has had tip cleaned",
file_filter_path(&gfiles[i].fltr), fromcol);
}
}
}
// Print steps
size_t step = 0;
status("Actions:\n");
if(covg_before_path != NULL)
status("%zu. Saving kmer coverage distribution to: %s", step++, covg_before_path);
if(len_before_path != NULL)
status("%zu. Saving unitig length distribution to: %s", step++, len_before_path);
if(min_keep_tip > 0)
status("%zu. Cleaning tips shorter than %i nodes", step++, min_keep_tip);
if(unitig_min > 0)
status("%zu. Cleaning unitigs with coverage < %i", step++, unitig_min);
if(unitig_min < 0)
status("%zu. Cleaning unitigs with auto-detected threshold", step++);
if(covg_after_path != NULL)
status("%zu. Saving kmer coverage distribution to: %s", step++, covg_after_path);
if(len_after_path != NULL)
status("%zu. Saving unitig length distribution to: %s", step++, len_after_path);
//
// Decide memory usage
//
bool all_colours_loaded = (ncols <= use_ncols);
bool use_mem_limit = (memargs.mem_to_use_set && num_gfiles > 1) || !ctx_max_kmers;
size_t kmers_in_hash, bits_per_kmer, graph_mem;
size_t per_col_bits = (sizeof(Covg)+sizeof(Edges)) * 8;
size_t extra_edge_bits = (all_colours_loaded ? 0 : sizeof(Edges) * 8);
bits_per_kmer = sizeof(BinaryKmer)*8 +
per_col_bits * use_ncols +
extra_edge_bits;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
use_mem_limit, &graph_mem);
// Maximise the number of colours we load to fill the mem
size_t max_usencols = (memargs.mem_to_use*8 -
sizeof(BinaryKmer)*8*kmers_in_hash +
extra_edge_bits*kmers_in_hash) /
(per_col_bits*kmers_in_hash);
use_ncols = MIN2(max_usencols, ncols);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
//
// Check output files are writable
//
futil_create_output(out_ctx_path);
// Does nothing if arg is NULL
futil_create_output(covg_before_path);
futil_create_output(covg_after_path);
futil_create_output(len_before_path);
futil_create_output(len_after_path);
// Create db_graph
// Load as many colours as possible
// Use an extra set of edge to take intersections
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, use_ncols, use_ncols,
kmers_in_hash, DBG_ALLOC_EDGES | DBG_ALLOC_COVGS);
// Extra edges required to hold union of kept edges
Edges *edges_union = NULL;
if(use_ncols < ncols)
edges_union = ctx_calloc(db_graph.ht.capacity, sizeof(Edges));
// Load graph into a single colour
GraphLoadingPrefs gprefs = graph_loading_prefs(&db_graph);
// Construct cleaned graph header
GraphFileHeader outhdr;
memset(&outhdr, 0, sizeof(GraphFileHeader));
for(i = 0; i < num_gfiles; i++)
graph_file_merge_header(&outhdr, &gfiles[i]);
if(ncols > use_ncols)
{
db_graph.num_of_cols = db_graph.num_edge_cols = 1;
SWAP(edges_union, db_graph.col_edges);
graphs_load_files_flat(gfiles, num_gfiles, gprefs, NULL);
SWAP(edges_union, db_graph.col_edges);
db_graph.num_of_cols = db_graph.num_edge_cols = use_ncols;
}
else {
for(i = 0; i < num_gfiles; i++)
graph_load(&gfiles[i], gprefs, NULL);
}
char num_kmers_str[100];
ulong_to_str(db_graph.ht.num_kmers, num_kmers_str);
status("Total kmers loaded: %s\n", num_kmers_str);
size_t initial_nkmers = db_graph.ht.num_kmers;
hash_table_print_stats(&db_graph.ht);
uint8_t *visited = ctx_calloc(roundup_bits2bytes(db_graph.ht.capacity), 1);
uint8_t *keep = ctx_calloc(roundup_bits2bytes(db_graph.ht.capacity), 1);
// Always estimate cleaning threshold
// if(unitig_min <= 0 || covg_before_path || len_before_path)
// {
// Get coverage distribution and estimate cleaning threshold
int est_min_covg = cleaning_get_threshold(nthreads,
covg_before_path,
len_before_path,
visited, &db_graph);
if(est_min_covg < 0) status("Cannot find recommended cleaning threshold");
else status("Recommended cleaning threshold is: %i", est_min_covg);
// Use estimated threshold if threshold not set
if(unitig_min < 0) {
if(fallback_thresh > 0 && est_min_covg < (int)fallback_thresh) {
status("Using fallback threshold: %i", fallback_thresh);
unitig_min = fallback_thresh;
}
else if(est_min_covg >= 0) unitig_min = est_min_covg;
}
// }
// Die if we failed to find suitable cleaning threshold
if(unitig_min < 0)
die("Need cleaning threshold (--unitigs=<D> or --fallback <D>)");
// Cleaning parameters should now be set (>0) or turned off (==0)
ctx_assert(unitig_min >= 0);
ctx_assert(min_keep_tip >= 0);
if(unitig_min || min_keep_tip)
{
// Clean graph of tips (if min_keep_tip > 0) and unitigs (if threshold > 0)
clean_graph(nthreads, unitig_min, min_keep_tip,
covg_after_path, len_after_path,
visited, keep, &db_graph);
}
ctx_free(visited);
ctx_free(keep);
if(out_ctx_path != NULL)
{
// Set output header ginfo cleaned
for(col = 0; col < ncols; col++)
{
cleaning = &outhdr.ginfo[col].cleaning;
cleaning->cleaned_snodes |= unitig_cleaning;
cleaning->cleaned_tips |= tip_cleaning;
// if(tip_cleaning) {
// strbuf_append_str(&outhdr.ginfo[col].sample_name, ".tipclean");
// }
if(unitig_cleaning) {
size_t thresh = cleaning->clean_snodes_thresh;
thresh = cleaning->cleaned_snodes ? MAX2(thresh, (uint32_t)unitig_min)
: (uint32_t)unitig_min;
cleaning->clean_snodes_thresh = thresh;
// char name_append[200];
// sprintf(name_append, ".supclean%zu", thresh);
// strbuf_append_str(&outhdr.ginfo[col].sample_name, name_append);
}
}
// Print stats on removed kmers
size_t removed_nkmers = initial_nkmers - db_graph.ht.num_kmers;
double removed_pct = (100.0 * removed_nkmers) / initial_nkmers;
char removed_str[100], init_str[100];
ulong_to_str(removed_nkmers, removed_str);
ulong_to_str(initial_nkmers, init_str);
status("Removed %s of %s (%.2f%%) kmers", removed_str, init_str, removed_pct);
// kmers_loaded=true
graph_writer_merge(out_ctx_path, gfiles, num_gfiles,
true, all_colours_loaded,
edges_union, &outhdr, &db_graph);
}
ctx_check(db_graph.ht.num_kmers == hash_table_count_kmers(&db_graph.ht));
// TODO: report kmer coverage for each sample
graph_header_dealloc(&outhdr);
for(i = 0; i < num_gfiles; i++) graph_file_close(&gfiles[i]);
ctx_free(gfiles);
ctx_free(edges_union);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_view(int argc, char **argv)
{
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// TODO:
// print_action actions[argc];
// bool read_kmers = false;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
cmd_print_usage("`"CMD" "SUBCMD" -h` for help. Bad option: %s", argv[optind-1]);
default:
cmd_print_usage("Programmer fail. Tell Isaac.");
}
}
if(print_kmers) parse_kmers = 1;
bool no_flags = (!print_info && !parse_kmers && !print_kmers);
if(no_flags) { print_info = parse_kmers = 1; }
if(optind+1 != argc) cmd_print_usage("Require one input graph file (.ctx)");
char *path = argv[optind];
size_t num_errors = 0, num_warnings = 0;
GraphFileReader gfile;
memset(&gfile, 0, sizeof(gfile));
int ret = graph_file_open(&gfile, path);
if(ret == 0) die("Cannot open file: %s", path);
if(print_info)
{
char fsize_str[50];
bytes_to_str((size_t)gfile.file_size, 0, fsize_str);
printf("Loading file: %s\n", file_filter_path(&gfile.fltr));
printf("File size: %s\n", fsize_str);
printf("----\n");
}
size_t i, col, ncols = file_filter_into_ncols(&gfile.fltr);
size_t kmer_size = gfile.hdr.kmer_size;
ctx_assert(ncols > 0);
GraphFileHeader hdr;
memset(&hdr, 0, sizeof(hdr));
graph_file_merge_header(&hdr, &gfile);
uint64_t nkmers_read = 0, nkmers_loaded = 0;
uint64_t num_all_zero_kmers = 0, num_zero_covg_kmers = 0;
uint64_t *col_nkmers, *col_sum_covgs;
col_nkmers = ctx_calloc(ncols, sizeof(col_nkmers[0]));
col_sum_covgs = ctx_calloc(ncols, sizeof(col_sum_covgs[0]));
// Print header
if(print_info) print_header(&hdr, gfile.num_of_kmers);
BinaryKmer bkmer;
Covg covgs[ncols], keep_kmer;
Edges edges[ncols];
bool direct_read = file_filter_is_direct(&gfile.fltr);
if(parse_kmers || print_kmers)
{
if(print_info && print_kmers) printf("----\n");
for(; graph_file_read_reset(&gfile, &bkmer, covgs, edges); nkmers_read++)
{
// If kmer has no covg in any samples -> don't load
keep_kmer = 0;
for(col = 0; col < ncols; col++) {
col_nkmers[col] += (covgs[col] > 0);
col_sum_covgs[col] += covgs[col];
keep_kmer |= covgs[col];
}
if(!direct_read && !keep_kmer) continue;
nkmers_loaded++;
/* Kmer Checks */
// graph_file_read_reset() already checks for:
// 1. oversized kmers
// 2. kmers with covg 0 in all colours
// 3. edges without coverage in a colour
// Check for all-zeros (i.e. all As kmer: AAAAAA)
uint64_t kmer_words_or = 0;
for(i = 0; i < hdr.num_of_bitfields; i++)
kmer_words_or |= bkmer.b[i];
if(kmer_words_or == 0)
{
if(num_all_zero_kmers == 1)
{
loading_error("more than one all 'A's kmers seen [index: %"PRIu64"]\n",
nkmers_read);
}
num_all_zero_kmers++;
}
// Check covg is 0 for all colours
for(i = 0; i < ncols && covgs[i] == 0; i++);
num_zero_covg_kmers += (i == ncols);
// Print
if(print_kmers)
db_graph_print_kmer2(bkmer, covgs, edges, ncols, kmer_size, stdout);
}
}
// check for various reading errors
// if(errno != 0)
// loading_error("errno set [%i]: %s\n", (int)errno, strerror(errno));
int err = ferror(gfile.fh);
if(err != 0)
loading_error("occurred after file reading [%i]\n", err);
char nstr[50];
if(print_kmers || parse_kmers)
{
// file_size is set to -1 if we are reading from a stream,
// therefore won't be able to check number of kmers read
if(gfile.file_size != -1 && nkmers_read != (uint64_t)gfile.num_of_kmers) {
loading_warning("Expected %zu kmers, read %zu\n",
(size_t)gfile.num_of_kmers, (size_t)nkmers_read);
}
if(num_all_zero_kmers > 1)
{
loading_error("%s all-zero-kmers seen\n",
ulong_to_str(num_all_zero_kmers, nstr));
}
if(num_zero_covg_kmers > 0)
{
loading_warning("%s kmers have no coverage in any colour\n",
ulong_to_str(num_zero_covg_kmers, nstr));
}
}
// Count warnings printed by graph_file_reader.c
num_warnings += gfile.error_zero_covg;
num_warnings += gfile.error_missing_covg;
// Can only print these stats if we're read in the kmers
if((print_kmers || parse_kmers) && print_info)
{
// print kmer coverage per sample
printf("\n---- Per colour stats\n");
printf("num. kmers:");
for(col = 0; col < ncols; col++)
printf("\t%s", ulong_to_str(col_nkmers[col], nstr));
printf("\n");
printf("sum coverage:");
for(col = 0; col < ncols; col++)
printf("\t%s", ulong_to_str(col_sum_covgs[col], nstr));
printf("\n");
printf("kmer coverage:");
for(col = 0; col < ncols; col++)
printf("\t%.2f", safe_frac(col_sum_covgs[col], col_nkmers[col]));
printf("\n");
// Overall stats
uint64_t sum_covgs = 0;
double mean_kmer_covg = 0.0;
for(col = 0; col < ncols; col++) sum_covgs += col_sum_covgs[col];
mean_kmer_covg = nkmers_loaded ? (double)sum_covgs / nkmers_loaded : 0.0;
printf("\n---- Overall stats\n");
printf("Total kmers: %s\n", ulong_to_str(nkmers_loaded, nstr));
printf("Total coverage: %s\n", ulong_to_str(sum_covgs, nstr));
printf("Mean coverage: %s\n", double_to_str(mean_kmer_covg, 2, nstr));
}
if(print_info)
{
// Print memory stats
uint64_t mem, capacity, num_buckets, req_capacity;
uint8_t bucket_size;
req_capacity = (size_t)(gfile.num_of_kmers / IDEAL_OCCUPANCY);
capacity = hash_table_cap(req_capacity, &num_buckets, &bucket_size);
mem = ht_mem(bucket_size, num_buckets,
sizeof(BinaryKmer)*8 + ncols*(sizeof(Covg)+sizeof(Edges))*8);
char memstr[100], capacitystr[100], bucket_size_str[100], num_buckets_str[100];
bytes_to_str(mem, 1, memstr);
ulong_to_str(capacity, capacitystr);
ulong_to_str(bucket_size, bucket_size_str);
ulong_to_str(num_buckets, num_buckets_str);
size_t mem_height = (size_t)__builtin_ctzl(num_buckets);
printf("\n---- Memory\n");
printf("memory required: %s [capacity: %s]\n", memstr, capacitystr);
printf(" bucket size: %s; number of buckets: %s\n",
bucket_size_str, num_buckets_str);
printf(" --kmer_size %zu --mem_height %zu --mem_width %i\n",
kmer_size, mem_height, bucket_size);
}
if((print_kmers || parse_kmers) && print_info)
{
printf("\n----\n");
if(num_warnings > 0 || num_errors > 0) {
printf("Warnings: %zu; Errors: %zu\n",
(size_t)num_warnings, (size_t)num_errors);
}
if(num_errors == 0)
printf(num_warnings ? "Graph may be ok\n" : "Graph is valid\n");
}
ctx_free(col_nkmers);
ctx_free(col_sum_covgs);
// Close file (which zeros it)
graph_file_close(&gfile);
graph_header_dealloc(&hdr);
return num_errors ? EXIT_FAILURE : EXIT_SUCCESS;
}
