int ctx_correct(int argc, char **argv)
{
size_t i;
struct ReadThreadCmdArgs args;
read_thread_args_alloc(&args);
read_thread_args_parse(&args, argc, argv, longopts, true);
GraphFileReader *gfile = &args.gfile;
GPathFileBuffer *gpfiles = &args.gpfiles;
CorrectAlnInputBuffer *inputs = &args.inputs;
// Update colours in graph file - sample in 0, all others in 1
size_t ncols = gpath_load_sample_pop(gfile, 1, gpfiles->b, gpfiles->len,
args.colour);
// Check for compatibility between graph files and link files
graphs_gpaths_compatible(gfile, 1, gpfiles->b, gpfiles->len, 1);
int64_t ctx_num_kmers = gfile->num_of_kmers;
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem, path_mem, total_mem;
// 1 bit needed per kmer if we need to keep track of noreseed
bits_per_kmer = sizeof(BinaryKmer)*8 + sizeof(Edges)*8 +
(gpfiles->len > 0 ? sizeof(GPath*)*8 : 0) +
ncols; // in colour
kmers_in_hash = cmd_get_kmers_in_hash(args.memargs.mem_to_use,
args.memargs.mem_to_use_set,
args.memargs.num_kmers,
args.memargs.num_kmers_set,
bits_per_kmer,
ctx_num_kmers, ctx_num_kmers,
false, &graph_mem);
// Paths memory
size_t rem_mem = args.memargs.mem_to_use - MIN2(args.memargs.mem_to_use, graph_mem);
path_mem = gpath_reader_mem_req(gpfiles->b, gpfiles->len, ncols, rem_mem, false,
kmers_in_hash, false);
cmd_print_mem(path_mem, "paths");
// Shift path store memory from graphs->paths
graph_mem -= sizeof(GPath*)*kmers_in_hash;
path_mem += sizeof(GPath*)*kmers_in_hash;
// Total memory
total_mem = graph_mem + path_mem;
cmd_check_mem_limit(args.memargs.mem_to_use, total_mem);
//
// Check we can write all output files
//
// Open output files
SeqOutput *outputs = ctx_calloc(inputs->len, sizeof(SeqOutput));
bool err_occurred = false;
for(i = 0; i < inputs->len && !err_occurred; i++)
{
CorrectAlnInput *input = &inputs->b[i];
// We loaded target colour into colour zero
input->crt_params.ctxcol = input->crt_params.ctpcol = 0;
bool is_pe = asyncio_task_is_pe(&input->files);
err_occurred = !seqout_open(&outputs[i], input->out_base, args.fmt, is_pe);
input->output = &outputs[i];
}
// Abandon if some of the output files already exist
if(err_occurred) {
for(i = 0; i < inputs->len; i++)
seqout_close(&outputs[i], true);
die("Error creating output files");
}
//
// Allocate memory
//
dBGraph db_graph;
db_graph_alloc(&db_graph, gfile->hdr.kmer_size, ncols, 1, kmers_in_hash,
DBG_ALLOC_EDGES | DBG_ALLOC_NODE_IN_COL);
// Create a path store that does not tracks path counts
gpath_reader_alloc_gpstore(gpfiles->b, gpfiles->len, path_mem, false, &db_graph);
//
// Load Graph and link files
//
GraphLoadingPrefs gprefs = graph_loading_prefs(&db_graph);
gprefs.empty_colours = true;
// Load graph, print stats, close file
graph_load(gfile, gprefs, NULL);
hash_table_print_stats_brief(&db_graph.ht);
graph_file_close(gfile);
// Load link files
for(i = 0; i < gpfiles->len; i++) {
gpath_reader_load(&gpfiles->b[i], GPATH_DIE_MISSING_KMERS, &db_graph);
gpath_reader_close(&gpfiles->b[i]);
}
//
// Run alignment
//
correct_reads(inputs->b, inputs->len,
args.dump_seq_sizes, args.dump_frag_sizes,
args.fq_zero, args.append_orig_seq,
args.nthreads, &db_graph);
// Close and free output files
for(i = 0; i < inputs->len; i++)
seqout_close(&outputs[i], false);
ctx_free(outputs);
// Closes input files
read_thread_args_dealloc(&args);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_contigs(int argc, char **argv)
{
size_t nthreads = 0;
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_path = NULL;
size_t i, contig_limit = 0, colour = 0;
bool cmd_reseed = false, cmd_no_reseed = false; // -r, -R
const char *conf_table_path = NULL; // save confidence table to here
bool use_missing_info_check = true, seed_with_unused_paths = false;
double min_step_confid = -1.0, min_cumul_confid = -1.0; // < 0 => no min
// Read length and expected depth for calculating confidences
size_t genome_size = 0;
seq_file_t *tmp_seed_file = NULL;
SeqFilePtrBuffer seed_buf;
seq_file_ptr_buf_alloc(&seed_buf, 16);
GPathReader tmp_gpfile;
GPathFileBuffer gpfiles;
gpfile_buf_alloc(&gpfiles, 8);
// Arg parsing
char cmd[100], shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'o': cmd_check(!out_path,cmd); out_path = optarg; break;
case 't': cmd_check(!nthreads,cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'p':
memset(&tmp_gpfile, 0, sizeof(GPathReader));
gpath_reader_open(&tmp_gpfile, optarg);
gpfile_buf_push(&gpfiles, &tmp_gpfile, 1);
break;
case '1':
case 's': // --seed <in.fa>
if((tmp_seed_file = seq_open(optarg)) == NULL)
die("Cannot read --seed file: %s", optarg);
seq_file_ptr_buf_add(&seed_buf, tmp_seed_file);
break;
case 'r': cmd_check(!cmd_reseed,cmd); cmd_reseed = true; break;
case 'R': cmd_check(!cmd_no_reseed,cmd); cmd_no_reseed = true; break;
case 'N':
cmd_check(!contig_limit,cmd);
contig_limit = cmd_uint32_nonzero(cmd, optarg);
break;
case 'c': cmd_check(!colour,cmd); colour = cmd_uint32(cmd, optarg); break;
case 'G': cmd_check(!genome_size,cmd); genome_size = cmd_bases(cmd, optarg); break;
case 'S': cmd_check(!conf_table_path,cmd); conf_table_path = optarg; break;
case 'M': cmd_check(use_missing_info_check,cmd); use_missing_info_check = false; break;
case 'P': cmd_check(!seed_with_unused_paths,cmd); seed_with_unused_paths = true; break;
case 'C':
cmd_check(min_cumul_confid < 0,cmd);
min_cumul_confid = cmd_udouble(cmd,optarg);
if(min_cumul_confid > 1) die("%s must be 0 <= x <= 1", cmd);
break;
case 'T':
cmd_check(min_step_confid < 0,cmd);
min_step_confid = cmd_udouble(cmd,optarg);
if(min_step_confid > 1) die("%s must be 0 <= x <= 1", cmd);
break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
die("`"CMD" contigs -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
if(cmd_no_reseed && cmd_reseed)
cmd_print_usage("Cannot specify both -r and -R");
if(contig_limit && seed_with_unused_paths)
cmd_print_usage("Cannot combine --ncontigs with --use-seed-paths");
bool sample_with_replacement = cmd_reseed;
// Defaults
if(nthreads == 0) nthreads = DEFAULT_NTHREADS;
if(!seed_buf.len && !contig_limit && sample_with_replacement) {
cmd_print_usage("Please specify one or more of: "
"--no-reseed | --ncontigs | --seed <in.fa>");
}
if(optind >= argc) cmd_print_usage("Require input graph files (.ctx)");
//
// Open graph files
//
const size_t num_gfiles = argc - optind;
char **graph_paths = argv + optind;
ctx_assert(num_gfiles > 0);
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
size_t ncols, ctx_max_kmers = 0, ctx_sum_kmers = 0;
graph_files_open(graph_paths, gfiles, num_gfiles,
&ctx_max_kmers, &ctx_sum_kmers);
// char *ctx_path = argv[optind];
//
// Open Graph file
//
// GraphFileReader gfile;
// memset(&gfile, 0, sizeof(GraphFileReader));
// graph_file_open(&gfile, ctx_path);
// Update colours in graph file - sample in 0, all others in 1
// never need more than two colours
ncols = gpath_load_sample_pop(gfiles, num_gfiles,
gpfiles.b, gpfiles.len, colour);
// Check for compatibility between graph files and path files
// pop_colour is colour 1
graphs_gpaths_compatible(gfiles, num_gfiles, gpfiles.b, gpfiles.len, 1);
if(!genome_size)
{
char nk_str[50];
if(ctx_max_kmers <= 0) die("Please pass --genome <G> if streaming");
genome_size = ctx_max_kmers;
ulong_to_str(genome_size, nk_str);
status("Taking number of kmers as genome size: %s", nk_str);
}
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem, path_mem, total_mem;
// 1 bit needed per kmer if we need to keep track of kmer usage
bits_per_kmer = sizeof(BinaryKmer)*8 + sizeof(Edges)*8 + sizeof(GPath*)*8 +
ncols + !sample_with_replacement;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
false, &graph_mem);
// Paths memory
size_t rem_mem = memargs.mem_to_use - MIN2(memargs.mem_to_use, graph_mem);
path_mem = gpath_reader_mem_req(gpfiles.b, gpfiles.len, ncols, rem_mem, false);
// Shift path store memory from graphs->paths
graph_mem -= sizeof(GPath*)*kmers_in_hash;
path_mem += sizeof(GPath*)*kmers_in_hash;
cmd_print_mem(path_mem, "paths");
// Total memory
total_mem = graph_mem + path_mem;
cmd_check_mem_limit(memargs.mem_to_use, total_mem);
// Load contig hist distribution from ctp files
ZeroSizeBuffer contig_hist;
memset(&contig_hist, 0, sizeof(contig_hist));
for(i = 0; i < gpfiles.len; i++) {
gpath_reader_load_contig_hist(gpfiles.b[i].json,
gpfiles.b[i].fltr.path.b,
file_filter_fromcol(&gpfiles.b[i].fltr, 0),
&contig_hist);
}
// Calculate confidences, only for one colour
ContigConfidenceTable conf_table;
conf_table_alloc(&conf_table, 1);
conf_table_update_hist(&conf_table, 0, genome_size,
contig_hist.b, contig_hist.len);
if(conf_table_path != NULL) {
conf_table_save(&conf_table, conf_table_path);
}
zsize_buf_dealloc(&contig_hist);
//
// Output file if printing
//
FILE *fout = out_path ? futil_fopen_create(out_path, "w") : NULL;
// Allocate
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, ncols, 1, kmers_in_hash,
DBG_ALLOC_EDGES | DBG_ALLOC_NODE_IN_COL);
// Paths
gpath_reader_alloc_gpstore(gpfiles.b, gpfiles.len, path_mem,
false, &db_graph);
uint8_t *visited = NULL;
if(!sample_with_replacement)
visited = ctx_calloc(roundup_bits2bytes(db_graph.ht.capacity), 1);
// Load graph
LoadingStats stats = LOAD_STATS_INIT_MACRO;
GraphLoadingPrefs gprefs = {.db_graph = &db_graph,
.boolean_covgs = false,
.must_exist_in_graph = false,
.empty_colours = true};
for(i = 0; i < num_gfiles; i++) {
graph_load(&gfiles[i], gprefs, &stats);
graph_file_close(&gfiles[i]);
gprefs.empty_colours = false;
}
ctx_free(gfiles);
hash_table_print_stats(&db_graph.ht);
// Load path files
for(i = 0; i < gpfiles.len; i++) {
gpath_reader_load(&gpfiles.b[i], GPATH_DIE_MISSING_KMERS, &db_graph);
gpath_reader_close(&gpfiles.b[i]);
}
gpfile_buf_dealloc(&gpfiles);
AssembleContigStats assem_stats;
assemble_contigs_stats_init(&assem_stats);
assemble_contigs(nthreads, seed_buf.b, seed_buf.len,
contig_limit, visited,
use_missing_info_check, seed_with_unused_paths,
min_step_confid, min_cumul_confid,
fout, out_path, &assem_stats, &conf_table,
&db_graph, 0); // Sample always loaded into colour zero
if(fout && fout != stdout) fclose(fout);
assemble_contigs_stats_print(&assem_stats);
assemble_contigs_stats_destroy(&assem_stats);
conf_table_dealloc(&conf_table);
for(i = 0; i < seed_buf.len; i++)
seq_close(seed_buf.b[i]);
seq_file_ptr_buf_dealloc(&seed_buf);
ctx_free(visited);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
