void _construct_graph_with_paths(dBGraph *graph,
size_t kmer_size, size_t ncols,
char **seqs, size_t nseqs,
CorrectAlnParam path_params)
{
size_t i;
db_graph_alloc(graph, kmer_size, ncols, ncols, 1024);
// Graph data
graph->bktlocks = ctx_calloc(roundup_bits2bytes(graph->ht.num_of_buckets), 1);
graph->col_edges = ctx_calloc(graph->ht.capacity * ncols, sizeof(Edges));
graph->col_covgs = ctx_calloc(graph->ht.capacity * ncols, sizeof(Covg));
graph->node_in_cols = ctx_calloc(roundup_bits2bytes(graph->ht.capacity) * ncols, 1);
// Path data
path_store_alloc(&graph->pstore, 1024, true, graph->ht.capacity, ncols);
graph->pstore.kmer_locks = ctx_calloc(roundup_bits2bytes(graph->ht.capacity), 1);
// Build graph
for(i = 0; i < nseqs; i++)
build_graph_from_str_mt(graph, 0, seqs[i], strlen(seqs[i]));
graph->num_of_cols_used = MAX2(graph->num_of_cols_used, 1);
GenPathWorker *gen_path_wrkr = gen_paths_workers_alloc(1, graph, NULL);
for(i = 0; i < nseqs; i++)
gen_paths_from_str_mt(gen_path_wrkr, seqs[i], path_params);
gen_paths_workers_dealloc(gen_path_wrkr, 1);
}
static void _test_add_paths()
{
test_status("Testing adding paths in generate_paths.c and gpath_fetch()");
// Construct 1 colour graph with kmer-size=11
dBGraph graph;
size_t kmer_size = 11, ncols = 1;
db_graph_alloc(&graph, kmer_size, ncols, ncols, 1024,
DBG_ALLOC_EDGES | DBG_ALLOC_COVGS |
DBG_ALLOC_BKTLOCKS | DBG_ALLOC_NODE_IN_COL);
// Create a path store that tracks path counts
gpath_store_alloc(&graph.gpstore,
graph.num_of_cols, graph.ht.capacity,
0, ONE_MEGABYTE, true, false);
// Create path hash table for fast lookup
gpath_hash_alloc(&graph.gphash, &graph.gpstore, ONE_MEGABYTE);
build_graph_from_str_mt(&graph, 0, seq0, strlen(seq0));
build_graph_from_str_mt(&graph, 0, seq1, strlen(seq1));
build_graph_from_str_mt(&graph, 0, seq2, strlen(seq2));
build_graph_from_str_mt(&graph, 0, seq3, strlen(seq3));
// Set up alignment correction params
CorrectAlnParam params = {.ctpcol = 0, .ctxcol = 0,
.frag_len_min = 0, .frag_len_max = 0,
.one_way_gap_traverse = true, .use_end_check = true,
.max_context = 10,
.gap_variance = 0.1, .gap_wiggle = 5};
all_tests_add_paths(&graph, seq0, params, 5, 5); // path lens: 3+3+2+2+2
all_tests_add_paths(&graph, seq1, params, 5, 2); // path lens: 3+3+2+2+2
all_tests_add_paths(&graph, seq2, params, 3, 2); // path lens: 1+1+1
all_tests_add_paths(&graph, seq3, params, 2, 1); // path lens: 1+1
// Test path store
gpath_checks_all_paths(&graph, 1); // use one thread
// Test path content
_check_node_paths(kmerA, kmerApaths, NPATHS_A, 0, &graph);
_check_node_paths(kmerB, kmerBpaths, NPATHS_B, 0, &graph);
_check_node_paths(kmerAB, kmerABpaths, NPATHS_AB, 0, &graph);
_check_node_paths(kmerC, kmerCpaths, NPATHS_C, 0, &graph);
_check_node_paths(kmerG, kmerGpaths, NPATHS_G, 0, &graph);
_check_node_paths(kmerF, kmerFpaths, NPATHS_F, 0, &graph);
_check_node_paths(kmerE, kmerEpaths, NPATHS_E, 0, &graph);
_check_node_paths(kmerD, kmerDpaths, NPATHS_D, 0, &graph);
_check_node_paths(kmerDEF,kmerDEFpaths,NPATHS_DEF,0, &graph);
_check_node_paths(kmerDE, kmerDEpaths, NPATHS_DE, 0, &graph);
db_graph_dealloc(&graph);
}
void test_paths()
{
_test_add_paths();
}
int main(int argc, char **argv)
{
(void)argc; (void)argv;
cortex_init();
cmd_init(argc, argv);
if(argc != 3) die("usage: ./debug <in.ctp> <in.ctx>");
const char *out_path = argv[2];
GPathReader pfile;
memset(&pfile, 0, sizeof(GPathReader));
gpath_reader_open(&pfile, argv[1], true);
status("Got file with %zu colours", pfile.ncolours);
size_t i, kmer_size = 7, ncols = 3;
gpath_reader_check(&pfile, kmer_size, ncols);
gzFile gzout = futil_gzopen_create(out_path, "w");
dBGraph db_graph;
db_graph_alloc(&db_graph, kmer_size, ncols, 1, 1024, DBG_ALLOC_EDGES);
// Create a path store that tracks path counts
gpath_store_alloc(&db_graph.gpstore,
db_graph.num_of_cols, db_graph.ht.capacity,
ONE_MEGABYTE, true, false);
// Create path hash table for fast lookup
gpath_hash_alloc(&db_graph.gphash, &db_graph.gpstore, ONE_MEGABYTE);
// Set sample names
for(i = 0; i < pfile.ncolours; i++) {
const char *sample_name = gpath_reader_get_sample_name(&pfile, i);
ctx_assert(sample_name != NULL);
strbuf_set(&db_graph.ginfo[i].sample_name, sample_name);
}
// Load path files, add kmers that are missing
gpath_reader_load(&pfile, GPATH_ADD_MISSING_KMERS, &db_graph);
hash_table_print_stats(&db_graph.ht);
// Write output file
gpath_save(gzout, out_path, 1, true, NULL, NULL, &pfile.json, 1, &db_graph);
gzclose(gzout);
// Checks
// gpath_checks_all_paths(&db_graph, 2); // use two threads
gpath_checks_counts(&db_graph);
// Clean up
gpath_reader_close(&pfile);
db_graph_dealloc(&db_graph);
cortex_destroy();
return EXIT_SUCCESS;
}
void test_supernode()
{
test_status("testing supernode_find()...");
// Construct 1 colour graph with kmer-size=11
dBGraph graph;
size_t kmer_size = 19, ncols = 1;
db_graph_alloc(&graph, kmer_size, ncols, ncols, 1024,
DBG_ALLOC_EDGES | DBG_ALLOC_COVGS | DBG_ALLOC_BKTLOCKS);
#define NSEQ 7
const char *seq[NSEQ]
= {"AGAGAGAGAGAGAGAGAGAGAGAG",
"AAAAAAAAAAAAAAAAAAAAAAAAAA",
"ATATATATATATATATATATATATATAT",
"CGTTCGCGCATGGCCCACG",
"GAACCAATCGGTCGACTGT",
"CCCCGCAAAGTCCACTTAGTGTAAGGTACAAATTCTGCAGAGTTGCTGGATCAGCGATAC",
"TCAATCCGATAGCAACCCGGTCCAA""TCAATCCGATAGCAACCCGGTCCAA"};
const char *ans[NSEQ]
= {"AGAGAGAGAGAGAGAGAGAG", // key AGAGAGAGAGAGAGAGAGA < CTCTCTCTCTCTCTCTCTC
"AAAAAAAAAAAAAAAAAAA",
"ATATATATATATATATATA",
"CGTGGGCCATGCGCGAACG",
"ACAGTCGACCGATTGGTTC",
"CCCCGCAAAGTCCACTTAGTGTAAGGTACAAATTCTGCAGAGTTGCTGGATCAGCGATAC",
"AACCCGGTCCAATCAATCCGATAGCAACCCGGTCCAATCAATC"};
// Load all seq into colour 0
size_t i;
for(i = 0; i < NSEQ; i++)
build_graph_from_str_mt(&graph, 0, seq[i], strlen(seq[i]), false);
pull_out_supernodes(seq, ans, NSEQ, &graph);
db_graph_dealloc(&graph);
}
int ctx_rmsubstr(int argc, char **argv)
{
struct MemArgs memargs = MEM_ARGS_INIT;
size_t kmer_size = 0, nthreads = 0;
const char *output_file = NULL;
seq_format fmt = SEQ_FMT_FASTA;
bool invert = false;
// Arg parsing
char cmd[100], shortopts[100];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'o': cmd_check(!output_file, cmd); output_file = optarg; break;
case 't': cmd_check(!nthreads, cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'k': cmd_check(!kmer_size,cmd); kmer_size = cmd_uint32(cmd, optarg); break;
case 'F': cmd_check(fmt==SEQ_FMT_FASTA, cmd); fmt = cmd_parse_format(cmd, optarg); break;
case 'v': cmd_check(!invert,cmd); invert = true; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
cmd_print_usage("`"CMD" rmsubstr -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
// Defaults
if(!nthreads) nthreads = DEFAULT_NTHREADS;
if(!kmer_size) kmer_size = DEFAULT_KMER;
if(!(kmer_size&1)) cmd_print_usage("Kmer size must be odd");
if(kmer_size < MIN_KMER_SIZE) cmd_print_usage("Kmer size too small (recompile)");
if(kmer_size > MAX_KMER_SIZE) cmd_print_usage("Kmer size too large (recompile?)");
if(optind >= argc)
cmd_print_usage("Please specify at least one input sequence file (.fq, .fq etc.)");
size_t i, num_seq_files = argc - optind;
char **seq_paths = argv + optind;
seq_file_t **seq_files = ctx_calloc(num_seq_files, sizeof(seq_file_t*));
for(i = 0; i < num_seq_files; i++)
if((seq_files[i] = seq_open(seq_paths[i])) == NULL)
die("Cannot read sequence file %s", seq_paths[i]);
// Estimate number of bases
// set to -1 if we cannot calc
int64_t est_num_bases = seq_est_seq_bases(seq_files, num_seq_files);
if(est_num_bases < 0) {
warn("Cannot get file sizes, using pipes");
est_num_bases = memargs.num_kmers * IDEAL_OCCUPANCY;
}
status("[memory] Estimated number of bases: %li", (long)est_num_bases);
// Use file sizes to decide on memory
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem;
bits_per_kmer = sizeof(BinaryKmer)*8 +
sizeof(KONodeList) + sizeof(KOccur) + // see kmer_occur.h
8; // 1 byte per kmer for each base to load sequence files
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
est_num_bases, est_num_bases,
false, &graph_mem);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
//
// Open output file
//
if(output_file == NULL) output_file = "-";
FILE *fout = futil_fopen_create(output_file, "w");
//
// Set up memory
//
dBGraph db_graph;
db_graph_alloc(&db_graph, kmer_size, 1, 0, kmers_in_hash, DBG_ALLOC_BKTLOCKS);
//
// Load reference sequence into a read buffer
//
ReadBuffer rbuf;
read_buf_alloc(&rbuf, 1024);
seq_load_all_reads(seq_files, num_seq_files, &rbuf);
// Check for reads too short
for(i = 0; i < rbuf.len && rbuf.b[i].seq.end >= kmer_size; i++) {}
if(i < rbuf.len)
warn("Reads shorter than kmer size (%zu) will not be filtered", kmer_size);
KOGraph kograph = kograph_create(rbuf.b, rbuf.len, true, 0,
nthreads, &db_graph);
size_t num_reads = rbuf.len, num_reads_printed = 0, num_bad_reads = 0;
// Loop over reads printing those that are not substrings
int ret;
for(i = 0; i < rbuf.len; i++) {
ret = _is_substr(&rbuf, i, &kograph, &db_graph);
if(ret == -1) num_bad_reads++;
else if((ret && invert) || (!ret && !invert)) {
seqout_print_read(&rbuf.b[i], fmt, fout);
num_reads_printed++;
}
}
char num_reads_str[100], num_reads_printed_str[100], num_bad_reads_str[100];
ulong_to_str(num_reads, num_reads_str);
ulong_to_str(num_reads_printed, num_reads_printed_str);
ulong_to_str(num_bad_reads, num_bad_reads_str);
status("Printed %s / %s (%.1f%%) to %s",
num_reads_printed_str, num_reads_str,
!num_reads ? 0.0 : (100.0 * num_reads_printed) / num_reads,
futil_outpath_str(output_file));
if(num_bad_reads > 0) {
status("Bad reads: %s / %s (%.1f%%) - no kmer {ACGT} of length %zu",
num_bad_reads_str, num_reads_str,
(100.0 * num_bad_reads) / num_reads,
kmer_size);
}
fclose(fout);
kograph_dealloc(&kograph);
// Free sequence memory
for(i = 0; i < rbuf.len; i++) seq_read_dealloc(&rbuf.b[i]);
read_buf_dealloc(&rbuf);
ctx_free(seq_files);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
void test_graph_crawler()
{
test_status("Testing graph crawler...");
// Construct 1 colour graph with kmer-size=11
dBGraph graph;
const size_t kmer_size = 11, ncols = 3;
db_graph_alloc(&graph, kmer_size, ncols, 1, 2048,
DBG_ALLOC_EDGES | DBG_ALLOC_NODE_IN_COL | DBG_ALLOC_BKTLOCKS);
char graphseq[3][77] =
// < X X X...............
{"GTTCCAGAGCGGAGGTCTCCCAACAACATGGTATAAGTTGTCTAGCCCCGGTTCGCGCGGGTACTTCTTACAGCGC",
"GTTCCAGAGCGGAGGTCTCCCAACAACTTGGTATAAGTTGTCTAGTCCCGGTTCGCGCGGCATTTCAGCATTGTTA",
"GTTCCAGAGCGCGACAGAGTGCATATCACGCTAAGCACAGCCCTCTTCTATCTGCTTTTAAATGGATCAATAATCG"};
build_graph_from_str_mt(&graph, 0, graphseq[0], strlen(graphseq[0]));
build_graph_from_str_mt(&graph, 1, graphseq[1], strlen(graphseq[1]));
build_graph_from_str_mt(&graph, 2, graphseq[2], strlen(graphseq[2]));
// Crawl graph
GraphCrawler crawler;
graph_crawler_alloc(&crawler, &graph);
dBNode node = db_graph_find_str(&graph, graphseq[0]);
dBNode next_node = db_graph_find_str(&graph, graphseq[0]+1);
TASSERT(node.key != HASH_NOT_FOUND);
TASSERT(next_node.key != HASH_NOT_FOUND);
BinaryKmer bkey = db_node_get_bkmer(&graph, node.key);
Edges edges = db_node_get_edges(&graph, node.key, 0);
dBNode next_nodes[4];
Nucleotide next_nucs[4];
size_t i, p, num_next, next_idx;
num_next = db_graph_next_nodes(&graph, bkey, node.orient, edges,
next_nodes, next_nucs);
next_idx = 0;
while(next_idx < num_next && !db_nodes_are_equal(next_nodes[next_idx],next_node))
next_idx++;
TASSERT(next_idx < num_next && db_nodes_are_equal(next_nodes[next_idx],next_node));
// Crawl in all colours
graph_crawler_fetch(&crawler, node, next_nodes, next_idx, num_next,
NULL, graph.num_of_cols, NULL, NULL, NULL);
TASSERT2(crawler.num_paths == 2, "crawler.num_paths: %u", crawler.num_paths);
// Fetch paths
dBNodeBuffer nbuf;
db_node_buf_alloc(&nbuf, 16);
StrBuf sbuf;
strbuf_alloc(&sbuf, 128);
for(p = 0; p < crawler.num_paths; p++) {
db_node_buf_reset(&nbuf);
graph_crawler_get_path_nodes(&crawler, p, &nbuf);
strbuf_ensure_capacity(&sbuf, nbuf.len+graph.kmer_size);
sbuf.end = db_nodes_to_str(nbuf.b, nbuf.len, &graph, sbuf.b);
for(i = 0; i < 3 && strcmp(graphseq[i]+1,sbuf.b) != 0; i++) {}
TASSERT2(i < 3, "seq: %s", sbuf.b);
TASSERT2(sbuf.end == 75, "sbuf.end: %zu", sbuf.end);
TASSERT2(nbuf.len == 65, "nbuf.len: %zu", nbuf.len);
}
strbuf_dealloc(&sbuf);
db_node_buf_dealloc(&nbuf);
graph_crawler_dealloc(&crawler);
db_graph_dealloc(&graph);
}
int ctx_correct(int argc, char **argv)
{
size_t i;
struct ReadThreadCmdArgs args;
read_thread_args_alloc(&args);
read_thread_args_parse(&args, argc, argv, longopts, true);
GraphFileReader *gfile = &args.gfile;
GPathFileBuffer *gpfiles = &args.gpfiles;
CorrectAlnInputBuffer *inputs = &args.inputs;
// Update colours in graph file - sample in 0, all others in 1
size_t ncols = gpath_load_sample_pop(gfile, 1, gpfiles->b, gpfiles->len,
args.colour);
// Check for compatibility between graph files and link files
graphs_gpaths_compatible(gfile, 1, gpfiles->b, gpfiles->len, 1);
int64_t ctx_num_kmers = gfile->num_of_kmers;
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem, path_mem, total_mem;
// 1 bit needed per kmer if we need to keep track of noreseed
bits_per_kmer = sizeof(BinaryKmer)*8 + sizeof(Edges)*8 +
(gpfiles->len > 0 ? sizeof(GPath*)*8 : 0) +
ncols; // in colour
kmers_in_hash = cmd_get_kmers_in_hash(args.memargs.mem_to_use,
args.memargs.mem_to_use_set,
args.memargs.num_kmers,
args.memargs.num_kmers_set,
bits_per_kmer,
ctx_num_kmers, ctx_num_kmers,
false, &graph_mem);
// Paths memory
size_t rem_mem = args.memargs.mem_to_use - MIN2(args.memargs.mem_to_use, graph_mem);
path_mem = gpath_reader_mem_req(gpfiles->b, gpfiles->len, ncols, rem_mem, false,
kmers_in_hash, false);
cmd_print_mem(path_mem, "paths");
// Shift path store memory from graphs->paths
graph_mem -= sizeof(GPath*)*kmers_in_hash;
path_mem += sizeof(GPath*)*kmers_in_hash;
// Total memory
total_mem = graph_mem + path_mem;
cmd_check_mem_limit(args.memargs.mem_to_use, total_mem);
//
// Check we can write all output files
//
// Open output files
SeqOutput *outputs = ctx_calloc(inputs->len, sizeof(SeqOutput));
bool err_occurred = false;
for(i = 0; i < inputs->len && !err_occurred; i++)
{
CorrectAlnInput *input = &inputs->b[i];
// We loaded target colour into colour zero
input->crt_params.ctxcol = input->crt_params.ctpcol = 0;
bool is_pe = asyncio_task_is_pe(&input->files);
err_occurred = !seqout_open(&outputs[i], input->out_base, args.fmt, is_pe);
input->output = &outputs[i];
}
// Abandon if some of the output files already exist
if(err_occurred) {
for(i = 0; i < inputs->len; i++)
seqout_close(&outputs[i], true);
die("Error creating output files");
}
//
// Allocate memory
//
dBGraph db_graph;
db_graph_alloc(&db_graph, gfile->hdr.kmer_size, ncols, 1, kmers_in_hash,
DBG_ALLOC_EDGES | DBG_ALLOC_NODE_IN_COL);
// Create a path store that does not tracks path counts
gpath_reader_alloc_gpstore(gpfiles->b, gpfiles->len, path_mem, false, &db_graph);
//
// Load Graph and link files
//
GraphLoadingPrefs gprefs = graph_loading_prefs(&db_graph);
gprefs.empty_colours = true;
// Load graph, print stats, close file
graph_load(gfile, gprefs, NULL);
hash_table_print_stats_brief(&db_graph.ht);
graph_file_close(gfile);
// Load link files
for(i = 0; i < gpfiles->len; i++) {
gpath_reader_load(&gpfiles->b[i], GPATH_DIE_MISSING_KMERS, &db_graph);
gpath_reader_close(&gpfiles->b[i]);
}
//
// Run alignment
//
correct_reads(inputs->b, inputs->len,
args.dump_seq_sizes, args.dump_frag_sizes,
args.fq_zero, args.append_orig_seq,
args.nthreads, &db_graph);
// Close and free output files
for(i = 0; i < inputs->len; i++)
seqout_close(&outputs[i], false);
ctx_free(outputs);
// Closes input files
read_thread_args_dealloc(&args);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_correct(int argc, char **argv)
{
size_t i, j;
struct ReadThreadCmdArgs args = READ_THREAD_CMD_ARGS_INIT;
read_thread_args_alloc(&args);
read_thread_args_parse(&args, argc, argv, longopts, true);
GraphFileReader *gfile = &args.gfile;
PathFileBuffer *pfiles = &args.pfiles;
CorrectAlnInputBuffer *inputs = &args.inputs;
size_t ctx_total_cols = gfile->hdr.num_of_cols;
size_t ctx_num_kmers = gfile->num_of_kmers;
if(args.colour > ctx_total_cols)
cmd_print_usage("-c %zu is too big [> %zu]", args.colour, ctx_total_cols);
size_t ctp_usedcols = 0;
for(i = 0; i < pfiles->len; i++) {
if(!file_filter_iscolloaded(&pfiles->data[i].fltr, args.colour)) {
cmd_print_usage("Path file doesn't load into colour %zu: %s",
args.colour, pfiles->data[i].fltr.orig_path.buff);
}
ctp_usedcols = MAX2(ctp_usedcols, path_file_usedcols(&pfiles->data[i]));
}
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem, path_mem, total_mem;
// 1 bit needed per kmer if we need to keep track of noreseed
bits_per_kmer = sizeof(Edges)*8 + ctx_num_kmers + sizeof(uint64_t)*8;
kmers_in_hash = cmd_get_kmers_in_hash2(args.memargs.mem_to_use,
args.memargs.mem_to_use_set,
args.memargs.num_kmers,
args.memargs.num_kmers_set,
bits_per_kmer,
ctx_num_kmers, ctx_num_kmers,
false, &graph_mem);
// Paths memory
path_mem = path_files_mem_required(pfiles->data, pfiles->len, false, false,
ctp_usedcols, 0);
cmd_print_mem(path_mem, "paths");
// Total memory
total_mem = graph_mem + path_mem;
cmd_check_mem_limit(args.memargs.mem_to_use, total_mem);
//
// Check we can read all output files
//
// Open output files
SeqOutput *outputs = ctx_calloc(inputs->len, sizeof(SeqOutput));
bool output_files_exist = false;
for(i = 0; i < inputs->len; i++)
{
CorrectAlnInput *input = &inputs->data[i];
input->crt_params.ctxcol = input->crt_params.ctpcol = args.colour;
SeqOutput *output = &outputs[i];
seq_output_alloc(output);
seq_output_set_paths(output, input->out_base,
async_task_pe_output(&input->files));
input->output = output;
// output check prints warnings and returns true if errors
output_files_exist |= seq_output_files_exist_check(output);
}
// Abandon if some of the output files already exist
if(output_files_exist) die("Output files already exist");
// Attempt to open all files
for(i = 0; i < inputs->len && seq_output_open(&outputs[i]); i++) {}
// Check if something went wrong - if so remove all output files
if(i < inputs->len) {
for(j = 0; j < i; j++) seq_output_delete(&outputs[i]);
die("Couldn't open output files");
}
//
// Allocate memory
//
dBGraph db_graph;
db_graph_alloc(&db_graph, gfile->hdr.kmer_size, ctx_total_cols, 1, kmers_in_hash);
size_t bytes_per_col = roundup_bits2bytes(db_graph.ht.capacity);
db_graph.col_edges = ctx_calloc(db_graph.ht.capacity, sizeof(Edges));
db_graph.node_in_cols = ctx_calloc(bytes_per_col * ctx_total_cols, 1);
// Paths
path_store_alloc(&db_graph.pstore, path_mem, false,
db_graph.ht.capacity, ctp_usedcols);
//
// Load Graph and Path files
//
LoadingStats gstats = LOAD_STATS_INIT_MACRO;
GraphLoadingPrefs gprefs = {.db_graph = &db_graph,
.boolean_covgs = false,
.must_exist_in_graph = false,
.must_exist_in_edges = NULL,
.empty_colours = true};
// Load graph, print stats, close file
graph_load(gfile, gprefs, &gstats);
hash_table_print_stats_brief(&db_graph.ht);
graph_file_close(gfile);
// Load path files (does nothing if num_fpiles == 0)
paths_format_merge(pfiles->data, pfiles->len, false, false,
args.num_of_threads, &db_graph);
//
// Run alignment
//
correct_reads(args.num_of_threads, MAX_IO_THREADS,
inputs->data, inputs->len,
&db_graph);
// Close and free output files
for(i = 0; i < inputs->len; i++) seq_output_dealloc(&outputs[i]);
ctx_free(outputs);
read_thread_args_dealloc(&args);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
// Returns 0 on success, otherwise != 0
int ctx_unitigs(int argc, char **argv)
{
size_t nthreads = 0;
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_path = NULL;
UnitigSyntax syntax = PRINT_FASTA;
bool dot_use_points = false;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case 't': cmd_check(!nthreads, cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'F': cmd_check(!syntax, cmd); syntax = PRINT_FASTA; break;
case 'g': cmd_check(!syntax, cmd); syntax = PRINT_GFA; break;
case 'd': cmd_check(!syntax, cmd); syntax = PRINT_DOT; break;
case 'P': cmd_check(!dot_use_points, cmd); dot_use_points = true; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
die("`"CMD" unitigs -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
if(dot_use_points && syntax == PRINT_FASTA)
cmd_print_usage("--point is only for use with --dot");
// Defaults for unset values
if(out_path == NULL) out_path = "-";
if(nthreads == 0) nthreads = DEFAULT_NTHREADS;
if(optind >= argc) cmd_print_usage(NULL);
size_t i, num_gfiles = (size_t)(argc - optind);
char **gfile_paths = argv + optind;
if(dot_use_points && syntax != PRINT_DOT)
cmd_print_usage("--points only valid with --graphviz / --dot");
ctx_assert(num_gfiles > 0);
// Open graph files
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
size_t ctx_max_kmers = 0, ctx_sum_kmers = 0;
graph_files_open(gfile_paths, gfiles, num_gfiles,
&ctx_max_kmers, &ctx_sum_kmers);
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem;
bits_per_kmer = sizeof(BinaryKmer)*8 + sizeof(Edges)*8 + 1;
if(syntax != PRINT_FASTA) bits_per_kmer += sizeof(UnitigEnd) * 8;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
true, &graph_mem);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
status("Output in %s format to %s\n", syntax_strs[syntax],
futil_outpath_str(out_path));
//
// Open output file
//
// Print to stdout unless --out <out> is specified
FILE *fout = futil_fopen_create(out_path, "w");
//
// Allocate memory
//
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, 1, 1, kmers_in_hash,
DBG_ALLOC_EDGES);
UnitigPrinter printer;
unitig_printer_init(&printer, &db_graph, nthreads, syntax, fout);
if(syntax == PRINT_DOT || syntax == PRINT_GFA)
unitig_graph_alloc(&printer.ugraph, &db_graph);
// Load graphs
GraphLoadingPrefs gprefs = {.db_graph = &db_graph,
.boolean_covgs = false,
.must_exist_in_graph = false,
.empty_colours = false};
for(i = 0; i < num_gfiles; i++) {
file_filter_flatten(&gfiles[i].fltr, 0);
graph_load(&gfiles[i], gprefs, NULL);
graph_file_close(&gfiles[i]);
}
ctx_free(gfiles);
hash_table_print_stats(&db_graph.ht);
switch(syntax)
{
case PRINT_FASTA:
status("Printing unitgs in FASTA using %zu threads", nthreads);
supernodes_iterate(nthreads, printer.visited, &db_graph,
print_unitig_fasta, &printer);
break;
case PRINT_GFA:
print_gfa_syntax(&printer);
break;
case PRINT_DOT:
print_dot_syntax(&printer, dot_use_points);
break;
default:
die("Invalid print syntax: %i", syntax);
}
char num_unitigs_str[50];
ulong_to_str(printer.num_unitigs, num_unitigs_str);
status("Dumped %s unitigs\n", num_unitigs_str);
fclose(fout);
unitig_printer_destroy(&printer);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_pop_bubbles(int argc, char **argv)
{
size_t nthreads = 0;
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_path = NULL;
int32_t max_covg = -1; // max mean coverage to remove <=0 => ignore
int32_t max_klen = -1; // max length (kmers) to remove <=0 => ignore
int32_t max_kdiff = -1; // max diff between bubble branch lengths <0 => ignore
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 't': cmd_check(!nthreads, cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'C': cmd_check(max_covg<0, cmd); max_covg = cmd_uint32(cmd, optarg); break;
case 'L': cmd_check(max_klen<0, cmd); max_klen = cmd_uint32(cmd, optarg); break;
case 'D': cmd_check(max_kdiff<0, cmd); max_kdiff = cmd_uint32(cmd, optarg); break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" pop -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
// Defaults for unset values
if(out_path == NULL) out_path = "-";
if(nthreads == 0) nthreads = DEFAULT_NTHREADS;
if(optind >= argc) cmd_print_usage("Require input graph files (.ctx)");
//
// Open graph files
//
const size_t num_gfiles = argc - optind;
char **graph_paths = argv + optind;
ctx_assert(num_gfiles > 0);
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
size_t i, ncols, ctx_max_kmers = 0, ctx_sum_kmers = 0;
ncols = graph_files_open(graph_paths, gfiles, num_gfiles,
&ctx_max_kmers, &ctx_sum_kmers);
bool reread_graph_to_filter = (num_gfiles == 1 &&
strcmp(file_filter_path(&gfiles[0].fltr),"-") != 0);
if(reread_graph_to_filter) {
file_filter_flatten(&gfiles[0].fltr, 0);
ncols = 1;
}
// Check graphs are compatible
graphs_gpaths_compatible(gfiles, num_gfiles, NULL, 0, -1);
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem;
bits_per_kmer = sizeof(BinaryKmer)*8 +
sizeof(Covg)*8*ncols +
sizeof(Edges)*8*ncols +
2; // 1 bit for visited, 1 for removed
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
false, &graph_mem);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
// Check out_path is writable
futil_create_output(out_path);
// Allocate memory
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, ncols, ncols,
kmers_in_hash, DBG_ALLOC_EDGES | DBG_ALLOC_COVGS);
size_t nkwords = roundup_bits2bytes(db_graph.ht.capacity);
uint8_t *visited = ctx_calloc(1, nkwords);
uint8_t *rmvbits = ctx_calloc(1, nkwords);
//
// Load graphs
//
GraphLoadingPrefs gprefs = graph_loading_prefs(&db_graph);
gprefs.empty_colours = true;
for(i = 0; i < num_gfiles; i++) {
graph_load(&gfiles[i], gprefs, NULL);
graph_file_close(&gfiles[i]);
gprefs.empty_colours = false;
}
ctx_free(gfiles);
hash_table_print_stats(&db_graph.ht);
PopBubblesPrefs prefs = {.max_rmv_covg = max_covg,
.max_rmv_klen = max_klen,
.max_rmv_kdiff = max_kdiff};
size_t npopped = 0;
char npopped_str[50];
status("Popping bubbles...");
npopped = pop_bubbles(&db_graph, nthreads, prefs, visited, rmvbits);
ulong_to_str(npopped, npopped_str);
status("Popped %s bubbles", npopped_str);
size_t nkmers0 = db_graph.ht.num_kmers;
status("Removing nodes...");
for(i = 0; i < nkwords; i++) rmvbits[i] = ~rmvbits[i];
prune_nodes_lacking_flag(nthreads, rmvbits, &db_graph);
size_t nkmers1 = db_graph.ht.num_kmers;
ctx_assert(nkmers1 <= nkmers0);
char nkmers0str[50], nkmers1str[50], ndiffstr[50];
ulong_to_str(nkmers0, nkmers0str);
ulong_to_str(nkmers1, nkmers1str);
ulong_to_str(nkmers0-nkmers1, ndiffstr);
status("Number of kmers %s -> %s (-%s)", nkmers0str, nkmers1str, ndiffstr);
if(reread_graph_to_filter)
{
status("Streaming filtered file to: %s\n", out_path);
GraphFileReader gfile;
memset(&gfile, 0, sizeof(GraphFileReader));
graph_file_open(&gfile, graph_paths[0]);
graph_writer_stream_mkhdr(out_path, &gfile, &db_graph,
db_graph.col_edges, NULL);
graph_file_close(&gfile);
}
else
{
status("Saving to: %s\n", out_path);
graph_writer_save_mkhdr(out_path, &db_graph, CTX_GRAPH_FILEFORMAT, NULL,
0, ncols);
}
ctx_free(visited);
ctx_free(rmvbits);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_contigs(int argc, char **argv)
{
size_t nthreads = 0;
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_path = NULL;
size_t i, contig_limit = 0, colour = 0;
bool cmd_reseed = false, cmd_no_reseed = false; // -r, -R
const char *conf_table_path = NULL; // save confidence table to here
bool use_missing_info_check = true, seed_with_unused_paths = false;
double min_step_confid = -1.0, min_cumul_confid = -1.0; // < 0 => no min
// Read length and expected depth for calculating confidences
size_t genome_size = 0;
seq_file_t *tmp_seed_file = NULL;
SeqFilePtrBuffer seed_buf;
seq_file_ptr_buf_alloc(&seed_buf, 16);
GPathReader tmp_gpfile;
GPathFileBuffer gpfiles;
gpfile_buf_alloc(&gpfiles, 8);
// Arg parsing
char cmd[100], shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'o': cmd_check(!out_path,cmd); out_path = optarg; break;
case 't': cmd_check(!nthreads,cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'p':
memset(&tmp_gpfile, 0, sizeof(GPathReader));
gpath_reader_open(&tmp_gpfile, optarg);
gpfile_buf_push(&gpfiles, &tmp_gpfile, 1);
break;
case '1':
case 's': // --seed <in.fa>
if((tmp_seed_file = seq_open(optarg)) == NULL)
die("Cannot read --seed file: %s", optarg);
seq_file_ptr_buf_add(&seed_buf, tmp_seed_file);
break;
case 'r': cmd_check(!cmd_reseed,cmd); cmd_reseed = true; break;
case 'R': cmd_check(!cmd_no_reseed,cmd); cmd_no_reseed = true; break;
case 'N':
cmd_check(!contig_limit,cmd);
contig_limit = cmd_uint32_nonzero(cmd, optarg);
break;
case 'c': cmd_check(!colour,cmd); colour = cmd_uint32(cmd, optarg); break;
case 'G': cmd_check(!genome_size,cmd); genome_size = cmd_bases(cmd, optarg); break;
case 'S': cmd_check(!conf_table_path,cmd); conf_table_path = optarg; break;
case 'M': cmd_check(use_missing_info_check,cmd); use_missing_info_check = false; break;
case 'P': cmd_check(!seed_with_unused_paths,cmd); seed_with_unused_paths = true; break;
case 'C':
cmd_check(min_cumul_confid < 0,cmd);
min_cumul_confid = cmd_udouble(cmd,optarg);
if(min_cumul_confid > 1) die("%s must be 0 <= x <= 1", cmd);
break;
case 'T':
cmd_check(min_step_confid < 0,cmd);
min_step_confid = cmd_udouble(cmd,optarg);
if(min_step_confid > 1) die("%s must be 0 <= x <= 1", cmd);
break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
die("`"CMD" contigs -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
if(cmd_no_reseed && cmd_reseed)
cmd_print_usage("Cannot specify both -r and -R");
if(contig_limit && seed_with_unused_paths)
cmd_print_usage("Cannot combine --ncontigs with --use-seed-paths");
bool sample_with_replacement = cmd_reseed;
// Defaults
if(nthreads == 0) nthreads = DEFAULT_NTHREADS;
if(!seed_buf.len && !contig_limit && sample_with_replacement) {
cmd_print_usage("Please specify one or more of: "
"--no-reseed | --ncontigs | --seed <in.fa>");
}
if(optind >= argc) cmd_print_usage("Require input graph files (.ctx)");
//
// Open graph files
//
const size_t num_gfiles = argc - optind;
char **graph_paths = argv + optind;
ctx_assert(num_gfiles > 0);
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
size_t ncols, ctx_max_kmers = 0, ctx_sum_kmers = 0;
graph_files_open(graph_paths, gfiles, num_gfiles,
&ctx_max_kmers, &ctx_sum_kmers);
// char *ctx_path = argv[optind];
//
// Open Graph file
//
// GraphFileReader gfile;
// memset(&gfile, 0, sizeof(GraphFileReader));
// graph_file_open(&gfile, ctx_path);
// Update colours in graph file - sample in 0, all others in 1
// never need more than two colours
ncols = gpath_load_sample_pop(gfiles, num_gfiles,
gpfiles.b, gpfiles.len, colour);
// Check for compatibility between graph files and path files
// pop_colour is colour 1
graphs_gpaths_compatible(gfiles, num_gfiles, gpfiles.b, gpfiles.len, 1);
if(!genome_size)
{
char nk_str[50];
if(ctx_max_kmers <= 0) die("Please pass --genome <G> if streaming");
genome_size = ctx_max_kmers;
ulong_to_str(genome_size, nk_str);
status("Taking number of kmers as genome size: %s", nk_str);
}
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem, path_mem, total_mem;
// 1 bit needed per kmer if we need to keep track of kmer usage
bits_per_kmer = sizeof(BinaryKmer)*8 + sizeof(Edges)*8 + sizeof(GPath*)*8 +
ncols + !sample_with_replacement;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
false, &graph_mem);
// Paths memory
size_t rem_mem = memargs.mem_to_use - MIN2(memargs.mem_to_use, graph_mem);
path_mem = gpath_reader_mem_req(gpfiles.b, gpfiles.len, ncols, rem_mem, false);
// Shift path store memory from graphs->paths
graph_mem -= sizeof(GPath*)*kmers_in_hash;
path_mem += sizeof(GPath*)*kmers_in_hash;
cmd_print_mem(path_mem, "paths");
// Total memory
total_mem = graph_mem + path_mem;
cmd_check_mem_limit(memargs.mem_to_use, total_mem);
// Load contig hist distribution from ctp files
ZeroSizeBuffer contig_hist;
memset(&contig_hist, 0, sizeof(contig_hist));
for(i = 0; i < gpfiles.len; i++) {
gpath_reader_load_contig_hist(gpfiles.b[i].json,
gpfiles.b[i].fltr.path.b,
file_filter_fromcol(&gpfiles.b[i].fltr, 0),
&contig_hist);
}
// Calculate confidences, only for one colour
ContigConfidenceTable conf_table;
conf_table_alloc(&conf_table, 1);
conf_table_update_hist(&conf_table, 0, genome_size,
contig_hist.b, contig_hist.len);
if(conf_table_path != NULL) {
conf_table_save(&conf_table, conf_table_path);
}
zsize_buf_dealloc(&contig_hist);
//
// Output file if printing
//
FILE *fout = out_path ? futil_fopen_create(out_path, "w") : NULL;
// Allocate
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, ncols, 1, kmers_in_hash,
DBG_ALLOC_EDGES | DBG_ALLOC_NODE_IN_COL);
// Paths
gpath_reader_alloc_gpstore(gpfiles.b, gpfiles.len, path_mem,
false, &db_graph);
uint8_t *visited = NULL;
if(!sample_with_replacement)
visited = ctx_calloc(roundup_bits2bytes(db_graph.ht.capacity), 1);
// Load graph
LoadingStats stats = LOAD_STATS_INIT_MACRO;
GraphLoadingPrefs gprefs = {.db_graph = &db_graph,
.boolean_covgs = false,
.must_exist_in_graph = false,
.empty_colours = true};
for(i = 0; i < num_gfiles; i++) {
graph_load(&gfiles[i], gprefs, &stats);
graph_file_close(&gfiles[i]);
gprefs.empty_colours = false;
}
ctx_free(gfiles);
hash_table_print_stats(&db_graph.ht);
// Load path files
for(i = 0; i < gpfiles.len; i++) {
gpath_reader_load(&gpfiles.b[i], GPATH_DIE_MISSING_KMERS, &db_graph);
gpath_reader_close(&gpfiles.b[i]);
}
gpfile_buf_dealloc(&gpfiles);
AssembleContigStats assem_stats;
assemble_contigs_stats_init(&assem_stats);
assemble_contigs(nthreads, seed_buf.b, seed_buf.len,
contig_limit, visited,
use_missing_info_check, seed_with_unused_paths,
min_step_confid, min_cumul_confid,
fout, out_path, &assem_stats, &conf_table,
&db_graph, 0); // Sample always loaded into colour zero
if(fout && fout != stdout) fclose(fout);
assemble_contigs_stats_print(&assem_stats);
assemble_contigs_stats_destroy(&assem_stats);
conf_table_dealloc(&conf_table);
for(i = 0; i < seed_buf.len; i++)
seq_close(seed_buf.b[i]);
seq_file_ptr_buf_dealloc(&seed_buf);
ctx_free(visited);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_exp_abc(int argc, char **argv)
{
size_t i, nthreads = 0, num_repeats = 0, max_AB_dist = 0;
struct MemArgs memargs = MEM_ARGS_INIT;
bool print_failed_contigs = false;
GPathReader tmp_gpfile;
GPathFileBuffer gpfiles;
gpfile_buf_alloc(&gpfiles, 8);
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 't': cmd_check(!nthreads,cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'p':
memset(&tmp_gpfile, 0, sizeof(GPathReader));
gpath_reader_open(&tmp_gpfile, optarg);
gpfile_buf_push(&gpfiles, &tmp_gpfile, 1);
break;
case 'N': cmd_check(!num_repeats,cmd); num_repeats = cmd_uint32_nonzero(cmd, optarg); break;
case 'M': cmd_check(!max_AB_dist,cmd); max_AB_dist = cmd_uint32_nonzero(cmd, optarg); break;
case 'P': cmd_check(!print_failed_contigs,cmd); print_failed_contigs = true; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" exp_abc -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
// Defaults
if(nthreads == 0) nthreads = DEFAULT_NTHREADS;
if(num_repeats == 0) num_repeats = DEFAULT_NUM_REPEATS;
if(max_AB_dist == 0) max_AB_dist = DEFAULT_MAX_AB_DIST;
if(print_failed_contigs && nthreads != 1) {
warn("--print forces nthreads to be one. soz.");
nthreads = 1;
}
if(optind+1 != argc) cmd_print_usage("Require exactly one input graph file (.ctx)");
const char *ctx_path = argv[optind];
//
// Open Graph file
//
GraphFileReader gfile;
memset(&gfile, 0, sizeof(GraphFileReader));
graph_file_open(&gfile, ctx_path);
size_t ncols = file_filter_into_ncols(&gfile.fltr);
// Check only loading one colour
if(ncols > 1) die("Only implemented for one colour currently");
// Check graph + paths are compatible
graphs_gpaths_compatible(&gfile, 1, gpfiles.b, gpfiles.len, -1);
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem, path_mem, total_mem;
// 1 bit needed per kmer if we need to keep track of kmer usage
bits_per_kmer = sizeof(BinaryKmer)*8 + sizeof(Edges)*8 + sizeof(GPath*)*8 +
ncols;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
gfile.num_of_kmers, gfile.num_of_kmers,
false, &graph_mem);
// Paths memory
size_t rem_mem = memargs.mem_to_use - MIN2(memargs.mem_to_use, graph_mem);
path_mem = gpath_reader_mem_req(gpfiles.b, gpfiles.len, ncols, rem_mem, false,
kmers_in_hash, false);
// Shift path store memory from graphs->paths
graph_mem -= sizeof(GPath*)*kmers_in_hash;
path_mem += sizeof(GPath*)*kmers_in_hash;
cmd_print_mem(path_mem, "paths");
total_mem = graph_mem + path_mem;
cmd_check_mem_limit(memargs.mem_to_use, total_mem);
//
// Allocate memory
//
dBGraph db_graph;
db_graph_alloc(&db_graph, gfile.hdr.kmer_size, 1, 1, kmers_in_hash,
DBG_ALLOC_EDGES | DBG_ALLOC_NODE_IN_COL);
// Paths
gpath_reader_alloc_gpstore(gpfiles.b, gpfiles.len, path_mem, false, &db_graph);
// Load the graph
GraphLoadingPrefs gprefs = graph_loading_prefs(&db_graph);
gprefs.empty_colours = true;
graph_load(&gfile, gprefs, NULL);
graph_file_close(&gfile);
hash_table_print_stats(&db_graph.ht);
// Load link files
for(i = 0; i < gpfiles.len; i++) {
gpath_reader_load(&gpfiles.b[i], GPATH_DIE_MISSING_KMERS, &db_graph);
gpath_reader_close(&gpfiles.b[i]);
}
gpfile_buf_dealloc(&gpfiles);
status("\n");
status("Test 1: Priming region A->B (n: %zu max_AB_dist: %zu)",
num_repeats, max_AB_dist);
run_exp_abc(&db_graph, true, nthreads, num_repeats,
max_AB_dist, print_failed_contigs);
status("\n");
status("Test 2: Trying to traverse A->B (n: %zu max_AB_dist: %zu)",
num_repeats, max_AB_dist);
run_exp_abc(&db_graph, false, nthreads, num_repeats,
max_AB_dist, print_failed_contigs);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_clean(int argc, char **argv)
{
size_t nthreads = 0, use_ncols = 0;
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_ctx_path = NULL;
bool tip_cleaning = false, supernode_cleaning = false;
size_t min_keep_tip = 0;
Covg threshold = 0, fallback_thresh = 0;
const char *len_before_path = NULL, *len_after_path = NULL;
const char *covg_before_path = NULL, *covg_after_path = NULL;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'o':
if(out_ctx_path != NULL) cmd_print_usage(NULL);
out_ctx_path = optarg;
break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'N': use_ncols = cmd_uint32_nonzero(cmd, optarg); break;
case 't': cmd_check(!nthreads, cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'T':
cmd_check(!tip_cleaning, cmd);
min_keep_tip = cmd_uint32_nonzero(cmd, optarg);
tip_cleaning = true;
break;
case 'S':
cmd_check(!supernode_cleaning, cmd);
if(optarg != NULL) threshold = cmd_uint32_nonzero(cmd, optarg);
supernode_cleaning = true;
break;
case 'B': cmd_check(!fallback_thresh, cmd); fallback_thresh = cmd_uint32_nonzero(cmd, optarg); break;
case 'l': cmd_check(!len_before_path, cmd); len_before_path = optarg; break;
case 'L': cmd_check(!len_after_path, cmd); len_after_path = optarg; break;
case 'c': cmd_check(!covg_before_path, cmd); covg_before_path = optarg; break;
case 'C': cmd_check(!covg_after_path, cmd); covg_after_path = optarg; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" clean -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
if(nthreads == 0) nthreads = DEFAULT_NTHREADS;
if(optind >= argc) cmd_print_usage("Please give input graph files");
// Default behaviour
if(!tip_cleaning && !supernode_cleaning) {
if(out_ctx_path != NULL)
supernode_cleaning = tip_cleaning = true; // do both
else
warn("No cleaning being done: you did not specify --out <out.ctx>");
}
bool doing_cleaning = (supernode_cleaning || tip_cleaning);
if(doing_cleaning && out_ctx_path == NULL) {
cmd_print_usage("Please specify --out <out.ctx> for cleaned graph");
}
if(!doing_cleaning && (covg_after_path || len_after_path)) {
cmd_print_usage("You gave --len-after <out> / --covg-after <out> without "
"any cleaning (set -s, --supernodes or -t, --tips)");
}
if(doing_cleaning && strcmp(out_ctx_path,"-") != 0 &&
!futil_get_force() && futil_file_exists(out_ctx_path))
{
cmd_print_usage("Output file already exists: %s", out_ctx_path);
}
if(fallback_thresh && !supernode_cleaning)
cmd_print_usage("-B, --fallback <T> without --supernodes");
// Use remaining args as graph files
char **gfile_paths = argv + optind;
size_t i, j, num_gfiles = (size_t)(argc - optind);
// Open graph files
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
size_t ncols, ctx_max_kmers = 0, ctx_sum_kmers = 0;
ncols = graph_files_open(gfile_paths, gfiles, num_gfiles,
&ctx_max_kmers, &ctx_sum_kmers);
size_t kmer_size = gfiles[0].hdr.kmer_size;
// default to one colour for now
if(use_ncols == 0) use_ncols = 1;
// Flatten if we don't have to remember colours / output a graph
if(!doing_cleaning)
{
ncols = use_ncols = 1;
for(i = 0; i < num_gfiles; i++)
file_filter_flatten(&gfiles[i].fltr, 0);
}
if(ncols < use_ncols) {
warn("I only need %zu colour%s ('--ncols %zu' ignored)",
ncols, util_plural_str(ncols), use_ncols);
use_ncols = ncols;
}
char max_kmers_str[100];
ulong_to_str(ctx_max_kmers, max_kmers_str);
status("%zu input graph%s, max kmers: %s, using %zu colours",
num_gfiles, util_plural_str(num_gfiles), max_kmers_str, use_ncols);
// If no arguments given we default to removing tips < 2*kmer_size
if(tip_cleaning && min_keep_tip == 0)
min_keep_tip = 2 * kmer_size;
// Warn if any graph files already cleaned
size_t fromcol, intocol;
ErrorCleaning *cleaning;
for(i = 0; i < num_gfiles; i++) {
for(j = 0; j < file_filter_num(&gfiles[i].fltr); j++) {
fromcol = file_filter_fromcol(&gfiles[i].fltr, j);
cleaning = &gfiles[i].hdr.ginfo[fromcol].cleaning;
if(cleaning->cleaned_snodes && supernode_cleaning) {
warn("%s:%zu already has supernode cleaning with threshold: <%zu",
file_filter_path(&gfiles[i].fltr), fromcol,
(size_t)cleaning->clean_snodes_thresh);
}
if(cleaning->cleaned_tips && tip_cleaning) {
warn("%s:%zu already has had tip cleaned",
file_filter_path(&gfiles[i].fltr), fromcol);
}
}
}
// Print steps
size_t step = 0;
status("Actions:\n");
if(covg_before_path != NULL)
status("%zu. Saving kmer coverage distribution to: %s", step++, covg_before_path);
if(len_before_path != NULL)
status("%zu. Saving supernode length distribution to: %s", step++, len_before_path);
if(tip_cleaning)
status("%zu. Cleaning tips shorter than %zu nodes", step++, min_keep_tip);
if(supernode_cleaning && threshold > 0)
status("%zu. Cleaning supernodes with coverage < %u", step++, threshold);
if(supernode_cleaning && threshold <= 0)
status("%zu. Cleaning supernodes with auto-detected threshold", step++);
if(covg_after_path != NULL)
status("%zu. Saving kmer coverage distribution to: %s", step++, covg_after_path);
if(len_after_path != NULL)
status("%zu. Saving supernode length distribution to: %s", step++, len_after_path);
//
// Decide memory usage
//
bool all_colours_loaded = (ncols <= use_ncols);
bool use_mem_limit = (memargs.mem_to_use_set && num_gfiles > 1) || !ctx_max_kmers;
size_t kmers_in_hash, bits_per_kmer, graph_mem;
size_t per_kmer_per_col_bits = (sizeof(BinaryKmer)+sizeof(Covg)+sizeof(Edges)) * 8;
size_t pop_edges_per_kmer_bits = (all_colours_loaded ? 0 : sizeof(Edges) * 8);
bits_per_kmer = per_kmer_per_col_bits * use_ncols + pop_edges_per_kmer_bits;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
use_mem_limit, &graph_mem);
// Maximise the number of colours we load to fill the mem
size_t max_usencols = (memargs.mem_to_use*8 - pop_edges_per_kmer_bits * kmers_in_hash) /
(per_kmer_per_col_bits * kmers_in_hash);
use_ncols = MIN2(max_usencols, ncols);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
//
// Check output files are writable
//
futil_create_output(out_ctx_path);
// Does nothing if arg is NULL
futil_create_output(covg_before_path);
futil_create_output(covg_after_path);
futil_create_output(len_before_path);
futil_create_output(len_after_path);
// Create db_graph
// Load as many colours as possible
// Use an extra set of edge to take intersections
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, use_ncols, use_ncols,
kmers_in_hash, DBG_ALLOC_COVGS);
// Edges is a special case
size_t num_edges = db_graph.ht.capacity * (use_ncols + !all_colours_loaded);
db_graph.col_edges = ctx_calloc(num_edges, sizeof(Edges));
// Load graph into a single colour
LoadingStats stats = LOAD_STATS_INIT_MACRO;
GraphLoadingPrefs gprefs = {.db_graph = &db_graph,
.boolean_covgs = false,
.must_exist_in_graph = false,
.must_exist_in_edges = NULL,
.empty_colours = false};
// Construct cleaned graph header
GraphFileHeader outhdr;
memset(&outhdr, 0, sizeof(GraphFileHeader));
outhdr.version = CTX_GRAPH_FILEFORMAT;
outhdr.kmer_size = db_graph.kmer_size;
outhdr.num_of_cols = ncols;
outhdr.num_of_bitfields = (db_graph.kmer_size*2+63)/64;
graph_header_alloc(&outhdr, ncols);
// Merge info into header
size_t gcol = 0;
for(i = 0; i < num_gfiles; i++) {
for(j = 0; j < file_filter_num(&gfiles[i].fltr); j++, gcol++) {
fromcol = file_filter_fromcol(&gfiles[i].fltr, j);
intocol = file_filter_intocol(&gfiles[i].fltr, j);
graph_info_merge(&outhdr.ginfo[intocol], &gfiles[i].hdr.ginfo[fromcol]);
}
}
if(ncols > use_ncols) {
graph_files_load_flat(gfiles, num_gfiles, gprefs, &stats);
} else {
for(i = 0; i < num_gfiles; i++)
graph_load(&gfiles[i], gprefs, &stats);
}
char num_kmers_str[100];
ulong_to_str(db_graph.ht.num_kmers, num_kmers_str);
status("Total kmers loaded: %s\n", num_kmers_str);
size_t initial_nkmers = db_graph.ht.num_kmers;
hash_table_print_stats(&db_graph.ht);
uint8_t *visited = ctx_calloc(roundup_bits2bytes(db_graph.ht.capacity), 1);
uint8_t *keep = ctx_calloc(roundup_bits2bytes(db_graph.ht.capacity), 1);
if((supernode_cleaning && threshold <= 0) || covg_before_path || len_before_path)
{
// Get coverage distribution and estimate cleaning threshold
int est_threshold = cleaning_get_threshold(nthreads,
covg_before_path,
len_before_path,
visited, &db_graph);
if(est_threshold < 0) status("Cannot find recommended cleaning threshold");
else status("Recommended cleaning threshold is: %i", est_threshold);
// Use estimated threshold if threshold not set
if(threshold <= 0) {
if(fallback_thresh > 0 && est_threshold < (int)fallback_thresh) {
status("Using fallback threshold: %i", fallback_thresh);
threshold = fallback_thresh;
}
else if(est_threshold >= 0) threshold = est_threshold;
}
}
// Die if we failed to find suitable cleaning threshold
if(supernode_cleaning && threshold <= 0)
die("Need cleaning threshold (--supernodes=<D> or --fallback <D>)");
if(doing_cleaning) {
// Clean graph of tips (if min_keep_tip > 0) and supernodes (if threshold > 0)
clean_graph(nthreads, threshold, min_keep_tip,
covg_after_path, len_after_path,
visited, keep, &db_graph);
}
ctx_free(visited);
ctx_free(keep);
if(doing_cleaning)
{
// Output graph file
Edges *intersect_edges = NULL;
bool kmers_loaded = true;
size_t col, thresh;
// Set output header ginfo cleaned
for(col = 0; col < ncols; col++)
{
cleaning = &outhdr.ginfo[col].cleaning;
cleaning->cleaned_snodes |= supernode_cleaning;
cleaning->cleaned_tips |= tip_cleaning;
// if(tip_cleaning) {
// strbuf_append_str(&outhdr.ginfo[col].sample_name, ".tipclean");
// }
if(supernode_cleaning) {
thresh = cleaning->clean_snodes_thresh;
thresh = cleaning->cleaned_snodes ? MAX2(thresh, (uint32_t)threshold)
: (uint32_t)threshold;
cleaning->clean_snodes_thresh = thresh;
// char name_append[200];
// sprintf(name_append, ".supclean%zu", thresh);
// strbuf_append_str(&outhdr.ginfo[col].sample_name, name_append);
}
}
if(!all_colours_loaded)
{
// We haven't loaded all the colours
// intersect_edges are edges to mask with
// resets graph edges
intersect_edges = db_graph.col_edges;
db_graph.col_edges += db_graph.ht.capacity;
}
// Print stats on removed kmers
size_t removed_nkmers = initial_nkmers - db_graph.ht.num_kmers;
double removed_pct = (100.0 * removed_nkmers) / initial_nkmers;
char removed_str[100], init_str[100];
ulong_to_str(removed_nkmers, removed_str);
ulong_to_str(initial_nkmers, init_str);
status("Removed %s of %s (%.2f%%) kmers", removed_str, init_str, removed_pct);
graph_files_merge(out_ctx_path, gfiles, num_gfiles,
kmers_loaded, all_colours_loaded,
intersect_edges, &outhdr, &db_graph);
// Swap back
if(!all_colours_loaded)
db_graph.col_edges = intersect_edges;
}
ctx_check(db_graph.ht.num_kmers == hash_table_count_kmers(&db_graph.ht));
graph_header_dealloc(&outhdr);
for(i = 0; i < num_gfiles; i++) graph_file_close(&gfiles[i]);
ctx_free(gfiles);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
// Load each sequence into a separate colour
static void test_bubbles(dBGraph *graph, const char **seqs, size_t nseqs,
const char *flank5p, const char *flank3p,
const char **alleles, size_t nalleles)
{
db_graph_reset(graph);
TASSERT(graph->num_of_cols >= nseqs);
size_t i;
for(i = 0; i < nseqs; i++)
build_graph_from_str_mt(graph, i, seqs[i], strlen(seqs[i]), false);
graph->num_of_cols_used = MAX2(graph->num_of_cols_used, 1);
StrBuf sbuf;
dBNodeBuffer nbuf;
strbuf_alloc(&sbuf, 128);
db_node_buf_alloc(&nbuf, 128);
BubbleCallingPrefs prefs = {.max_allele_len = 100, .max_flank_len = 100,
.haploid_cols = NULL, .nhaploid_cols = 0,
.remove_serial_bubbles = true};
BubbleCaller *caller = bubble_callers_new(1, &prefs, NULL, graph);
_call_bubble(caller, flank5p, flank3p, alleles, nalleles, &nbuf, &sbuf);
strbuf_dealloc(&sbuf);
db_node_buf_dealloc(&nbuf);
bubble_callers_destroy(caller, 1);
}
void test_bubble_caller()
{
test_status("Testing bubble calling...");
// Construct 1 colour graph with kmer-size=11
dBGraph graph;
const size_t kmer_size = 11, ncols = 3;
// Create graph
db_graph_alloc(&graph, kmer_size, ncols, 1, 2000,
DBG_ALLOC_EDGES | DBG_ALLOC_NODE_IN_COL | DBG_ALLOC_BKTLOCKS);
// mutations: x
const char *seqs0[] = {"AGGGATAAAACTCTGTACTGGATCTCCCT",
"AGGGATAAAACTCTcTACTGGATCTCCCT"};
const char flank5p0[] = "AGGGATAAAACTCT";
const char flank3p0[] = "TACTGGATCTCCCT";
const char *alleles0[] = {"ATAAAACTCTGTACTGGATCT", "ATAAAACTCTcTACTGGATCT"};
test_bubbles(&graph, seqs0, 2, flank5p0, flank3p0, alleles0, 2);
// mutations: x y
const char *seqs1[] = {"CCCGTAGGTAAGGGCGTTAGTGCAAGGCCACATTGGGACACGAGTTGATA",
"CCCGTAGGTAAGtGCGTTAGTGCAAGGCCACATTGGGACACGAGTTGATA",
"CCCGTAGGTAAGGGCGTTAGTGCAAGGCCACtTTGGGACACGAGTTGATA"};
// forwards
const char flank5p1a[] = "CCCGTAGGTAAG";
const char flank3p1a[] = "GCGTTAGTGCAAGGCCAC";
const char *alleles1a[] = {"CGTAGGTAAGGGCGTTAGTGC", "CGTAGGTAAGtGCGTTAGTGC"};
const char flank5p1b[] = "GCGTTAGTGCAAGGCCAC";
const char flank3p1b[] = "TTGGGACACGAGTTGATA";
const char *alleles1b[] = {"GCAAGGCCACATTGGGACACG", "GCAAGGCCACtTTGGGACACG"};
test_bubbles(&graph, seqs1, 3, flank5p1a, flank3p1a, alleles1a, 2);
test_bubbles(&graph, seqs1, 3, flank5p1b, flank3p1b, alleles1b, 2);
// reverse
// mutations: y x
// TATCAACTCGTGTCCCAATGTGGCCTTGCACTAACGCCCTTACCTACGGG
// TATCAACTCGTGTCCCAATGTGGCCTTGCACTAACGCaCTTACCTACGGG
// TATCAACTCGTGTCCCAAaGTGGCCTTGCACTAACGCCCTTACCTACGGG
//
const char flank5p1c[] = "GTGGCCTTGCACTAACGC";
const char flank3p1c[] = "CTTACCTACGGG";
const char *alleles1c[] = {"GCACTAACGCCCTTACCTACG", "GCACTAACGCaCTTACCTACG"};
const char flank5p1d[] = "TATCAACTCGTGTCCCAA";
const char flank3p1d[] = "GTGGCCTTGCACTAACGC";
const char *alleles1d[] = {"CGTGTCCCAATGTGGCCTTGC", "CGTGTCCCAAaGTGGCCTTGC"};
test_bubbles(&graph, seqs1, 3, flank5p1c, flank3p1c, alleles1c, 2);
test_bubbles(&graph, seqs1, 3, flank5p1d, flank3p1d, alleles1d, 2);
db_graph_dealloc(&graph);
}
int ctx_bubbles(int argc, char **argv)
{
size_t nthreads = 0;
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_path = NULL;
size_t max_allele_len = 0, max_flank_len = 0;
bool remove_serial_bubbles = true;
// List of haploid colours
size_t *hapcols = NULL;
int nhapcols = 0;
char *hapcols_arg = NULL;
GPathReader tmp_gpfile;
GPathFileBuffer gpfiles;
gpfile_buf_alloc(&gpfiles, 8);
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'p':
memset(&tmp_gpfile, 0, sizeof(GPathReader));
gpath_reader_open(&tmp_gpfile, optarg);
gpfile_buf_push(&gpfiles, &tmp_gpfile, 1);
break;
case 't': cmd_check(!nthreads, cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'H': cmd_check(!hapcols_arg, cmd); hapcols_arg = optarg; break;
case 'A': cmd_check(!max_allele_len, cmd); max_allele_len = cmd_uint32_nonzero(cmd, optarg); break;
case 'F': cmd_check(!max_flank_len, cmd); max_flank_len = cmd_uint32_nonzero(cmd, optarg); break;
case 'S': cmd_check(remove_serial_bubbles,cmd); remove_serial_bubbles = false; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" "SUBCMD" -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
// Defaults for unset values
if(out_path == NULL) out_path = "-";
if(nthreads == 0) nthreads = DEFAULT_NTHREADS;
if(max_allele_len == 0) max_allele_len = DEFAULT_MAX_ALLELE;
if(max_flank_len == 0) max_flank_len = DEFAULT_MAX_FLANK;
if(optind >= argc) cmd_print_usage("Require input graph files (.ctx)");
//
// Open graph files
//
const size_t num_gfiles = argc - optind;
char **graph_paths = argv + optind;
ctx_assert(num_gfiles > 0);
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
size_t i, ncols, ctx_max_kmers = 0, ctx_sum_kmers = 0;
ncols = graph_files_open(graph_paths, gfiles, num_gfiles,
&ctx_max_kmers, &ctx_sum_kmers);
// Check graph + paths are compatible
graphs_gpaths_compatible(gfiles, num_gfiles, gpfiles.b, gpfiles.len, -1);
//
// Check haploid colours are valid
//
if(hapcols_arg != NULL) {
if((nhapcols = range_get_num(hapcols_arg, ncols)) < 0)
die("Invalid haploid colour list: %s", hapcols_arg);
hapcols = ctx_calloc(nhapcols, sizeof(hapcols[0]));
if(range_parse_array(hapcols_arg, hapcols, ncols) < 0)
die("Invalid haploid colour list: %s", hapcols_arg);
}
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem, path_mem, thread_mem;
char thread_mem_str[100];
// edges(1bytes) + kmer_paths(8bytes) + in_colour(1bit/col) +
// visitedfw/rv(2bits/thread)
bits_per_kmer = sizeof(BinaryKmer)*8 + sizeof(Edges)*8 +
(gpfiles.len > 0 ? sizeof(GPath*)*8 : 0) +
ncols + 2*nthreads;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
false, &graph_mem);
// Thread memory
thread_mem = roundup_bits2bytes(kmers_in_hash) * 2;
bytes_to_str(thread_mem * nthreads, 1, thread_mem_str);
status("[memory] (of which threads: %zu x %zu = %s)\n",
nthreads, thread_mem, thread_mem_str);
// Paths memory
size_t rem_mem = memargs.mem_to_use - MIN2(memargs.mem_to_use, graph_mem+thread_mem);
path_mem = gpath_reader_mem_req(gpfiles.b, gpfiles.len, ncols, rem_mem, false,
kmers_in_hash, false);
// Shift path store memory from graphs->paths
graph_mem -= sizeof(GPath*)*kmers_in_hash;
path_mem += sizeof(GPath*)*kmers_in_hash;
cmd_print_mem(path_mem, "paths");
size_t total_mem = graph_mem + thread_mem + path_mem;
cmd_check_mem_limit(memargs.mem_to_use, total_mem);
//
// Open output file
//
gzFile gzout = futil_gzopen_create(out_path, "w");
// Allocate memory
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, ncols, 1, kmers_in_hash,
DBG_ALLOC_EDGES | DBG_ALLOC_NODE_IN_COL);
// Paths
gpath_reader_alloc_gpstore(gpfiles.b, gpfiles.len, path_mem, false, &db_graph);
//
// Load graphs
//
GraphLoadingPrefs gprefs = graph_loading_prefs(&db_graph);
gprefs.empty_colours = true;
for(i = 0; i < num_gfiles; i++) {
graph_load(&gfiles[i], gprefs, NULL);
graph_file_close(&gfiles[i]);
gprefs.empty_colours = false;
}
ctx_free(gfiles);
hash_table_print_stats(&db_graph.ht);
// Load link files
for(i = 0; i < gpfiles.len; i++)
gpath_reader_load(&gpfiles.b[i], GPATH_DIE_MISSING_KMERS, &db_graph);
// Create array of cJSON** from input files
cJSON **hdrs = ctx_malloc(gpfiles.len * sizeof(cJSON*));
for(i = 0; i < gpfiles.len; i++) hdrs[i] = gpfiles.b[i].json;
// Now call variants
BubbleCallingPrefs call_prefs = {.max_allele_len = max_allele_len,
.max_flank_len = max_flank_len,
.haploid_cols = hapcols,
.nhaploid_cols = nhapcols,
.remove_serial_bubbles = remove_serial_bubbles};
invoke_bubble_caller(nthreads, &call_prefs,
gzout, out_path,
hdrs, gpfiles.len,
&db_graph);
status(" saved to: %s\n", out_path);
gzclose(gzout);
ctx_free(hdrs);
// Close input link files
for(i = 0; i < gpfiles.len; i++)
gpath_reader_close(&gpfiles.b[i]);
gpfile_buf_dealloc(&gpfiles);
ctx_free(hapcols);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_clean(int argc, char **argv)
{
size_t nthreads = 0, use_ncols = 0;
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_ctx_path = NULL;
int min_keep_tip = -1, unitig_min = -1; // <0 => default, 0 => noclean
uint32_t fallback_thresh = 0;
const char *len_before_path = NULL, *len_after_path = NULL;
const char *covg_before_path = NULL, *covg_after_path = NULL;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'o':
if(out_ctx_path != NULL) cmd_print_usage(NULL);
out_ctx_path = optarg;
break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'N': use_ncols = cmd_uint32_nonzero(cmd, optarg); break;
case 't': cmd_check(!nthreads, cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'T':
cmd_check(min_keep_tip<0, cmd);
min_keep_tip = (optarg != NULL ? (int)cmd_uint32(cmd, optarg) : -1);
break;
case 'S':
case 'U':
cmd_check(unitig_min<0, cmd);
unitig_min = (optarg != NULL ? cmd_uint32(cmd, optarg) : -1);
break;
case 'B': cmd_check(!fallback_thresh, cmd); fallback_thresh = cmd_uint32_nonzero(cmd, optarg); break;
case 'l': cmd_check(!len_before_path, cmd); len_before_path = optarg; break;
case 'L': cmd_check(!len_after_path, cmd); len_after_path = optarg; break;
case 'c': cmd_check(!covg_before_path, cmd); covg_before_path = optarg; break;
case 'C': cmd_check(!covg_after_path, cmd); covg_after_path = optarg; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" clean -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
if(nthreads == 0) nthreads = DEFAULT_NTHREADS;
if(optind >= argc) cmd_print_usage("Please give input graph files");
bool unitig_cleaning = (unitig_min != 0);
bool tip_cleaning = (min_keep_tip != 0);
bool doing_cleaning = (unitig_cleaning || tip_cleaning);
// If you ever want to estimate cleaning threshold without outputting
// a graph, change this to a warning
if(doing_cleaning && out_ctx_path == NULL) {
cmd_print_usage("Please specify --out <out.ctx> for cleaned graph");
// warn("No cleaning being done: you did not specify --out <out.ctx>");
}
if(!doing_cleaning && (covg_after_path || len_after_path)) {
warn("You gave --len-after <out> / --covg-after <out> without "
"any cleaning (set -U, --unitigs or -t, --tips)");
}
if(doing_cleaning && strcmp(out_ctx_path,"-") != 0 &&
!futil_get_force() && futil_file_exists(out_ctx_path))
{
cmd_print_usage("Output file already exists: %s", out_ctx_path);
}
if(fallback_thresh && !unitig_cleaning)
warn("-B, --fallback <T> without --unitigs");
// Use remaining args as graph files
char **gfile_paths = argv + optind;
size_t i, j, num_gfiles = (size_t)(argc - optind);
// Open graph files
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
size_t col, ncols, ctx_max_kmers = 0, ctx_sum_kmers = 0;
ncols = graph_files_open(gfile_paths, gfiles, num_gfiles,
&ctx_max_kmers, &ctx_sum_kmers);
size_t kmer_size = gfiles[0].hdr.kmer_size;
// default to one colour for now
if(use_ncols == 0) use_ncols = 1;
// Flatten if we don't have to remember colours / output a graph
if(out_ctx_path == NULL)
{
ncols = use_ncols = 1;
for(i = 0; i < num_gfiles; i++)
file_filter_flatten(&gfiles[i].fltr, 0);
}
if(ncols < use_ncols) {
warn("I only need %zu colour%s ('--ncols %zu' ignored)",
ncols, util_plural_str(ncols), use_ncols);
use_ncols = ncols;
}
char max_kmers_str[100];
ulong_to_str(ctx_max_kmers, max_kmers_str);
status("%zu input graph%s, max kmers: %s, using %zu colours",
num_gfiles, util_plural_str(num_gfiles), max_kmers_str, use_ncols);
// If no arguments given we default to removing tips < 2*kmer_size
if(min_keep_tip < 0)
min_keep_tip = 2 * kmer_size;
// Warn if any graph files already cleaned
size_t fromcol;
ErrorCleaning *cleaning;
for(i = 0; i < num_gfiles; i++) {
for(j = 0; j < file_filter_num(&gfiles[i].fltr); j++) {
fromcol = file_filter_fromcol(&gfiles[i].fltr, j);
cleaning = &gfiles[i].hdr.ginfo[fromcol].cleaning;
if(cleaning->cleaned_snodes && unitig_cleaning) {
warn("%s:%zu already has unitig cleaning with threshold: <%zu",
file_filter_path(&gfiles[i].fltr), fromcol,
(size_t)cleaning->clean_snodes_thresh);
}
if(cleaning->cleaned_tips && tip_cleaning) {
warn("%s:%zu already has had tip cleaned",
file_filter_path(&gfiles[i].fltr), fromcol);
}
}
}
// Print steps
size_t step = 0;
status("Actions:\n");
if(covg_before_path != NULL)
status("%zu. Saving kmer coverage distribution to: %s", step++, covg_before_path);
if(len_before_path != NULL)
status("%zu. Saving unitig length distribution to: %s", step++, len_before_path);
if(min_keep_tip > 0)
status("%zu. Cleaning tips shorter than %i nodes", step++, min_keep_tip);
if(unitig_min > 0)
status("%zu. Cleaning unitigs with coverage < %i", step++, unitig_min);
if(unitig_min < 0)
status("%zu. Cleaning unitigs with auto-detected threshold", step++);
if(covg_after_path != NULL)
status("%zu. Saving kmer coverage distribution to: %s", step++, covg_after_path);
if(len_after_path != NULL)
status("%zu. Saving unitig length distribution to: %s", step++, len_after_path);
//
// Decide memory usage
//
bool all_colours_loaded = (ncols <= use_ncols);
bool use_mem_limit = (memargs.mem_to_use_set && num_gfiles > 1) || !ctx_max_kmers;
size_t kmers_in_hash, bits_per_kmer, graph_mem;
size_t per_col_bits = (sizeof(Covg)+sizeof(Edges)) * 8;
size_t extra_edge_bits = (all_colours_loaded ? 0 : sizeof(Edges) * 8);
bits_per_kmer = sizeof(BinaryKmer)*8 +
per_col_bits * use_ncols +
extra_edge_bits;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
use_mem_limit, &graph_mem);
// Maximise the number of colours we load to fill the mem
size_t max_usencols = (memargs.mem_to_use*8 -
sizeof(BinaryKmer)*8*kmers_in_hash +
extra_edge_bits*kmers_in_hash) /
(per_col_bits*kmers_in_hash);
use_ncols = MIN2(max_usencols, ncols);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
//
// Check output files are writable
//
futil_create_output(out_ctx_path);
// Does nothing if arg is NULL
futil_create_output(covg_before_path);
futil_create_output(covg_after_path);
futil_create_output(len_before_path);
futil_create_output(len_after_path);
// Create db_graph
// Load as many colours as possible
// Use an extra set of edge to take intersections
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, use_ncols, use_ncols,
kmers_in_hash, DBG_ALLOC_EDGES | DBG_ALLOC_COVGS);
// Extra edges required to hold union of kept edges
Edges *edges_union = NULL;
if(use_ncols < ncols)
edges_union = ctx_calloc(db_graph.ht.capacity, sizeof(Edges));
// Load graph into a single colour
GraphLoadingPrefs gprefs = graph_loading_prefs(&db_graph);
// Construct cleaned graph header
GraphFileHeader outhdr;
memset(&outhdr, 0, sizeof(GraphFileHeader));
for(i = 0; i < num_gfiles; i++)
graph_file_merge_header(&outhdr, &gfiles[i]);
if(ncols > use_ncols)
{
db_graph.num_of_cols = db_graph.num_edge_cols = 1;
SWAP(edges_union, db_graph.col_edges);
graphs_load_files_flat(gfiles, num_gfiles, gprefs, NULL);
SWAP(edges_union, db_graph.col_edges);
db_graph.num_of_cols = db_graph.num_edge_cols = use_ncols;
}
else {
for(i = 0; i < num_gfiles; i++)
graph_load(&gfiles[i], gprefs, NULL);
}
char num_kmers_str[100];
ulong_to_str(db_graph.ht.num_kmers, num_kmers_str);
status("Total kmers loaded: %s\n", num_kmers_str);
size_t initial_nkmers = db_graph.ht.num_kmers;
hash_table_print_stats(&db_graph.ht);
uint8_t *visited = ctx_calloc(roundup_bits2bytes(db_graph.ht.capacity), 1);
uint8_t *keep = ctx_calloc(roundup_bits2bytes(db_graph.ht.capacity), 1);
// Always estimate cleaning threshold
// if(unitig_min <= 0 || covg_before_path || len_before_path)
// {
// Get coverage distribution and estimate cleaning threshold
int est_min_covg = cleaning_get_threshold(nthreads,
covg_before_path,
len_before_path,
visited, &db_graph);
if(est_min_covg < 0) status("Cannot find recommended cleaning threshold");
else status("Recommended cleaning threshold is: %i", est_min_covg);
// Use estimated threshold if threshold not set
if(unitig_min < 0) {
if(fallback_thresh > 0 && est_min_covg < (int)fallback_thresh) {
status("Using fallback threshold: %i", fallback_thresh);
unitig_min = fallback_thresh;
}
else if(est_min_covg >= 0) unitig_min = est_min_covg;
}
// }
// Die if we failed to find suitable cleaning threshold
if(unitig_min < 0)
die("Need cleaning threshold (--unitigs=<D> or --fallback <D>)");
// Cleaning parameters should now be set (>0) or turned off (==0)
ctx_assert(unitig_min >= 0);
ctx_assert(min_keep_tip >= 0);
if(unitig_min || min_keep_tip)
{
// Clean graph of tips (if min_keep_tip > 0) and unitigs (if threshold > 0)
clean_graph(nthreads, unitig_min, min_keep_tip,
covg_after_path, len_after_path,
visited, keep, &db_graph);
}
ctx_free(visited);
ctx_free(keep);
if(out_ctx_path != NULL)
{
// Set output header ginfo cleaned
for(col = 0; col < ncols; col++)
{
cleaning = &outhdr.ginfo[col].cleaning;
cleaning->cleaned_snodes |= unitig_cleaning;
cleaning->cleaned_tips |= tip_cleaning;
// if(tip_cleaning) {
// strbuf_append_str(&outhdr.ginfo[col].sample_name, ".tipclean");
// }
if(unitig_cleaning) {
size_t thresh = cleaning->clean_snodes_thresh;
thresh = cleaning->cleaned_snodes ? MAX2(thresh, (uint32_t)unitig_min)
: (uint32_t)unitig_min;
cleaning->clean_snodes_thresh = thresh;
// char name_append[200];
// sprintf(name_append, ".supclean%zu", thresh);
// strbuf_append_str(&outhdr.ginfo[col].sample_name, name_append);
}
}
// Print stats on removed kmers
size_t removed_nkmers = initial_nkmers - db_graph.ht.num_kmers;
double removed_pct = (100.0 * removed_nkmers) / initial_nkmers;
char removed_str[100], init_str[100];
ulong_to_str(removed_nkmers, removed_str);
ulong_to_str(initial_nkmers, init_str);
status("Removed %s of %s (%.2f%%) kmers", removed_str, init_str, removed_pct);
// kmers_loaded=true
graph_writer_merge(out_ctx_path, gfiles, num_gfiles,
true, all_colours_loaded,
edges_union, &outhdr, &db_graph);
}
ctx_check(db_graph.ht.num_kmers == hash_table_count_kmers(&db_graph.ht));
// TODO: report kmer coverage for each sample
graph_header_dealloc(&outhdr);
for(i = 0; i < num_gfiles; i++) graph_file_close(&gfiles[i]);
ctx_free(gfiles);
ctx_free(edges_union);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_reads(int argc, char **argv)
{
parse_args(argc, argv);
//
// Open input graphs
//
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
size_t i, ctx_max_kmers = 0, ctx_sum_kmers = 0;
graph_files_open(gfile_paths, gfiles, num_gfiles,
&ctx_max_kmers, &ctx_sum_kmers);
// Will exit and remove output files on error
inputs_attempt_open();
//
// Calculate memory use
//
size_t kmers_in_hash, graph_mem, bits_per_kmer = sizeof(BinaryKmer)*8;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
true, &graph_mem);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
//
// Set up graph
//
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, 1, 0, kmers_in_hash, 0);
// Load graphs
LoadingStats gstats = LOAD_STATS_INIT_MACRO;
GraphLoadingPrefs gprefs = {.db_graph = &db_graph,
.must_exist_in_graph = false,
.empty_colours = true,
.boolean_covgs = false};
for(i = 0; i < num_gfiles; i++) {
file_filter_flatten(&gfiles[i].fltr, 0);
graph_load(&gfiles[i], gprefs, &gstats);
graph_file_close(&gfiles[i]);
gprefs.empty_colours = false;
}
ctx_free(gfiles);
status("Printing reads that do %stouch the graph\n",
inputs.b[0].invert ? "not " : "");
//
// Filter reads using async io
//
LoadingStats seq_stats = LOAD_STATS_INIT_MACRO;
for(i = 0; i < inputs.len; i++) {
inputs.b[i].stats = &seq_stats;
inputs.b[i].db_graph = &db_graph;
}
// Deal with a set of files at once
size_t start, end;
for(start = 0; start < inputs.len; start += MAX_IO_THREADS)
{
// Can have different numbers of inputs vs threads
end = MIN2(inputs.len, start+MAX_IO_THREADS);
asyncio_run_pool(files.b+start, end-start, filter_reads, NULL, nthreads, 0);
}
size_t total_reads_printed = 0;
size_t total_reads = seq_stats.num_se_reads + seq_stats.num_pe_reads;
for(i = 0; i < inputs.len; i++)
total_reads_printed += inputs.b[i].num_of_reads_printed;
for(i = 0; i < inputs.len; i++) {
seqout_close(&inputs.b[i].seqout, false);
asyncio_task_close(&files.b[i]);
}
aln_reads_buf_dealloc(&inputs);
asyncio_buf_dealloc(&files);
status("Total printed %zu / %zu (%.2f%%) reads\n",
total_reads_printed, total_reads,
total_reads ? (100.0 * total_reads_printed) / total_reads : 0.0);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_infer_edges(int argc, char **argv)
{
size_t num_of_threads = DEFAULT_NTHREADS;
struct MemArgs memargs = MEM_ARGS_INIT;
char *out_ctx_path = NULL;
bool add_pop_edges = false, add_all_edges = false;
// Arg parsing
char cmd[100];
char shortopts[100];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'o': cmd_check(!out_ctx_path,cmd); out_ctx_path = optarg; break;
case 't': num_of_threads = cmd_uint32_nonzero(cmd, optarg); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'A': add_all_edges = true; break;
case 'P': add_pop_edges = true; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" inferedges -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
// Default to adding all edges
if(!add_pop_edges && !add_all_edges) add_all_edges = true;
// Can only specify one of --pop --all
if(add_pop_edges && add_all_edges)
cmd_print_usage("Please specify only one of --all --pop");
// Check that optind+1 == argc
if(optind+1 > argc)
cmd_print_usage("Expected exactly one graph file");
else if(optind+1 < argc)
cmd_print_usage("Expected only one graph file. What is this: '%s'", argv[optind]);
//
// Open graph file
//
char *graph_path = argv[optind];
status("Reading graph: %s", graph_path);
if(strchr(graph_path,':') != NULL)
cmd_print_usage("Cannot use ':' in input graph for `"CMD" inferedges`");
GraphFileReader file;
memset(&file, 0, sizeof(file));
file_filter_open(&file.fltr, graph_path);
// Use stat to detect if we are reading from a stream
struct stat st;
bool reading_stream = (stat(file.fltr.path.b, &st) != 0);
// Mode r+ means open (not create) for update (read & write)
graph_file_open2(&file, graph_path, reading_stream ? "r" : "r+", 0);
if(!file_filter_is_direct(&file.fltr))
cmd_print_usage("Inferedges with filter not implemented - sorry");
bool editing_file = !(out_ctx_path || reading_stream);
FILE *fout = NULL;
// Editing input file or writing a new file
if(!editing_file)
fout = futil_fopen_create(out_ctx_path ? out_ctx_path : "-", "w");
// Print output status
if(fout == stdout) status("Writing to STDOUT");
else if(fout != NULL) status("Writing to: %s", out_ctx_path);
else status("Editing file in place: %s", graph_path);
status("Inferring all missing %sedges", add_pop_edges ? "population " : "");
//
// Decide on memory
//
const size_t ncols = file.hdr.num_of_cols;
size_t kmers_in_hash, graph_mem, bits_per_kmer;
// reading stream: all covgs + edges
// reading file: one bit per kmer per colour for 'in colour'
bits_per_kmer = sizeof(BinaryKmer)*8;
if(reading_stream) {
bits_per_kmer += ncols * 8 * (sizeof(Edges) + sizeof(Covg));
} else {
bits_per_kmer += ncols; // in colour
}
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
file.num_of_kmers, file.num_of_kmers,
memargs.mem_to_use_set, &graph_mem);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
//
// Allocate memory
//
int alloc_flags = reading_stream ? DBG_ALLOC_EDGES | DBG_ALLOC_COVGS
: DBG_ALLOC_NODE_IN_COL;
dBGraph db_graph;
db_graph_alloc(&db_graph, file.hdr.kmer_size,
ncols, reading_stream ? ncols : 1,
kmers_in_hash, alloc_flags);
LoadingStats stats = LOAD_STATS_INIT_MACRO;
GraphLoadingPrefs gprefs = {.db_graph = &db_graph,
.boolean_covgs = false,
.must_exist_in_graph = false,
.must_exist_in_edges = NULL,
.empty_colours = false};
// We need to load the graph for both --pop and --all since we need to check
// if the next kmer is in each of the colours
graph_load(&file, gprefs, &stats);
if(add_pop_edges) status("Inferring edges from population...\n");
else status("Inferring all missing edges...\n");
size_t num_kmers_edited;
if(reading_stream)
{
ctx_assert(fout != NULL);
num_kmers_edited = infer_edges(num_of_threads, add_all_edges, &db_graph);
graph_write_header(fout, &file.hdr);
graph_write_all_kmers(fout, &db_graph);
}
else if(fout == NULL) {
num_kmers_edited = inferedges_on_mmap(&db_graph, add_all_edges, &file);
} else {
num_kmers_edited = inferedges_on_file(&db_graph, add_all_edges, &file, fout);
}
if(fout != NULL && fout != stdout) fclose(fout);
char modified_str[100], kmers_str[100];
ulong_to_str(num_kmers_edited, modified_str);
ulong_to_str(db_graph.ht.num_kmers, kmers_str);
double modified_rate = 0;
if(db_graph.ht.num_kmers)
modified_rate = (100.0 * num_kmers_edited) / db_graph.ht.num_kmers;
status("%s of %s (%.2f%%) nodes modified\n",
modified_str, kmers_str, modified_rate);
if(editing_file)
{
// Close and re-open
fclose(file.fh);
file.fh = NULL;
futil_update_timestamp(file.fltr.path.b);
}
graph_file_close(&file);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_thread(int argc, char **argv)
{
struct ReadThreadCmdArgs args;
read_thread_args_alloc(&args);
read_thread_args_parse(&args, argc, argv, longopts, false);
GraphFileReader *gfile = &args.gfile;
GPathFileBuffer *gpfiles = &args.gpfiles;
CorrectAlnInputBuffer *inputs = &args.inputs;
size_t i;
if(args.zero_link_counts && gpfiles->len == 0)
cmd_print_usage("-0,--zero-paths without -p,--paths <in.ctp> has no meaning");
// Check each path file only loads one colour
gpaths_only_for_colour(gpfiles->b, gpfiles->len, 0);
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem, total_mem;
size_t path_hash_mem, path_store_mem, path_mem;
bool sep_path_list = (!args.use_new_paths && gpfiles->len > 0);
bits_per_kmer = sizeof(BinaryKmer)*8 + sizeof(Edges)*8 + sizeof(GPath*)*8 +
2 * args.nthreads; // Have traversed
// false -> don't use mem_to_use to decide how many kmers to store in hash
// since we need some of that memory for storing paths
kmers_in_hash = cmd_get_kmers_in_hash(args.memargs.mem_to_use,
args.memargs.mem_to_use_set,
args.memargs.num_kmers,
args.memargs.num_kmers_set,
bits_per_kmer,
gfile->num_of_kmers,
gfile->num_of_kmers,
false, &graph_mem);
// Paths memory
size_t min_path_mem = 0;
gpath_reader_sum_mem(gpfiles->b, gpfiles->len, 1, true, true, &min_path_mem);
if(graph_mem + min_path_mem > args.memargs.mem_to_use) {
char buf[50];
die("Require at least %s memory", bytes_to_str(graph_mem+min_path_mem, 1, buf));
}
path_mem = args.memargs.mem_to_use - graph_mem;
size_t pentry_hash_mem = sizeof(GPEntry)/0.7;
size_t pentry_store_mem = sizeof(GPath) + 8 + // struct + sequence
1 + // in colour
sizeof(uint8_t) + // counts
sizeof(uint32_t); // kmer length
size_t max_paths = path_mem / (pentry_store_mem + pentry_hash_mem);
path_store_mem = max_paths * pentry_store_mem;
path_hash_mem = max_paths * pentry_hash_mem;
cmd_print_mem(path_hash_mem, "paths hash");
cmd_print_mem(path_store_mem, "paths store");
total_mem = graph_mem + path_mem;
cmd_check_mem_limit(args.memargs.mem_to_use, total_mem);
//
// Open output file
//
gzFile gzout = futil_gzopen_create(args.out_ctp_path, "w");
status("Creating paths file: %s", futil_outpath_str(args.out_ctp_path));
//
// Allocate memory
//
dBGraph db_graph;
size_t kmer_size = gfile->hdr.kmer_size;
db_graph_alloc(&db_graph, kmer_size, 1, 1, kmers_in_hash,
DBG_ALLOC_EDGES | DBG_ALLOC_NODE_IN_COL);
// Split path memory 2:1 between store and hash
// Create a path store that tracks path counts
gpath_store_alloc(&db_graph.gpstore,
db_graph.num_of_cols, db_graph.ht.capacity,
0, path_store_mem, true, sep_path_list);
// Create path hash table for fast lookup
gpath_hash_alloc(&db_graph.gphash, &db_graph.gpstore, path_hash_mem);
if(args.use_new_paths) {
status("Using paths as they are added (risky)");
} else {
status("Not using new paths as they are added (safe)");
}
//
// Start up workers to add paths to the graph
//
GenPathWorker *workers;
workers = gen_paths_workers_alloc(args.nthreads, &db_graph);
// Setup for loading graphs graph
LoadingStats gstats;
loading_stats_init(&gstats);
// Path statistics
LoadingStats *load_stats = gen_paths_get_stats(workers);
CorrectAlnStats *aln_stats = gen_paths_get_aln_stats(workers);
// Load contig hist distribution
for(i = 0; i < gpfiles->len; i++) {
gpath_reader_load_contig_hist(gpfiles->b[i].json,
gpfiles->b[i].fltr.path.b,
file_filter_fromcol(&gpfiles->b[i].fltr, 0),
&aln_stats->contig_histgrm);
}
GraphLoadingPrefs gprefs = {.db_graph = &db_graph,
.boolean_covgs = false,
.must_exist_in_graph = false,
.must_exist_in_edges = NULL,
.empty_colours = false}; // already loaded paths
// Load graph, print stats, close file
graph_load(gfile, gprefs, &gstats);
hash_table_print_stats_brief(&db_graph.ht);
graph_file_close(gfile);
// Load existing paths
for(i = 0; i < gpfiles->len; i++)
gpath_reader_load(&gpfiles->b[i], GPATH_DIE_MISSING_KMERS, &db_graph);
// zero link counts of already loaded links
if(args.zero_link_counts) {
status("Zeroing link counts for loaded links");
gpath_set_zero_nseen(&db_graph.gpstore.gpset);
}
if(!args.use_new_paths)
gpath_store_split_read_write(&db_graph.gpstore);
// Deal with a set of files at once
// Can have different numbers of inputs vs threads
size_t start, end;
for(start = 0; start < inputs->len; start += MAX_IO_THREADS)
{
end = MIN2(inputs->len, start+MAX_IO_THREADS);
generate_paths(inputs->b+start, end-start, workers, args.nthreads);
}
// Print memory statistics
gpath_hash_print_stats(&db_graph.gphash);
gpath_store_print_stats(&db_graph.gpstore);
correct_aln_dump_stats(aln_stats, load_stats,
args.dump_seq_sizes,
args.dump_frag_sizes,
db_graph.ht.num_kmers);
// Don't need GPathHash anymore
gpath_hash_dealloc(&db_graph.gphash);
cJSON **hdrs = ctx_malloc(gpfiles->len * sizeof(cJSON*));
for(i = 0; i < gpfiles->len; i++) hdrs[i] = gpfiles->b[i].json;
size_t output_threads = MIN2(args.nthreads, MAX_IO_THREADS);
// Generate a cJSON header for all inputs
cJSON *thread_hdr = cJSON_CreateObject();
cJSON *inputs_hdr = cJSON_CreateArray();
cJSON_AddItemToObject(thread_hdr, "inputs", inputs_hdr);
for(i = 0; i < inputs->len; i++)
cJSON_AddItemToArray(inputs_hdr, correct_aln_input_json_hdr(&inputs->b[i]));
// Write output file
gpath_save(gzout, args.out_ctp_path, output_threads, true,
"thread", thread_hdr, hdrs, gpfiles->len,
&aln_stats->contig_histgrm, 1,
&db_graph);
gzclose(gzout);
ctx_free(hdrs);
// Optionally run path checks for debugging
// gpath_checks_all_paths(&db_graph, args.nthreads);
// ins_gap, err_gap no longer allocated after this line
gen_paths_workers_dealloc(workers, args.nthreads);
// Close and free input files etc.
read_thread_args_dealloc(&args);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
void all_tests_add_paths_multi(dBGraph *graph, const char **seqs, size_t nseqs,
CorrectAlnParam params,
int exp_npaths, int exp_nkmers)
{
size_t npaths = graph->gpstore.num_paths;
size_t nkmers = graph->gpstore.num_kmers_with_paths;
size_t i, nworkers = 1;
GenPathWorker *wrkrs = gen_paths_workers_alloc(nworkers, graph);
// Set up asyncio input data
AsyncIOInput io = {.file1 = NULL, .file2 = NULL,
.fq_offset = 0, .interleaved = false};
CorrectAlnInput task = {.files = io, .fq_cutoff = 0, .hp_cutoff = 0,
.matedir = READPAIR_FR, .crt_params = params,
.out_base = NULL, .output = NULL};
AsyncIOData iodata;
asynciodata_alloc(&iodata);
seq_read_reset(&iodata.r2);
iodata.fq_offset1 = iodata.fq_offset2 = 0;
iodata.ptr = NULL;
// Add paths
for(i = 0; i < nseqs; i++) {
seq_read_set(&iodata.r1, seqs[i]);
gen_paths_worker_seq(wrkrs, &iodata, &task);
}
asynciodata_dealloc(&iodata);
gen_paths_workers_dealloc(wrkrs, nworkers);
// Check we added the right number of paths
if(exp_npaths >= 0) {
TASSERT2(graph->gpstore.num_paths == npaths + (size_t)exp_npaths, "%zu %zu %zu",
(size_t)graph->gpstore.num_paths, (size_t)npaths, (size_t)exp_npaths);
}
if(exp_nkmers >= 0) {
TASSERT(graph->gpstore.num_kmers_with_paths == nkmers + (size_t)exp_nkmers);
}
}
void all_tests_add_paths(dBGraph *graph, const char *seq,
CorrectAlnParam params,
int exp_npaths, int exp_nkmers)
{
all_tests_add_paths_multi(graph, &seq, 1, params, exp_npaths, exp_nkmers);
}
void all_tests_construct_graph(dBGraph *graph,
size_t kmer_size, size_t ncols,
const char **seqs, size_t nseqs,
CorrectAlnParam path_params)
{
size_t i;
db_graph_alloc(graph, kmer_size, ncols, ncols, 1024,
DBG_ALLOC_EDGES | DBG_ALLOC_COVGS |
DBG_ALLOC_NODE_IN_COL | DBG_ALLOC_BKTLOCKS);
// Path data
gpath_store_alloc(&graph->gpstore, ncols, graph->ht.capacity,
0, ONE_MEGABYTE, true, false);
// Don't use links to add new links
gpath_store_split_read_write(&graph->gpstore);
// Allocate path hash table just in case
gpath_hash_alloc(&graph->gphash, &graph->gpstore, ONE_MEGABYTE);
// Build graph
for(i = 0; i < nseqs; i++)
build_graph_from_str_mt(graph, 0, seqs[i], strlen(seqs[i]), false);
gpath_store_merge_read_write(&graph->gpstore);
graph->num_of_cols_used = MAX2(graph->num_of_cols_used, 1);
all_tests_add_paths_multi(graph, seqs, nseqs, path_params, -1, -1);
}
static void test_kmer_occur_filter()
{
// Construct 1 colour graph with kmer-size=11
dBGraph graph;
const size_t kmer_size = 11, ncols = 3;
size_t i;
// Create graph
db_graph_alloc(&graph, kmer_size, ncols, 1, 2000,
DBG_ALLOC_EDGES | DBG_ALLOC_NODE_IN_COL | DBG_ALLOC_BKTLOCKS);
// xyz------->>> y > < X
// TTCGACCCGACAGGGCAACGTAGTCCGACAGGGCACAGCCCTGTCGGGGGGTGCA
#define NUM_NODES 3
#define NUM_READS 3
const char *tmp[NUM_READS]
= {
"AACA",
"TTCGACCCGACAGGGCAACGTAGTCCGACAGGGCACAGCCCTGTCGGGGGGTGCA",
"TCTAGCATGTGTGTT"};
read_t reads[NUM_READS];
for(i = 0; i < NUM_READS; i++) {
seq_read_alloc(&reads[i]);
seq_read_set(&reads[i], tmp[i]);
}
KOGraph kograph = kograph_create(reads, NUM_READS, true, 0, 1, &graph);
TASSERT(kograph.nchroms == NUM_READS);
TASSERT(kograph.koccurs != NULL);
KOccurRunBuffer koruns, koruns_tmp, koruns_ended;
korun_buf_alloc(&koruns, 16);
korun_buf_alloc(&koruns_tmp, 16);
korun_buf_alloc(&koruns_ended, 16);
// Check CCCGACAGGGCAA starts at CCCGACAGGGC
// x=CCCGACAGGGC, y=CCGACAGGGCA, z=CGACAGGGCAA
// X=GCCCTGTCGGG, Y=TGCCCTGTCGG, Z=TTGCCCTGTCG
dBNode nodes[NUM_NODES];
for(i = 0; i < NUM_NODES; i++)
nodes[i] = db_graph_find_str(&graph, &"CCCGACAGGGCAA"[i]);
korun_buf_reset(&koruns);
korun_buf_reset(&koruns_ended);
kograph_filter_extend(&kograph, nodes, NUM_NODES, true, 0, 0,
&koruns, &koruns_tmp, &koruns_ended);
// Checks
TASSERT2(koruns.len == 1, "koruns.len: %zu", koruns.len);
TASSERT(koruns.b[0].strand == STRAND_PLUS); // left-to-right with ref
TASSERT2(koruns.b[0].chrom == 1, "chrom: %zu", (size_t)koruns.b[0].chrom);
TASSERT2(koruns.b[0].first == 5, "offset: %zu", (size_t)koruns.b[0].first);
TASSERT2(koruns.b[0].last == 7, "last: %zu", (size_t)koruns.b[0].last);
// Test reverse
db_nodes_reverse_complement(nodes, NUM_NODES);
korun_buf_reset(&koruns);
korun_buf_reset(&koruns_ended);
kograph_filter_extend(&kograph, nodes, 1, true, 0, 0, &koruns, &koruns_tmp, &koruns_ended);
kograph_filter_extend(&kograph, nodes+1, 1, true, 0, 1, &koruns, &koruns_tmp, &koruns_ended);
kograph_filter_extend(&kograph, nodes+2, 1, true, 0, 2, &koruns, &koruns_tmp, &koruns_ended);
// Print out for debugging
// printf("koruns: ");
// koruns_print(koruns.b, koruns.len, kmer_size, stdout);
// printf("\nkoruns_ended: ");
// koruns_print(koruns_ended.b, koruns_ended.len, kmer_size, stdout);
// printf("\n");
// Check results match:
// koruns: chromid:1:17-5:-, chromid:1:37-47:+
// koruns_ended: chromid:1:34-24:-
TASSERT2(koruns.len == 2, "koruns.len: %zu", koruns.len);
TASSERT2(koruns_ended.len == 1, "koruns_ended.len: %zu", koruns_ended.len);
TASSERT(koruns.b[0].strand == STRAND_MINUS); // reverse complement of ref
TASSERT2(koruns.b[0].chrom == 1, "chrom: %zu", (size_t)koruns.b[0].chrom);
TASSERT2(koruns.b[0].first == 7, "offset: %zu", (size_t)koruns.b[0].first);
TASSERT2(koruns.b[0].last == 5, "last: %zu", (size_t)koruns.b[0].last);
korun_buf_dealloc(&koruns);
korun_buf_dealloc(&koruns_tmp);
korun_buf_dealloc(&koruns_ended);
for(i = 0; i < NUM_READS; i++) seq_read_dealloc(&reads[i]);
kograph_dealloc(&kograph);
db_graph_dealloc(&graph);
}
int ctx_join(int argc, char **argv)
{
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_path = NULL;
size_t use_ncols = 0;
GraphFileReader tmp_gfile;
GraphFileBuffer isec_gfiles_buf;
gfile_buf_alloc(&isec_gfiles_buf, 8);
// Arg parsing
char cmd[100], shortopts[100];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'N': cmd_check(!use_ncols, cmd); use_ncols = cmd_uint32_nonzero(cmd, optarg); break;
case 'i':
graph_file_reset(&tmp_gfile);
graph_file_open(&tmp_gfile, optarg);
if(file_filter_into_ncols(&tmp_gfile.fltr) > 1)
warn("Flattening intersection graph into colour 0: %s", optarg);
file_filter_flatten(&tmp_gfile.fltr, 0);
gfile_buf_push(&isec_gfiles_buf, &tmp_gfile, 1);
break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" join -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
GraphFileReader *igfiles = isec_gfiles_buf.b;
size_t num_igfiles = isec_gfiles_buf.len;
if(!out_path) cmd_print_usage("--out <out.ctx> required");
if(optind >= argc)
cmd_print_usage("Please specify at least one input graph file");
// optind .. argend-1 are graphs to load
size_t num_gfiles = (size_t)(argc - optind);
char **gfile_paths = argv + optind;
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
status("Probing %zu graph files and %zu intersect files", num_gfiles, num_igfiles);
// Check all binaries are valid binaries with matching kmer size
size_t i;
size_t ctx_max_cols = 0;
uint64_t min_intersect_num_kmers = 0, ctx_max_kmers = 0, ctx_sum_kmers = 0;
for(i = 0; i < num_gfiles; i++)
{
graph_file_open2(&gfiles[i], gfile_paths[i], "r", true, ctx_max_cols);
if(gfiles[0].hdr.kmer_size != gfiles[i].hdr.kmer_size) {
cmd_print_usage("Kmer sizes don't match [%u vs %u]",
gfiles[0].hdr.kmer_size, gfiles[i].hdr.kmer_size);
}
ctx_max_cols = MAX2(ctx_max_cols, file_filter_into_ncols(&gfiles[i].fltr));
ctx_max_kmers = MAX2(ctx_max_kmers, graph_file_nkmers(&gfiles[i]));
ctx_sum_kmers += graph_file_nkmers(&gfiles[i]);
}
// Probe intersection graph files
for(i = 0; i < num_igfiles; i++)
{
if(gfiles[0].hdr.kmer_size != igfiles[i].hdr.kmer_size) {
cmd_print_usage("Kmer sizes don't match [%u vs %u]",
gfiles[0].hdr.kmer_size, igfiles[i].hdr.kmer_size);
}
uint64_t nkmers = graph_file_nkmers(&igfiles[i]);
if(i == 0) min_intersect_num_kmers = nkmers;
else if(nkmers < min_intersect_num_kmers)
{
// Put smallest intersection binary first
SWAP(igfiles[i], igfiles[0]);
min_intersect_num_kmers = nkmers;
}
}
bool take_intersect = (num_igfiles > 0);
// If we are taking an intersection,
// all kmers intersection kmers will need to be loaded
if(take_intersect)
ctx_max_kmers = ctx_sum_kmers = min_intersect_num_kmers;
bool use_ncols_set = (use_ncols > 0);
bool output_to_stdout = (strcmp(out_path,"-") == 0);
// if(use_ncols == 0) use_ncols = 1;
if(use_ncols_set) {
if(use_ncols < ctx_max_cols && output_to_stdout)
die("I need %zu colours if outputting to STDOUT (--ncols)", ctx_max_cols);
if(use_ncols > ctx_max_cols) {
warn("I only need %zu colour%s ('--ncols %zu' ignored)",
ctx_max_cols, util_plural_str(ctx_max_cols), use_ncols);
use_ncols = ctx_max_cols;
}
}
else {
use_ncols = output_to_stdout ? ctx_max_cols : 1;
}
// Check out_path is writable
futil_create_output(out_path);
status("Output %zu cols; from %zu files; intersecting %zu graphs; ",
ctx_max_cols, num_gfiles, num_igfiles);
if(num_gfiles == 1 && num_igfiles == 0)
{
// Loading only one file with no intersection files
// Don't need to store a graph in memory, can filter as stream
// Don't actually store anything in the de Bruijn graph, but we need to
// pass it, so mock one up
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size,
file_filter_into_ncols(&gfiles[0].fltr), 0, 1024, 0);
graph_writer_stream_mkhdr(out_path, &gfiles[0], &db_graph, NULL, NULL);
graph_file_close(&gfiles[0]);
gfile_buf_dealloc(&isec_gfiles_buf);
ctx_free(gfiles);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem;
bits_per_kmer = sizeof(BinaryKmer)*8 +
(sizeof(Covg) + sizeof(Edges)) * 8 * use_ncols;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
true, &graph_mem);
if(!use_ncols_set)
{
// Maximise use_ncols
size_t max_usencols = (memargs.mem_to_use*8) / bits_per_kmer;
use_ncols = MIN2(max_usencols, ctx_max_cols);
bits_per_kmer = sizeof(BinaryKmer)*8 +
(sizeof(Covg) + sizeof(Edges)) * 8 * use_ncols;
// Re-check memory used
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
ctx_max_kmers, ctx_sum_kmers,
true, &graph_mem);
}
status("Using %zu colour%s in memory", use_ncols, util_plural_str(use_ncols));
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
// Create db_graph
dBGraph db_graph;
Edges *intersect_edges = NULL;
size_t edge_cols = (use_ncols + take_intersect);
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, use_ncols, use_ncols,
kmers_in_hash, DBG_ALLOC_COVGS);
// We allocate edges ourself since it's a special case
db_graph.col_edges = ctx_calloc(db_graph.ht.capacity*edge_cols, sizeof(Edges));
// Load intersection binaries
char *intsct_gname_ptr = NULL;
StrBuf intersect_gname;
strbuf_alloc(&intersect_gname, 1024);
if(take_intersect)
{
GraphLoadingPrefs gprefs = graph_loading_prefs(&db_graph);
gprefs.boolean_covgs = true; // covg++ only
for(i = 0; i < num_igfiles; i++)
{
graph_load(&igfiles[i], gprefs, NULL);
// Update intersect header
// note: intersection graphs all load exactly one colour into colour 0
graph_info_make_intersect(&igfiles[i].hdr.ginfo[0], &intersect_gname);
gprefs.must_exist_in_graph = true;
gprefs.must_exist_in_edges = db_graph.col_edges;
}
if(num_igfiles > 1)
{
// Remove nodes where covg != num_igfiles
HASH_ITERATE_SAFE(&db_graph.ht, remove_non_intersect_nodes,
db_graph.col_covgs, (Covg)num_igfiles, &db_graph.ht);
}
status("Loaded intersection set\n");
intsct_gname_ptr = intersect_gname.b;
for(i = 0; i < num_igfiles; i++) graph_file_close(&igfiles[i]);
// Reset graph info
for(i = 0; i < db_graph.num_of_cols; i++)
graph_info_init(&db_graph.ginfo[i]);
// Zero covgs
memset(db_graph.col_covgs, 0, db_graph.ht.capacity * sizeof(Covg));
// Use union edges we loaded to intersect new edges
intersect_edges = db_graph.col_edges;
db_graph.col_edges += db_graph.ht.capacity;
}
bool kmers_loaded = take_intersect, colours_loaded = false;
graph_writer_merge_mkhdr(out_path, gfiles, num_gfiles,
kmers_loaded, colours_loaded, intersect_edges,
intsct_gname_ptr, &db_graph);
if(take_intersect)
db_graph.col_edges -= db_graph.ht.capacity;
for(i = 0; i < num_gfiles; i++) graph_file_close(&gfiles[i]);
strbuf_dealloc(&intersect_gname);
gfile_buf_dealloc(&isec_gfiles_buf);
ctx_free(gfiles);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_vcfcov(int argc, char **argv)
{
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_path = NULL, *out_type = NULL;
uint32_t max_allele_len = 0, max_gt_vars = 0;
char *ref_path = NULL;
bool low_mem = false;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
size_t i;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case 'O': cmd_check(!out_type, cmd); out_type = optarg; break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'r': cmd_check(!ref_path, cmd); ref_path = optarg; break;
case 'L': cmd_check(!max_allele_len,cmd); max_allele_len = cmd_uint32(cmd,optarg); break;
case 'N': cmd_check(!max_gt_vars,cmd); max_gt_vars = cmd_uint32(cmd,optarg); break;
case 'M': cmd_check(!low_mem, cmd); low_mem = true; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" "SUBCMD" -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
// Defaults for unset values
if(out_path == NULL) out_path = "-";
if(ref_path == NULL) cmd_print_usage("Require a reference (-r,--ref <ref.fa>)");
if(optind+2 > argc) cmd_print_usage("Require VCF and graph files");
if(!max_allele_len) max_allele_len = DEFAULT_MAX_ALLELE_LEN;
if(!max_gt_vars) max_gt_vars = DEFAULT_MAX_GT_VARS;
status("[vcfcov] max allele length: %u; max number of variants: %u",
max_allele_len, max_gt_vars);
// open ref
// index fasta with: samtools faidx ref.fa
faidx_t *fai = fai_load(ref_path);
if(fai == NULL) die("Cannot load ref index: %s / %s.fai", ref_path, ref_path);
// Open input VCF file
const char *vcf_path = argv[optind++];
htsFile *vcffh = hts_open(vcf_path, "r");
if(vcffh == NULL) die("Cannot open VCF file: %s", vcf_path);
bcf_hdr_t *vcfhdr = bcf_hdr_read(vcffh);
if(vcfhdr == NULL) die("Cannot read VCF header: %s", vcf_path);
// Test we can close and reopen files
if(low_mem) {
if((vcffh = hts_open(vcf_path, "r")) == NULL)
die("Cannot re-open VCF file: %s", vcf_path);
if((vcfhdr = bcf_hdr_read(vcffh)) == NULL)
die("Cannot re-read VCF header: %s", vcf_path);
}
//
// Open graph files
//
const size_t num_gfiles = argc - optind;
char **graph_paths = argv + optind;
ctx_assert(num_gfiles > 0);
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
size_t ncols, ctx_max_kmers = 0, ctx_sum_kmers = 0;
ncols = graph_files_open(graph_paths, gfiles, num_gfiles,
&ctx_max_kmers, &ctx_sum_kmers);
// Check graph + paths are compatible
graphs_gpaths_compatible(gfiles, num_gfiles, NULL, 0, -1);
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem;
bits_per_kmer = sizeof(BinaryKmer)*8 + sizeof(Covg)*8 * ncols;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
low_mem ? -1 : (int64_t)ctx_max_kmers,
ctx_sum_kmers,
true, &graph_mem);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
//
// Open output file
//
// v=>vcf, z=>compressed vcf, b=>bcf, bu=>uncompressed bcf
int mode = vcf_misc_get_outtype(out_type, out_path);
futil_create_output(out_path);
htsFile *outfh = hts_open(out_path, modes_htslib[mode]);
status("[vcfcov] Output format: %s", hsmodes_htslib[mode]);
// Allocate memory
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, ncols, 1, kmers_in_hash,
DBG_ALLOC_COVGS);
//
// Set up tag names
//
// *R => ref, *A => alt
sprintf(kcov_ref_tag, "K%zuR", db_graph.kmer_size); // mean coverage
sprintf(kcov_alt_tag, "K%zuA", db_graph.kmer_size);
// #SAMPLE=<ID=...,K29KCOV=...,K29NK=...,K29RLK>
// - K29_kcov is empirical kmer coverage
// - K29_nkmers is the number of kmers in the sample
// - mean_read_length is the mean read length in bases
char sample_kcov_tag[20], sample_nk_tag[20], sample_rlk_tag[20];
sprintf(sample_kcov_tag, "K%zu_kcov", db_graph.kmer_size); // mean coverage
sprintf(sample_nk_tag, "K%zu_nkmers", db_graph.kmer_size);
sprintf(sample_rlk_tag, "mean_read_length");
//
// Load kmers if we are using --low-mem
//
VcfCovStats st;
memset(&st, 0, sizeof(st));
VcfCovPrefs prefs = {.kcov_ref_tag = kcov_ref_tag,
.kcov_alt_tag = kcov_alt_tag,
.max_allele_len = max_allele_len,
.max_gt_vars = max_gt_vars,
.load_kmers_only = false};
if(low_mem)
{
status("[vcfcov] Loading kmers from VCF+ref");
prefs.load_kmers_only = true;
vcfcov_file(vcffh, vcfhdr, NULL, NULL, vcf_path, fai,
NULL, &prefs, &st, &db_graph);
// Close files
hts_close(vcffh);
bcf_hdr_destroy(vcfhdr);
// Re-open files
if((vcffh = hts_open(vcf_path, "r")) == NULL)
die("Cannot re-open VCF file: %s", vcf_path);
if((vcfhdr = bcf_hdr_read(vcffh)) == NULL)
die("Cannot re-read VCF header: %s", vcf_path);
prefs.load_kmers_only = false;
}
//
// Load graphs
//
GraphLoadingStats gstats;
memset(&gstats, 0, sizeof(gstats));
GraphLoadingPrefs gprefs = graph_loading_prefs(&db_graph);
gprefs.must_exist_in_graph = low_mem;
for(i = 0; i < num_gfiles; i++) {
graph_load(&gfiles[i], gprefs, &gstats);
graph_file_close(&gfiles[i]);
}
ctx_free(gfiles);
hash_table_print_stats(&db_graph.ht);
//
// Set up VCF header / graph matchup
//
size_t *samplehdrids = ctx_malloc(db_graph.num_of_cols * sizeof(size_t));
// Add samples to vcf header
bcf_hdr_t *outhdr = bcf_hdr_dup(vcfhdr);
bcf_hrec_t *hrec;
int sid;
char hdrstr[200];
for(i = 0; i < db_graph.num_of_cols; i++) {
char *sname = db_graph.ginfo[i].sample_name.b;
if((sid = bcf_hdr_id2int(outhdr, BCF_DT_SAMPLE, sname)) < 0) {
bcf_hdr_add_sample(outhdr, sname);
sid = bcf_hdr_id2int(outhdr, BCF_DT_SAMPLE, sname);
}
samplehdrids[i] = sid;
// Add SAMPLE field
hrec = bcf_hdr_get_hrec(outhdr, BCF_HL_STR, "ID", sname, "SAMPLE");
if(hrec == NULL) {
sprintf(hdrstr, "##SAMPLE=<ID=%s,%s=%"PRIu64",%s=%"PRIu64",%s=%zu>", sname,
sample_kcov_tag,
gstats.nkmers[i] ? gstats.sumcov[i] / gstats.nkmers[i] : 0,
sample_nk_tag, gstats.nkmers[i],
sample_rlk_tag, (size_t)db_graph.ginfo[i].mean_read_length);
bcf_hdr_append(outhdr, hdrstr);
}
else {
// mean kcovg
sprintf(hdrstr, "%"PRIu64, gstats.sumcov[i] / gstats.nkmers[i]);
vcf_misc_add_update_hrec(hrec, sample_kcov_tag, hdrstr);
// num kmers
sprintf(hdrstr, "%"PRIu64, gstats.nkmers[i]);
vcf_misc_add_update_hrec(hrec, sample_nk_tag, hdrstr);
// mean read length in kmers
sprintf(hdrstr, "%zu", (size_t)db_graph.ginfo[i].mean_read_length);
vcf_misc_add_update_hrec(hrec, sample_rlk_tag, hdrstr);
}
status("[vcfcov] Colour %zu: %s [VCF column %zu]", i, sname, samplehdrids[i]);
}
// Add genotype format fields
// One field per alternative allele
sprintf(hdrstr, "##FORMAT=<ID=%s,Number=A,Type=Integer,"
"Description=\"Coverage on ref (k=%zu): sum(kmer_covs) / exp_num_kmers\">\n",
kcov_ref_tag, db_graph.kmer_size);
bcf_hdr_append(outhdr, hdrstr);
sprintf(hdrstr, "##FORMAT=<ID=%s,Number=A,Type=Integer,"
"Description=\"Coverage on alt (k=%zu): sum(kmer_covs) / exp_num_kmers\">\n",
kcov_alt_tag, db_graph.kmer_size);
bcf_hdr_append(outhdr, hdrstr);
bcf_hdr_set_version(outhdr, "VCFv4.2");
// Add command string to header
vcf_misc_hdr_add_cmd(outhdr, cmd_get_cmdline(), cmd_get_cwd());
if(bcf_hdr_write(outfh, outhdr) != 0)
die("Cannot write header to: %s", futil_outpath_str(out_path));
status("[vcfcov] Reading %s and adding coverage", vcf_path);
// Reset stats and get coverage
memset(&st, 0, sizeof(st));
vcfcov_file(vcffh, vcfhdr, outfh, outhdr, vcf_path, fai,
samplehdrids, &prefs, &st, &db_graph);
// Print statistics
char ns0[50], ns1[50];
status("[vcfcov] Read %s VCF lines", ulong_to_str(st.nvcf_lines, ns0));
status("[vcfcov] Read %s ALTs", ulong_to_str(st.nalts_read, ns0));
status("[vcfcov] Used %s kmers", ulong_to_str(st.ngt_kmers, ns0));
status("[vcfcov] ALTs used: %s / %s (%.2f%%)",
ulong_to_str(st.nalts_loaded, ns0), ulong_to_str(st.nalts_read, ns1),
st.nalts_read ? (100.0*st.nalts_loaded) / st.nalts_read : 0.0);
status("[vcfcov] ALTs too long (>%ubp): %s / %s (%.2f%%)", max_allele_len,
ulong_to_str(st.nalts_too_long, ns0), ulong_to_str(st.nalts_read, ns1),
st.nalts_read ? (100.0*st.nalts_too_long) / st.nalts_read : 0.0);
status("[vcfcov] ALTs too dense (>%u within %zubp): %s / %s (%.2f%%)",
max_gt_vars, db_graph.kmer_size,
ulong_to_str(st.nalts_no_covg, ns0), ulong_to_str(st.nalts_read, ns1),
st.nalts_read ? (100.0*st.nalts_no_covg) / st.nalts_read : 0.0);
status("[vcfcov] ALTs printed with coverage: %s / %s (%.2f%%)",
ulong_to_str(st.nalts_with_covg, ns0), ulong_to_str(st.nalts_read, ns1),
st.nalts_read ? (100.0*st.nalts_with_covg) / st.nalts_read : 0.0);
status("[vcfcov] Saved to: %s\n", out_path);
ctx_free(samplehdrids);
graph_loading_stats_destroy(&gstats);
bcf_hdr_destroy(vcfhdr);
bcf_hdr_destroy(outhdr);
hts_close(vcffh);
hts_close(outfh);
fai_destroy(fai);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}