int load_seqs(const char *path, char ***seqs_ptr, int *cap_ptr)
{
int cap = 1024;
char **seqs = my_malloc(sizeof(char*) * cap,__FILE__,__LINE__);
read_t read;
seq_read_alloc(&read);
seq_file_t *file = seq_open(path);
if(file == NULL) die("Cannot open file: %s.", path);
int num = 0;
while(seq_read(file, &read))
{
if(num == cap) {
cap *= 2;
seqs = realloc(seqs, sizeof(char*) * cap);
}
seqs[num++] = strdup(read.seq.b);
}
seq_read_dealloc(&read);
seq_close(file);
*seqs_ptr = seqs;
*cap_ptr = cap;
return num;
}
void filelist_dealloc(FileList *flist)
{
size_t i;
for(i = 0; i < flist->num_files; i++) seq_close(flist->files[i]);
seq_read_dealloc(&flist->read);
free(flist->files);
free(flist->fqoffsets);
free(flist->errors);
}
// Load reads from a file, apply sequence error, dump
// Return total number of bases
size_t mutate_reads(seq_file_t *sfile, gzFile gzout, FileList *flist, float err)
{
printf(" reading: %s\n", sfile->path);
read_t r;
seq_read_alloc(&r);
size_t num_bases = 0;
while(seq_read(sfile, &r) > 0) {
if(err > 0) add_seq_error_rate(r.seq.b, r.seq.end, err);
else add_seq_error_profile(r.seq.b, r.seq.end, flist);
gzprintf(gzout, "@%s\n%s\n+\n%s\n", r.name.b, r.seq.b, r.qual.b);
num_bases += r.seq.end;
}
seq_read_dealloc(&r);
return num_bases;
}
// Load all reads from files into a read buffer and close the seq_files
// Returns the number of reads loaded
size_t seq_load_all_reads(seq_file_t **seq_files, size_t num_files,
ReadBuffer *rbuf)
{
status("Loading sequences...");
size_t i, nreads = rbuf->len;
read_t r;
seq_read_alloc(&r);
for(i = 0; i < num_files; i++) {
status(" file: %s", seq_files[i]->path);
while(seq_read_primary(seq_files[i], &r) > 0) {
read_buf_push(rbuf, &r, 1); // copy read
seq_read_alloc(&r); // allocate new read
}
seq_close(seq_files[i]);
}
seq_read_dealloc(&r);
return rbuf->len - nreads;
}
int ctx_calls2vcf(int argc, char **argv)
{
const char *in_path = NULL, *out_path = NULL, *out_type = NULL;
// Filtering parameters
int32_t min_mapq = -1, max_align_len = -1, max_allele_len = -1;
// Alignment parameters
int nwmatch = 1, nwmismatch = -2, nwgapopen = -4, nwgapextend = -1;
// ref paths
char const*const* ref_paths = NULL;
size_t nref_paths = 0;
// flank file
const char *sam_path = NULL;
//
// Things we figure out by looking at the input
//
bool isbubble = false;
// samples in VCF, (0 for bubble, does not include ref in breakpoint calls)
size_t i, kmer_size, num_samples;
//
// Reference genome
//
// Hash map of chromosome name -> sequence
ChromHash *genome;
ReadBuffer chroms;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case 'O': cmd_check(!out_type, cmd); out_type = optarg; break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'F': cmd_check(!sam_path,cmd); sam_path = optarg; break;
case 'Q': cmd_check(min_mapq < 0,cmd); min_mapq = cmd_uint32(cmd, optarg); break;
case 'A': cmd_check(max_align_len < 0,cmd); max_align_len = cmd_uint32(cmd, optarg); break;
case 'L': cmd_check(max_allele_len < 0,cmd); max_allele_len = cmd_uint32(cmd, optarg); break;
case 'm': nwmatch = cmd_int32(cmd, optarg); break;
case 'M': nwmismatch = cmd_int32(cmd, optarg); break;
case 'g': nwgapopen = cmd_int32(cmd, optarg); break;
case 'G': nwgapextend = cmd_int32(cmd, optarg); break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
die("`"CMD" "SUBCMD" -h` for help. Bad option: %s", argv[optind-1]);
default: ctx_assert2(0, "shouldn't reach here: %c", c);
}
}
// Defaults for unset values
if(out_path == NULL) out_path = "-";
if(max_align_len < 0) max_align_len = DEFAULT_MAX_ALIGN;
if(max_allele_len < 0) max_allele_len = DEFAULT_MAX_ALLELE;
if(optind+2 > argc)
cmd_print_usage("Require <in.txt.gz> and at least one reference");
in_path = argv[optind++];
ref_paths = (char const*const*)argv + optind;
nref_paths = argc - optind;
// These functions call die() on error
gzFile gzin = futil_gzopen(in_path, "r");
// Read call file header
cJSON *json = json_hdr_load(gzin, in_path);
// Check we can handle the kmer size
kmer_size = json_hdr_get_kmer_size(json, in_path);
db_graph_check_kmer_size(kmer_size, in_path);
// Get format (bubble or breakpoint file)
cJSON *json_fmt = json_hdr_get(json, "file_format", cJSON_String, in_path);
if(strcmp(json_fmt->valuestring,"CtxBreakpoints") == 0) isbubble = false;
else if(strcmp(json_fmt->valuestring,"CtxBubbles") == 0) isbubble = true;
else die("Unknown format: '%s'", json_fmt->valuestring);
status("Reading %s in %s format", futil_inpath_str(in_path),
isbubble ? "bubble" : "breakpoint");
if(isbubble) {
// bubble specific
if(sam_path == NULL)
cmd_print_usage("Require -F <flanks.sam> with bubble file");
if(min_mapq < 0) min_mapq = DEFAULT_MIN_MAPQ;
}
else {
// breakpoint specific
if(min_mapq >= 0)
cmd_print_usage("-Q,--min-mapq <Q> only valid with bubble calls");
}
// Open flank file if it exists
htsFile *samfh = NULL;
bam_hdr_t *bam_hdr = NULL;
bam1_t *mflank = NULL;
if(sam_path)
{
if((samfh = hts_open(sam_path, "r")) == NULL)
die("Cannot open SAM/BAM %s", sam_path);
// Load BAM header
bam_hdr = sam_hdr_read(samfh);
if(bam_hdr == NULL) die("Cannot load BAM header: %s", sam_path);
mflank = bam_init1();
}
// Output VCF has 0 samples if bubbles file, otherwise has N where N is
// number of samples/colours in the breakpoint graph
size_t num_graph_samples = json_hdr_get_ncols(json, in_path);
size_t num_graph_nonref = json_hdr_get_nonref_ncols(json, in_path);
num_samples = 0;
if(!isbubble) {
// If last colour has "is_ref", drop number of samples by one
num_samples = num_graph_nonref < num_graph_samples ? num_graph_samples-1
: num_graph_samples;
}
//
// Open output file
//
if(!out_path) out_path = "-";
int mode = vcf_misc_get_outtype(out_type, out_path);
futil_create_output(out_path);
htsFile *vcffh = hts_open(out_path, modes_htslib[mode]);
status("[calls2vcf] Reading %s call file with %zu samples",
isbubble ? "Bubble" : "Breakpoint", num_graph_samples);
status("[calls2vcf] %zu sample output to: %s format: %s",
num_samples, futil_outpath_str(out_path), hsmodes_htslib[mode]);
if(isbubble) status("[calls2vcf] min. MAPQ: %i", min_mapq);
status("[calls2vcf] max alignment length: %i", max_align_len);
status("[calls2vcf] max VCF allele length: %i", max_allele_len);
status("[calls2vcf] alignment match:%i mismatch:%i gap open:%i extend:%i",
nwmatch, nwmismatch, nwgapopen, nwgapextend);
// Load reference genome
read_buf_alloc(&chroms, 1024);
genome = chrom_hash_init();
chrom_hash_load(ref_paths, nref_paths, &chroms, genome);
// convert to upper case
char *s;
for(i = 0; i < chroms.len; i++)
for(s = chroms.b[i].seq.b; *s; s++) *s = toupper(*s);
if(!isbubble) brkpnt_check_refs_match(json, genome, in_path);
bcf_hdr_t *vcfhdr = make_vcf_hdr(json, in_path, !isbubble, kmer_size,
ref_paths, nref_paths,
chroms.b, chroms.len);
if(bcf_hdr_write(vcffh, vcfhdr) != 0) die("Cannot write VCF header");
AlignedCall *call = acall_init();
CallDecomp *aligner = call_decomp_init(vcffh, vcfhdr);
scoring_t *scoring = call_decomp_get_scoring(aligner);
scoring_init(scoring, nwmatch, nwmismatch, nwgapopen, nwgapextend,
false, false, 0, 0, 0, 0);
CallFileEntry centry;
call_file_entry_alloc(¢ry);
char kmer_str[50];
sprintf(kmer_str, ";K%zu", kmer_size);
if(isbubble)
{
// Bubble calls
DecompBubble *bubbles = decomp_bubble_init();
// Set scoring for aligning 3' flank
scoring = decomp_bubble_get_scoring(bubbles);
scoring_init(scoring, nwmatch, nwmismatch, nwgapopen, nwgapextend,
true, true, 0, 0, 0, 0);
while(call_file_read(gzin, in_path, ¢ry)) {
do {
if(sam_read1(samfh, bam_hdr, mflank) < 0)
die("We've run out of SAM entries!");
} while(mflank->core.flag & (BAM_FSECONDARY | BAM_FSUPPLEMENTARY));
// Align call
strbuf_reset(&call->info);
decomp_bubble_call(bubbles, genome, kmer_size, min_mapq,
¢ry, mflank, bam_hdr, call);
strbuf_append_str(&call->info, kmer_str);
acall_decompose(aligner, call, max_align_len, max_allele_len);
}
// print bubble stats
DecompBubbleStats *bub_stats = ctx_calloc(1, sizeof(*bub_stats));
decomp_bubble_cpy_stats(bub_stats, bubbles);
print_bubble_stats(bub_stats);
ctx_free(bub_stats);
decomp_bubble_destroy(bubbles);
}
else
{
// Breakpoint calls
DecompBreakpoint *breakpoints = decomp_brkpt_init();
while(call_file_read(gzin, in_path, ¢ry)) {
strbuf_reset(&call->info);
decomp_brkpt_call(breakpoints, genome, num_samples, ¢ry, call);
strbuf_append_str(&call->info, kmer_str);
acall_decompose(aligner, call, max_align_len, max_allele_len);
}
// print bubble stats
DecompBreakpointStats *brk_stats = ctx_calloc(1, sizeof(*brk_stats));
decomp_brkpt_cpy_stats(brk_stats, breakpoints);
print_breakpoint_stats(brk_stats);
ctx_free(brk_stats);
decomp_brkpt_destroy(breakpoints);
}
// Print stats
DecomposeStats *astats = ctx_calloc(1, sizeof(*astats));
call_decomp_cpy_stats(astats, aligner);
print_acall_stats(astats);
ctx_free(astats);
call_file_entry_dealloc(¢ry);
call_decomp_destroy(aligner);
acall_destroy(call);
// Finished - clean up
cJSON_Delete(json);
gzclose(gzin);
bcf_hdr_destroy(vcfhdr);
hts_close(vcffh);
for(i = 0; i < chroms.len; i++) seq_read_dealloc(&chroms.b[i]);
read_buf_dealloc(&chroms);
chrom_hash_destroy(genome);
if(sam_path) {
hts_close(samfh);
bam_hdr_destroy(bam_hdr);
bam_destroy1(mflank);
}
return EXIT_SUCCESS;
}
int ctx_rmsubstr(int argc, char **argv)
{
struct MemArgs memargs = MEM_ARGS_INIT;
size_t kmer_size = 0, nthreads = 0;
const char *output_file = NULL;
seq_format fmt = SEQ_FMT_FASTA;
bool invert = false;
// Arg parsing
char cmd[100], shortopts[100];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'o': cmd_check(!output_file, cmd); output_file = optarg; break;
case 't': cmd_check(!nthreads, cmd); nthreads = cmd_uint32_nonzero(cmd, optarg); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'k': cmd_check(!kmer_size,cmd); kmer_size = cmd_uint32(cmd, optarg); break;
case 'F': cmd_check(fmt==SEQ_FMT_FASTA, cmd); fmt = cmd_parse_format(cmd, optarg); break;
case 'v': cmd_check(!invert,cmd); invert = true; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
cmd_print_usage("`"CMD" rmsubstr -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
// Defaults
if(!nthreads) nthreads = DEFAULT_NTHREADS;
if(!kmer_size) kmer_size = DEFAULT_KMER;
if(!(kmer_size&1)) cmd_print_usage("Kmer size must be odd");
if(kmer_size < MIN_KMER_SIZE) cmd_print_usage("Kmer size too small (recompile)");
if(kmer_size > MAX_KMER_SIZE) cmd_print_usage("Kmer size too large (recompile?)");
if(optind >= argc)
cmd_print_usage("Please specify at least one input sequence file (.fq, .fq etc.)");
size_t i, num_seq_files = argc - optind;
char **seq_paths = argv + optind;
seq_file_t **seq_files = ctx_calloc(num_seq_files, sizeof(seq_file_t*));
for(i = 0; i < num_seq_files; i++)
if((seq_files[i] = seq_open(seq_paths[i])) == NULL)
die("Cannot read sequence file %s", seq_paths[i]);
// Estimate number of bases
// set to -1 if we cannot calc
int64_t est_num_bases = seq_est_seq_bases(seq_files, num_seq_files);
if(est_num_bases < 0) {
warn("Cannot get file sizes, using pipes");
est_num_bases = memargs.num_kmers * IDEAL_OCCUPANCY;
}
status("[memory] Estimated number of bases: %li", (long)est_num_bases);
// Use file sizes to decide on memory
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem;
bits_per_kmer = sizeof(BinaryKmer)*8 +
sizeof(KONodeList) + sizeof(KOccur) + // see kmer_occur.h
8; // 1 byte per kmer for each base to load sequence files
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
est_num_bases, est_num_bases,
false, &graph_mem);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
//
// Open output file
//
if(output_file == NULL) output_file = "-";
FILE *fout = futil_fopen_create(output_file, "w");
//
// Set up memory
//
dBGraph db_graph;
db_graph_alloc(&db_graph, kmer_size, 1, 0, kmers_in_hash, DBG_ALLOC_BKTLOCKS);
//
// Load reference sequence into a read buffer
//
ReadBuffer rbuf;
read_buf_alloc(&rbuf, 1024);
seq_load_all_reads(seq_files, num_seq_files, &rbuf);
// Check for reads too short
for(i = 0; i < rbuf.len && rbuf.b[i].seq.end >= kmer_size; i++) {}
if(i < rbuf.len)
warn("Reads shorter than kmer size (%zu) will not be filtered", kmer_size);
KOGraph kograph = kograph_create(rbuf.b, rbuf.len, true, 0,
nthreads, &db_graph);
size_t num_reads = rbuf.len, num_reads_printed = 0, num_bad_reads = 0;
// Loop over reads printing those that are not substrings
int ret;
for(i = 0; i < rbuf.len; i++) {
ret = _is_substr(&rbuf, i, &kograph, &db_graph);
if(ret == -1) num_bad_reads++;
else if((ret && invert) || (!ret && !invert)) {
seqout_print_read(&rbuf.b[i], fmt, fout);
num_reads_printed++;
}
}
char num_reads_str[100], num_reads_printed_str[100], num_bad_reads_str[100];
ulong_to_str(num_reads, num_reads_str);
ulong_to_str(num_reads_printed, num_reads_printed_str);
ulong_to_str(num_bad_reads, num_bad_reads_str);
status("Printed %s / %s (%.1f%%) to %s",
num_reads_printed_str, num_reads_str,
!num_reads ? 0.0 : (100.0 * num_reads_printed) / num_reads,
futil_outpath_str(output_file));
if(num_bad_reads > 0) {
status("Bad reads: %s / %s (%.1f%%) - no kmer {ACGT} of length %zu",
num_bad_reads_str, num_reads_str,
(100.0 * num_bad_reads) / num_reads,
kmer_size);
}
fclose(fout);
kograph_dealloc(&kograph);
// Free sequence memory
for(i = 0; i < rbuf.len; i++) seq_read_dealloc(&rbuf.b[i]);
read_buf_dealloc(&rbuf);
ctx_free(seq_files);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
int ctx_calls2vcf(int argc, char **argv)
{
parse_cmdline_args(argc, argv);
size_t i;
// These functions call die() on error
gzFile gzin = futil_gzopen(input_path, "r");
nw_aligner_setup();
// Read file header
cJSON *json = read_input_header(gzin);
// Get format (bubble or breakpoint file)
cJSON *json_fmt = json_hdr_get(json, "file_format", cJSON_String, input_path);
if(strcmp(json_fmt->valuestring,"CtxBreakpoints") == 0) input_bubble_format = false;
else if(strcmp(json_fmt->valuestring,"CtxBubbles") == 0) input_bubble_format = true;
else die("Unknown format: '%s'", json_fmt->valuestring);
status("Reading %s in %s format", futil_inpath_str(input_path),
input_bubble_format ? "bubble" : "breakpoint");
if(input_bubble_format && sam_path == NULL)
cmd_print_usage("Require -F <flanks.sam> with bubble file");
// Open flank file if it exists
if(sam_path) flanks_sam_open();
// Open output file
FILE *fout = futil_fopen_create(out_path, "w");
// Load reference genome
read_buf_alloc(&chroms, 1024);
genome = kh_init(ChromHash);
seq_reader_load_ref_genome(ref_paths, num_ref_paths, &chroms, genome);
// convert to upper case
char *s;
for(i = 0; i < chroms.len; i++)
for(s = chroms.b[i].seq.b; *s; s++) *s = toupper(*s);
if(!input_bubble_format) brkpnt_check_refs_match(json, input_path);
// Output VCF has 0 samples if bubbles file, otherwise has N where N is
// number of samples/colours in the breakpoint graph
size_t num_graph_samples = json_hdr_get_ncols(json, input_path);
size_t num_graph_nonref = json_hdr_get_nonref_ncols(json, input_path);
num_samples = 0;
if(!input_bubble_format) {
// If last colour has "is_ref", drop number of samples by one
num_samples = num_graph_nonref < num_graph_samples ? num_graph_samples-1
: num_graph_samples;
}
print_vcf_header(json, !input_bubble_format, fout);
status("Reading %s call file with %zu samples",
input_bubble_format ? "Bubble" : "Breakpoint", num_graph_samples);
status("Writing a VCF with %zu samples", num_samples);
parse_entries(gzin, fout);
// Print stats
char num_entries_read_str[50];
char num_vars_printed_str[50];
ulong_to_str(num_entries_read, num_entries_read_str);
ulong_to_str(num_vars_printed, num_vars_printed_str);
status("Read %s entries, printed %s vcf entries to: %s",
num_entries_read_str, num_vars_printed_str, futil_outpath_str(out_path));
if(input_bubble_format) {
char msg[200];
// Bubble caller specific
print_stat(num_flank5p_unmapped, num_entries_read, "flank 5p unmapped");
sprintf(msg, "flank 5p low mapq (<%zu)", min_mapq);
print_stat(num_flank5p_lowqual, num_entries_read, msg);
print_stat(num_flank3p_not_found, num_entries_read, "flank 3p not found");
print_stat(num_flank3p_multihits, num_entries_read, "flank 3p multiple hits");
print_stat(num_flank3p_approx_match,num_entries_read, "flank 3p approx match used");
print_stat(num_flank3p_exact_match, num_entries_read, "flank 3p exact match");
} else {
// Breakpoint caller specific
print_stat(num_flanks_not_uniquely_mapped, num_entries_read, "flank pairs contain one flank not mapped uniquely");
print_stat(num_flanks_diff_chroms, num_entries_read, "flank pairs map to diff chroms");
print_stat(num_flanks_diff_strands, num_entries_read, "flank pairs map to diff strands");
}
print_stat(num_flanks_too_far_apart, num_entries_read, "flank pairs too far apart");
print_stat(num_flanks_overlap_too_large, num_entries_read, "flank pairs overlap too much");
print_stat(num_entries_well_mapped, num_entries_read, "flank pairs map well");
status("Aligned %zu allele pairs and %zu flanks", num_nw_allele, num_nw_flank);
// Finished - clean up
cJSON_Delete(json);
gzclose(gzin);
fclose(fout);
for(i = 0; i < chroms.len; i++) seq_read_dealloc(&chroms.b[i]);
read_buf_dealloc(&chroms);
kh_destroy_ChromHash(genome);
nw_aligner_destroy();
if(sam_path) flanks_sam_close();
// hide unused method warnings
(void)kh_del_ChromHash;
(void)kh_put_ChromHash;
(void)kh_get_ChromHash;
(void)kh_clear_ChromHash;
(void)kh_destroy_ChromHash;
(void)kh_init_ChromHash;
return EXIT_SUCCESS;
}
int main(int argc, char **argv)
{
// compiler complains about unused function without these linese
(void)kh_clear_ghash;
(void)kh_del_ghash;
if(argc < 2) print_usage(usage, NULL);
char swap_alleles = 0;
int c;
while((c = getopt(argc, argv, "s")) >= 0) {
switch (c) {
case 's': swap_alleles = 1; break;
default: die("Unknown option: %c", c);
}
}
if(optind == argc) print_usage(usage, "Not enough arguments");
char *inputpath = argv[optind];
char **refpaths = argv + optind + 1;
size_t num_refs = argc - optind - 1;
gzFile gzin = gzopen(inputpath, "r");
if(gzin == NULL) die("Cannot read file: %s", inputpath);
size_t i, nchroms = 0, capacity = 1024;
khash_t(ghash) *genome = kh_init(ghash);
read_t *reads = malloc(capacity * sizeof(read_t)), *r;
int hret;
khiter_t k;
for(i = 0; i < num_refs; i++) {
fprintf(stderr, "Loading %s\n", refpaths[i]);
load_reads(refpaths[i], &reads, &capacity, &nchroms);
}
if(num_refs == 0) {
fprintf(stderr, "Loading from stdin\n");
load_reads("-", &reads, &capacity, &nchroms);
}
if(nchroms == 0) die("No chromosomes loaded");
for(i = 0; i < nchroms; i++) {
r = reads + i;
fprintf(stderr, "Loaded: '%s'\n", r->name.b);
k = kh_put(ghash, genome, r->name.b, &hret);
if(hret == 0) warn("Duplicate read name (taking first): %s", r->name.b);
else kh_value(genome, k) = r;
}
// Now read VCF
StrBuf line;
strbuf_alloc(&line, 1024);
char *fields[9];
char *chr;
int pos, reflen, altlen;
while(strbuf_reset_gzreadline(&line, gzin) > 0)
{
if(line.b[0] == '#') fputs(line.b, stdout);
else
{
strbuf_chomp(&line);
vcf_columns(line.b, fields);
fields[1][-1] = fields[2][-1] = '\0';
chr = line.b;
pos = atoi(fields[1])-1;
k = kh_get(ghash, genome, chr);
r = kh_value(genome, k);
fields[1][-1] = fields[2][-1] = '\t';
reflen = fields[4] - fields[3] - 1;
altlen = fields[5] - fields[4] - 1;
if(k == kh_end(genome)) warn("Cannot find chrom: %s", chr);
else if(pos < 0) warn("Bad line: %s\n", line.b);
else if((reflen == 1 && altlen == 1) || fields[3][0] == fields[4][0])
{
if((unsigned)pos + reflen <= r->seq.end &&
strncasecmp(r->seq.b+pos,fields[3],reflen) == 0)
{
fputs(line.b, stdout);
fputc('\n', stdout);
}
else if(swap_alleles && (unsigned)pos + altlen <= r->seq.end &&
strncasecmp(r->seq.b+pos,fields[4],altlen) == 0)
{
// swap alleles
char tmp[altlen], *ref = fields[3], *alt = fields[4];
memcpy(tmp, alt, altlen);
memmove(ref+altlen+1, ref, reflen);
memcpy(ref, tmp, altlen);
ref[altlen] = '\t';
fputs(line.b, stdout);
fputc('\n', stdout);
}
// else printf("FAIL0\n");
}
// else printf("FAIL1\n");
}
}
kh_destroy(ghash, genome);
strbuf_dealloc(&line);
gzclose(gzin);
for(i = 0; i < nchroms; i++) seq_read_dealloc(reads+i);
free(reads);
fprintf(stderr, " Done.\n");
return 0;
}
static void test_kmer_occur_filter()
{
// Construct 1 colour graph with kmer-size=11
dBGraph graph;
const size_t kmer_size = 11, ncols = 3;
size_t i;
// Create graph
db_graph_alloc(&graph, kmer_size, ncols, 1, 2000,
DBG_ALLOC_EDGES | DBG_ALLOC_NODE_IN_COL | DBG_ALLOC_BKTLOCKS);
// xyz------->>> y > < X
// TTCGACCCGACAGGGCAACGTAGTCCGACAGGGCACAGCCCTGTCGGGGGGTGCA
#define NUM_NODES 3
#define NUM_READS 3
const char *tmp[NUM_READS]
= {
"AACA",
"TTCGACCCGACAGGGCAACGTAGTCCGACAGGGCACAGCCCTGTCGGGGGGTGCA",
"TCTAGCATGTGTGTT"};
read_t reads[NUM_READS];
for(i = 0; i < NUM_READS; i++) {
seq_read_alloc(&reads[i]);
seq_read_set(&reads[i], tmp[i]);
}
KOGraph kograph = kograph_create(reads, NUM_READS, true, 0, 1, &graph);
TASSERT(kograph.nchroms == NUM_READS);
TASSERT(kograph.koccurs != NULL);
KOccurRunBuffer koruns, koruns_tmp, koruns_ended;
korun_buf_alloc(&koruns, 16);
korun_buf_alloc(&koruns_tmp, 16);
korun_buf_alloc(&koruns_ended, 16);
// Check CCCGACAGGGCAA starts at CCCGACAGGGC
// x=CCCGACAGGGC, y=CCGACAGGGCA, z=CGACAGGGCAA
// X=GCCCTGTCGGG, Y=TGCCCTGTCGG, Z=TTGCCCTGTCG
dBNode nodes[NUM_NODES];
for(i = 0; i < NUM_NODES; i++)
nodes[i] = db_graph_find_str(&graph, &"CCCGACAGGGCAA"[i]);
korun_buf_reset(&koruns);
korun_buf_reset(&koruns_ended);
kograph_filter_extend(&kograph, nodes, NUM_NODES, true, 0, 0,
&koruns, &koruns_tmp, &koruns_ended);
// Checks
TASSERT2(koruns.len == 1, "koruns.len: %zu", koruns.len);
TASSERT(koruns.b[0].strand == STRAND_PLUS); // left-to-right with ref
TASSERT2(koruns.b[0].chrom == 1, "chrom: %zu", (size_t)koruns.b[0].chrom);
TASSERT2(koruns.b[0].first == 5, "offset: %zu", (size_t)koruns.b[0].first);
TASSERT2(koruns.b[0].last == 7, "last: %zu", (size_t)koruns.b[0].last);
// Test reverse
db_nodes_reverse_complement(nodes, NUM_NODES);
korun_buf_reset(&koruns);
korun_buf_reset(&koruns_ended);
kograph_filter_extend(&kograph, nodes, 1, true, 0, 0, &koruns, &koruns_tmp, &koruns_ended);
kograph_filter_extend(&kograph, nodes+1, 1, true, 0, 1, &koruns, &koruns_tmp, &koruns_ended);
kograph_filter_extend(&kograph, nodes+2, 1, true, 0, 2, &koruns, &koruns_tmp, &koruns_ended);
// Print out for debugging
// printf("koruns: ");
// koruns_print(koruns.b, koruns.len, kmer_size, stdout);
// printf("\nkoruns_ended: ");
// koruns_print(koruns_ended.b, koruns_ended.len, kmer_size, stdout);
// printf("\n");
// Check results match:
// koruns: chromid:1:17-5:-, chromid:1:37-47:+
// koruns_ended: chromid:1:34-24:-
TASSERT2(koruns.len == 2, "koruns.len: %zu", koruns.len);
TASSERT2(koruns_ended.len == 1, "koruns_ended.len: %zu", koruns_ended.len);
TASSERT(koruns.b[0].strand == STRAND_MINUS); // reverse complement of ref
TASSERT2(koruns.b[0].chrom == 1, "chrom: %zu", (size_t)koruns.b[0].chrom);
TASSERT2(koruns.b[0].first == 7, "offset: %zu", (size_t)koruns.b[0].first);
TASSERT2(koruns.b[0].last == 5, "last: %zu", (size_t)koruns.b[0].last);
korun_buf_dealloc(&koruns);
korun_buf_dealloc(&koruns_tmp);
korun_buf_dealloc(&koruns_ended);
for(i = 0; i < NUM_READS; i++) seq_read_dealloc(&reads[i]);
kograph_dealloc(&kograph);
db_graph_dealloc(&graph);
}
// If seq2 is NULL, read pair of entries from first file
// Otherwise read an entry from each
void align_from_file(const char *path1, const char *path2,
void (align)(read_t *r1, read_t *r2),
bool use_zlib)
{
seq_file_t *sf1, *sf2;
if((sf1 = open_seq_file(path1, use_zlib)) == NULL)
{
fprintf(stderr, "Alignment Error: couldn't open file %s\n", path1);
fflush(stderr);
return;
}
if(path2 == NULL)
{
sf2 = sf1;
}
else if((sf2 = open_seq_file(path2, use_zlib)) == NULL)
{
fprintf(stderr, "Alignment Error: couldn't open file %s\n", path1);
fflush(stderr);
return;
}
// fprintf(stderr, "File buffer %zu zlib: %i\n", sf1->in.size, seq_use_gzip(sf1));
read_t read1, read2;
seq_read_alloc(&read1);
seq_read_alloc(&read2);
// Loop while we can read a sequence from the first file
unsigned long alignments;
for(alignments = 0; seq_read(sf1, &read1) > 0; alignments++)
{
if(seq_read(sf2, &read2) <= 0)
{
fprintf(stderr, "Alignment Error: Odd number of sequences - "
"I read in pairs!\n");
fflush(stderr);
break;
}
(align)(&read1, &read2);
}
// warn if no bases read
if(alignments == 0)
{
fprintf(stderr, "Alignment Warning: empty input\n");
fflush(stderr);
}
// Close files
seq_close(sf1);
if(path2 != NULL)
seq_close(sf2);
// Free memory
seq_read_dealloc(&read1);
seq_read_dealloc(&read2);
}
// Returns num of bases printed
size_t sim_reads(seq_file_t *reffile, gzFile out0, gzFile out1,
FileList *flist, float err_rate,
size_t insert, double insert_stddev, size_t rlen, double depth)
{
size_t i, chromcap = 16, nchroms, glen = 0, nreads, chr, pos0, pos1, tlen;
read_t *chroms;
tlen = rlen + (out1 == NULL ? 0 : insert + rlen);
chroms = malloc(chromcap * sizeof(read_t));
nchroms = 0;
// Load genome
printf(" Loaded contigs:");
while(1)
{
if(nchroms == chromcap) chroms = realloc(chroms, (chromcap*=2)*sizeof(read_t));
seq_read_alloc(&chroms[nchroms]);
if(seq_read(reffile, &chroms[nchroms]) <= 0)
{ seq_read_dealloc(&chroms[nchroms]); break; }
if(chroms[nchroms].seq.end < tlen) { seq_read_dealloc(&chroms[nchroms]); }
else {
seq_read_truncate_name(&chroms[nchroms]);
printf(" %s[%zu]", chroms[nchroms].name.b, chroms[nchroms].seq.end);
glen += chroms[nchroms].seq.end;
nchroms++;
}
}
printf("\n Genome size: %zu\n", glen);
if(nchroms == 0) {
die("No sequences long enough in ref genome file [min len: %zu]: %s",
tlen, reffile->path);
}
// Sample
nreads = (glen * depth) / (out1 == NULL ? rlen : (2 * rlen));
char read0[rlen+1], read1[rlen+1];
read0[rlen] = read1[rlen] = '\0';
printf("Sampling %zu %sreads...\n", nreads,
out1 == NULL ? "single " : "paired-end ");
// Sample paired-end if out1 != NULL
for(i = 0; i < nreads; i++)
{
chr = (nchroms == 1) ? 0 : rand_chrom(chroms, nchroms, glen);
pos0 = random_uniform(chroms[chr].seq.end - (out1 == NULL ? rlen : tlen));
pos1 = pos0;
memcpy(read0, chroms[chr].seq.b+pos0, rlen);
if(out1 != NULL) {
pos1 = pos0 + rlen + insert + ran_normal()*insert_stddev;
if(pos1 + rlen > chroms[chr].seq.end) pos1 = chroms[chr].seq.end-rlen;
memcpy(read1, chroms[chr].seq.b+pos1, rlen);
}
if(flist != NULL) {
add_seq_error_profile(read0, rlen, flist);
if(out1 != NULL)
add_seq_error_profile(read1, rlen, flist);
}
else if(err_rate >= 0) {
add_seq_error_rate(read0, rlen, err_rate);
}
gzprintf(out0, ">r%zu:%s:%zu:%zu%s\n%.*s\n", i, chroms[chr].name.b,
pos0, pos1, (out1 != NULL ? "/1" : ""), (int)rlen, read0);
if(out1 != NULL) {
dna_revcmp(read1, rlen);
gzprintf(out1, ">r%zu:%s:%zu:%zu/2\n%.*s\n", i, chroms[chr].name.b,
pos0, pos1, (int)rlen, read1);
}
}
for(i = 0; i < nchroms; i++) seq_read_dealloc(&chroms[i]);
free(chroms);
size_t num_bases = nreads * rlen;
if(out1 != NULL) num_bases *= 2;
return num_bases;
}
