/**
* Reads next record, hides the random access of different regions from the user.
*/
bool BCFOrderedReader::read(bcf1_t *v)
{
if (random_access_enabled)
{
if (ftype.format==bcf)
{
while(true)
{
if (itr && bcf_itr_next(file, itr, v)>=0)
{
return true;
}
else if (!initialize_next_interval())
{
return false;
}
}
}
else
{
while(true)
{
if (itr && tbx_itr_next(file, tbx, itr, &s)>=0)
{
vcf_parse1(&s, hdr, v);
return true;
}
else if (!initialize_next_interval())
{
return false;
}
}
}
}
else
{
if (bcf_read(file, hdr, v)==0)
{
return true;
}
else
{
return false;
}
}
return false;
};
static int view_vcf(hFILE *hfp, const char *filename)
{
vcfFile *in = hts_hopen(hfp, filename, "r");
if (in == NULL) return 0;
vcfFile *out = dup_stdout("w");
bcf_hdr_t *hdr = bcf_hdr_read(in);
if (show_headers) bcf_hdr_write(out, hdr);
if (mode == view_all) {
bcf1_t *rec = bcf_init();
while (bcf_read(in, hdr, rec) >= 0)
bcf_write(out, hdr, rec);
bcf_destroy(rec);
}
bcf_hdr_destroy(hdr);
hts_close(out);
hts_close(in);
return 1;
}
static void init_data(args_t *args)
{
bcf1_t *line = NULL;
// With phased concat, the chunks overlap and come in the right order. To
// avoid opening all files at once, store start positions to recognise need
// for the next one. This way we can keep only two open chunks at once.
if ( args->phased_concat )
{
args->start_pos = (int*) malloc(sizeof(int)*args->nfnames);
line = bcf_init();
}
kstring_t str = {0,0,0};
int i, prev_chrid = -1;
for (i=0; i<args->nfnames; i++)
{
htsFile *fp = hts_open(args->fnames[i], "r"); if ( !fp ) error("Failed to open: %s\n", args->fnames[i]);
bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to parse header: %s\n", args->fnames[i]);
args->out_hdr = bcf_hdr_merge(args->out_hdr,hdr);
if ( bcf_hdr_nsamples(hdr) != bcf_hdr_nsamples(args->out_hdr) )
error("Different number of samples in %s. Perhaps \"bcftools merge\" is what you are looking for?\n", args->fnames[i]);
int j;
for (j=0; j<bcf_hdr_nsamples(hdr); j++)
if ( strcmp(args->out_hdr->samples[j],hdr->samples[j]) )
error("Different sample names in %s. Perhaps \"bcftools merge\" is what you are looking for?\n", args->fnames[i]);
if ( args->phased_concat )
{
int ret = bcf_read(fp, hdr, line);
if ( ret!=0 ) args->start_pos[i] = -2; // empty file
else
{
int chrid = bcf_hdr_id2int(args->out_hdr,BCF_DT_CTG,bcf_seqname(hdr,line));
args->start_pos[i] = chrid==prev_chrid ? line->pos : -1;
prev_chrid = chrid;
}
}
bcf_hdr_destroy(hdr);
hts_close(fp);
}
free(str.s);
if ( line ) bcf_destroy(line);
args->seen_seq = (int*) calloc(args->out_hdr->n[BCF_DT_CTG],sizeof(int));
if ( args->phased_concat )
{
bcf_hdr_append(args->out_hdr,"##FORMAT=<ID=PQ,Number=1,Type=Integer,Description=\"Phasing Quality (bigger is better)\">");
bcf_hdr_append(args->out_hdr,"##FORMAT=<ID=PS,Number=1,Type=Integer,Description=\"Phase Set\">");
}
if (args->record_cmd_line) bcf_hdr_append_version(args->out_hdr, args->argc, args->argv, "bcftools_concat");
args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
if ( args->n_threads ) hts_set_threads(args->out_fh, args->n_threads);
bcf_hdr_write(args->out_fh, args->out_hdr);
if ( args->allow_overlaps )
{
args->files = bcf_sr_init();
args->files->require_index = 1;
if ( args->regions_list )
{
if ( bcf_sr_set_regions(args->files, args->regions_list, args->regions_is_file)<0 )
error("Failed to read the regions: %s\n", args->regions_list);
}
if ( args->remove_dups )
{
if ( !strcmp(args->remove_dups,"snps") ) args->files->collapse |= COLLAPSE_SNPS;
else if ( !strcmp(args->remove_dups,"indels") ) args->files->collapse |= COLLAPSE_INDELS;
else if ( !strcmp(args->remove_dups,"both") ) args->files->collapse |= COLLAPSE_SNPS | COLLAPSE_INDELS;
else if ( !strcmp(args->remove_dups,"any") ) args->files->collapse |= COLLAPSE_ANY;
else if ( !strcmp(args->remove_dups,"all") ) args->files->collapse |= COLLAPSE_ANY;
else if ( !strcmp(args->remove_dups,"none") ) args->files->collapse = COLLAPSE_NONE;
else error("The -D string \"%s\" not recognised.\n", args->remove_dups);
}
for (i=0; i<args->nfnames; i++)
if ( !bcf_sr_add_reader(args->files,args->fnames[i]) ) error("Failed to open %s: %s\n", args->fnames[i],bcf_sr_strerror(args->files->errnum));
}
else if ( args->phased_concat )
{
// Remove empty files from the list
int nok = 0;
while (1)
{
while ( nok<args->nfnames && args->start_pos[nok]!=-2 ) nok++;
if ( nok==args->nfnames ) break;
i = nok;
while ( i<args->nfnames && args->start_pos[i]==-2 ) i++;
if ( i==args->nfnames ) break;
int tmp = args->start_pos[nok]; args->start_pos[nok] = args->start_pos[i]; args->start_pos[i] = tmp;
char *str = args->fnames[nok]; args->fnames[nok] = args->fnames[i]; args->fnames[i] = str;
}
for (i=nok; i<args->nfnames; i++) free(args->fnames[i]);
args->nfnames = nok;
for (i=1; i<args->nfnames; i++)
if ( args->start_pos[i-1]!=-1 && args->start_pos[i]!=-1 && args->start_pos[i]<args->start_pos[i-1] )
error("The files not in ascending order: %d in %s, %d in %s\n", args->start_pos[i-1]+1,args->fnames[i-1],args->start_pos[i]+1,args->fnames[i]);
args->prev_chr = -1;
args->swap_phase = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->nmatch = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->nmism = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->phase_qual = (int32_t*) malloc(bcf_hdr_nsamples(args->out_hdr)*sizeof(int32_t));
args->phase_set = (int32_t*) malloc(bcf_hdr_nsamples(args->out_hdr)*sizeof(int32_t));
args->files = bcf_sr_init();
args->files->require_index = 1;
args->ifname = 0;
}
}
static void concat(args_t *args)
{
int i;
if ( args->phased_concat ) // phased concat
{
// keep only two open files at a time
while ( args->ifname < args->nfnames )
{
int new_file = 0;
while ( args->files->nreaders < 2 && args->ifname < args->nfnames )
{
if ( !bcf_sr_add_reader(args->files,args->fnames[args->ifname]) ) error("Failed to open %s: %s\n", args->fnames[args->ifname],bcf_sr_strerror(args->files->errnum));
new_file = 1;
args->ifname++;
if ( args->start_pos[args->ifname-1]==-1 ) break; // new chromosome, start with only one file open
if ( args->ifname < args->nfnames && args->start_pos[args->ifname]==-1 ) break; // next file starts on a different chromosome
}
// is there a line from the previous run? Seek the newly opened reader to that position
int seek_pos = -1;
int seek_chr = -1;
if ( bcf_sr_has_line(args->files,0) )
{
bcf1_t *line = bcf_sr_get_line(args->files,0);
bcf_sr_seek(args->files, bcf_seqname(args->files->readers[0].header,line), line->pos);
seek_pos = line->pos;
seek_chr = bcf_hdr_name2id(args->out_hdr, bcf_seqname(args->files->readers[0].header,line));
}
else if ( new_file )
bcf_sr_seek(args->files,NULL,0); // set to start
int nret;
while ( (nret = bcf_sr_next_line(args->files)) )
{
if ( !bcf_sr_has_line(args->files,0) ) // no input from the first reader
{
// We are assuming that there is a perfect overlap, sites which are not present in both files are dropped
if ( ! bcf_sr_region_done(args->files,0) ) continue;
phased_flush(args);
bcf_sr_remove_reader(args->files, 0);
}
// Get a line to learn about current position
for (i=0; i<args->files->nreaders; i++)
if ( bcf_sr_has_line(args->files,i) ) break;
bcf1_t *line = bcf_sr_get_line(args->files,i);
// This can happen after bcf_sr_seek: indel may start before the coordinate which we seek to.
if ( seek_chr>=0 && seek_pos>line->pos && seek_chr==bcf_hdr_name2id(args->out_hdr, bcf_seqname(args->files->readers[i].header,line)) ) continue;
seek_pos = seek_chr = -1;
// Check if the position overlaps with the next, yet unopened, reader
int must_seek = 0;
while ( args->ifname < args->nfnames && args->start_pos[args->ifname]!=-1 && line->pos >= args->start_pos[args->ifname] )
{
must_seek = 1;
if ( !bcf_sr_add_reader(args->files,args->fnames[args->ifname]) ) error("Failed to open %s: %s\n", args->fnames[args->ifname],bcf_sr_strerror(args->files->errnum));
args->ifname++;
}
if ( must_seek )
{
bcf_sr_seek(args->files, bcf_seqname(args->files->readers[i].header,line), line->pos);
seek_pos = line->pos;
seek_chr = bcf_hdr_name2id(args->out_hdr, bcf_seqname(args->files->readers[i].header,line));
continue;
}
// We are assuming that there is a perfect overlap, sites which are not present in both files are dropped
if ( args->files->nreaders>1 && !bcf_sr_has_line(args->files,1) && !bcf_sr_region_done(args->files,1) ) continue;
phased_push(args, bcf_sr_get_line(args->files,0), args->files->nreaders>1 ? bcf_sr_get_line(args->files,1) : NULL);
}
if ( args->files->nreaders )
{
phased_flush(args);
while ( args->files->nreaders )
bcf_sr_remove_reader(args->files, 0);
}
}
}
else if ( args->files ) // combining overlapping files, using synced reader
{
while ( bcf_sr_next_line(args->files) )
{
for (i=0; i<args->files->nreaders; i++)
{
bcf1_t *line = bcf_sr_get_line(args->files,i);
if ( !line ) continue;
bcf_translate(args->out_hdr, args->files->readers[i].header, line);
bcf_write1(args->out_fh, args->out_hdr, line);
if ( args->remove_dups ) break;
}
}
}
else // concatenating
{
kstring_t tmp = {0,0,0};
int prev_chr_id = -1, prev_pos;
bcf1_t *line = bcf_init();
for (i=0; i<args->nfnames; i++)
{
htsFile *fp = hts_open(args->fnames[i], "r"); if ( !fp ) error("Failed to open: %s\n", args->fnames[i]);
bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to parse header: %s\n", args->fnames[i]);
if ( !fp->is_bin && args->output_type&FT_VCF )
{
line->max_unpack = BCF_UN_STR;
// if VCF is on both input and output, avoid VCF to BCF conversion
while ( hts_getline(fp, KS_SEP_LINE, &fp->line) >=0 )
{
char *str = fp->line.s;
while ( *str && *str!='\t' ) str++;
tmp.l = 0;
kputsn(fp->line.s,str-fp->line.s,&tmp);
int chr_id = bcf_hdr_name2id(args->out_hdr, tmp.s);
if ( chr_id<0 ) error("The sequence \"%s\" not defined in the header: %s\n(Quick workaround: index the file.)\n", tmp.s, args->fnames[i]);
if ( prev_chr_id!=chr_id )
{
prev_pos = -1;
if ( args->seen_seq[chr_id] )
error("\nThe chromosome block %s is not contiguous, consider running with -a.\n", tmp.s);
}
char *end;
int pos = strtol(str+1,&end,10) - 1;
if ( end==str+1 ) error("Could not parse line: %s\n", fp->line.s);
if ( prev_pos > pos )
error("The chromosome block %s is not sorted, consider running with -a.\n", tmp.s);
args->seen_seq[chr_id] = 1;
prev_chr_id = chr_id;
if ( vcf_write_line(args->out_fh, &fp->line)!=0 ) error("Failed to write %d bytes\n", fp->line.l);
}
}
else
{
// BCF conversion is required
line->max_unpack = 0;
while ( bcf_read(fp, hdr, line)==0 )
{
bcf_translate(args->out_hdr, hdr, line);
if ( prev_chr_id!=line->rid )
{
prev_pos = -1;
if ( args->seen_seq[line->rid] )
error("\nThe chromosome block %s is not contiguous, consider running with -a.\n", bcf_seqname(args->out_hdr, line));
}
if ( prev_pos > line->pos )
error("The chromosome block %s is not sorted, consider running with -a.\n", bcf_seqname(args->out_hdr, line));
args->seen_seq[line->rid] = 1;
prev_chr_id = line->rid;
if ( bcf_write(args->out_fh, args->out_hdr, line)!=0 ) error("Failed to write\n");
}
}
bcf_hdr_destroy(hdr);
hts_close(fp);
}
bcf_destroy(line);
free(tmp.s);
}
}
//{{{ void get_bcf_query_result(uint32_t *mask,
void get_bcf_query_result(uint32_t *mask,
uint32_t mask_len,
struct gqt_query *q,
char **id_query_list,
uint32_t *id_lens,
uint32_t num_qs,
uint32_t num_fields,
char *vid_file_name,
char *bcf_file_name,
int bcf_output)
{
/* The VID file contains the line numbers of the variants after they have
* been sorted. To reach back into the BCF file to print the metadata
* associated with the variants marked in the mask, we need to create a
* sorted list of line numbers we want. So first we intersect the VID file
* and the mask, then sort it.
*/
/*
FILE *vid_f = fopen(vid_file_name, "rb");
if (!vid_f)
err(EX_NOINPUT, "Cannot read file\"%s\"", vid_file_name);
uint32_t *vids = (uint32_t *) malloc(num_fields*sizeof(uint32_t));
if (!vids )
err(EX_OSERR, "malloc error");
size_t fr = fread(vids, sizeof(uint32_t), num_fields, vid_f);
check_file_read(vid_file_name, vid_f, num_fields, fr);
fclose(vid_f);
*/
struct vid_file *vid_f = open_vid_file(vid_file_name);
load_vid_data(vid_f);
uint32_t i, j, masked_vid_count = 0;
for (i = 0; i < mask_len; ++i)
masked_vid_count += popcount(mask[i]);
uint32_t *masked_vids = (uint32_t *)
malloc(masked_vid_count*sizeof(uint32_t));
if (!masked_vids )
err(EX_OSERR, "malloc error");
uint32_t masked_vid_i = 0;
for (i = 0; i < mask_len; ++i) {
uint32_t bytes = mask[i];
if (bytes == 0)
continue; /* skip a bunch of ops if you can */
for (j = 0; j < 32; j++) {
if (bytes & (1 << (31 - j))) {
masked_vids[masked_vid_i] = vid_f->vids[i*32 + j];
masked_vid_i+=1;
}
}
if (masked_vid_i == masked_vid_count)
break;
}
destroy_vid_file(vid_f);
qsort(masked_vids, masked_vid_count, sizeof(uint32_t), compare_uint32_t);
htsFile *fp = hts_open(bcf_file_name,"rb");
bcf_hdr_t *hdr = bcf_hdr_read(fp);
bcf1_t *line = bcf_init1();
//bcf_hdr_set_samples(hdr, print_name_csv, 0);
htsFile *out;
if (!bcf_output)
out = hts_open("-", "w");
else
out = hts_open("-", "wb");
int r = bcf_hdr_write(out, hdr);
uint32_t bcf_line_i = 0;
masked_vid_i = 0;
while ( bcf_read(fp, hdr, line) != -1) {
if (masked_vids[masked_vid_i] == bcf_line_i) {
r = bcf_unpack(line, BCF_UN_ALL);
r = bcf_write1(out, hdr, line);
masked_vid_i+=1;
}
if (masked_vid_i == masked_vid_count)
break;
bcf_line_i += 1;
}
hts_close(out);
hts_close(fp);
}
int vcf_read(bcf_t *bp, bcf_hdr_t *h, bcf1_t *b)
{
int dret, k, i, sync = 0;
vcf_t *v = (vcf_t*)bp->v;
char *p, *q;
kstring_t str, rn;
ks_tokaux_t aux, a2;
if (!bp->is_vcf) return bcf_read(bp, h, b);
v->line.l = 0;
str.l = 0; str.m = b->m_str; str.s = b->str;
rn.l = rn.m = h->l_nm; rn.s = h->name;
if (ks_getuntil(v->ks, '\n', &v->line, &dret) < 0) return -1;
b->n_smpl = h->n_smpl;
for (p = kstrtok(v->line.s, "\t", &aux), k = 0; p; p = kstrtok(0, 0, &aux), ++k) {
*(char*)aux.p = 0;
if (k == 0) { // ref
int tid = bcf_str2id(v->refhash, p);
if (tid < 0) {
tid = bcf_str2id_add(v->refhash, strdup(p));
kputs(p, &rn); kputc('\0', &rn);
sync = 1;
}
b->tid = tid;
} else if (k == 1) { // pos
b->pos = atoi(p) - 1;
} else if (k == 5) { // qual
b->qual = (p[0] >= '0' && p[0] <= '9')? atof(p) : 0;
} else if (k <= 8) { // variable length strings
kputs(p, &str); kputc('\0', &str);
b->l_str = str.l; b->m_str = str.m; b->str = str.s;
if (k == 8) bcf_sync(b);
} else { // k > 9
if (strncmp(p, "./.", 3) == 0) {
for (i = 0; i < b->n_gi; ++i) {
if (b->gi[i].fmt == bcf_str2int("GT", 2)) {
((uint8_t*)b->gi[i].data)[k-9] = 1<<7;
} else if (b->gi[i].fmt == bcf_str2int("GQ", 2)) {
((uint8_t*)b->gi[i].data)[k-9] = 0;
} else if (b->gi[i].fmt == bcf_str2int("SP", 2)) {
((int32_t*)b->gi[i].data)[k-9] = 0;
} else if (b->gi[i].fmt == bcf_str2int("DP", 2)) {
((uint16_t*)b->gi[i].data)[k-9] = 0;
} else if (b->gi[i].fmt == bcf_str2int("PL", 2)) {
int y = b->n_alleles * (b->n_alleles + 1) / 2;
memset((uint8_t*)b->gi[i].data + (k - 9) * y, 0, y);
} else if (b->gi[i].fmt == bcf_str2int("GL", 2)) {
int y = b->n_alleles * (b->n_alleles + 1) / 2;
memset((float*)b->gi[i].data + (k - 9) * y, 0, y * 4);
}
}
goto endblock;
}
for (q = kstrtok(p, ":", &a2), i = 0; q && i < b->n_gi; q = kstrtok(0, 0, &a2), ++i) {
if (b->gi[i].fmt == bcf_str2int("GT", 2)) {
((uint8_t*)b->gi[i].data)[k-9] = (q[0] - '0')<<3 | (q[2] - '0') | (q[1] == '/'? 0 : 1) << 6;
} else if (b->gi[i].fmt == bcf_str2int("GQ", 2)) {
double _x = strtod(q, &q);
int x = (int)(_x + .499);
if (x > 255) x = 255;
((uint8_t*)b->gi[i].data)[k-9] = x;
} else if (b->gi[i].fmt == bcf_str2int("SP", 2)) {
int x = strtol(q, &q, 10);
if (x > 0xffff) x = 0xffff;
((uint32_t*)b->gi[i].data)[k-9] = x;
} else if (b->gi[i].fmt == bcf_str2int("DP", 2)) {
int x = strtol(q, &q, 10);
if (x > 0xffff) x = 0xffff;
((uint16_t*)b->gi[i].data)[k-9] = x;
} else if (b->gi[i].fmt == bcf_str2int("PL", 2)) {
int x, y, j;
uint8_t *data = (uint8_t*)b->gi[i].data;
y = b->n_alleles * (b->n_alleles + 1) / 2;
for (j = 0; j < y; ++j) {
x = strtol(q, &q, 10);
if (x > 255) x = 255;
data[(k-9) * y + j] = x;
++q;
}
} else if (b->gi[i].fmt == bcf_str2int("GL", 2)) {
int j, y;
float x, *data = (float*)b->gi[i].data;
y = b->n_alleles * (b->n_alleles + 1) / 2;
for (j = 0; j < y; ++j) {
x = strtod(q, &q);
data[(k-9) * y + j] = x > 0? -x/10. : x;
++q;
}
}
}
endblock: i = i;
}
}
h->l_nm = rn.l; h->name = rn.s;
if (sync) bcf_hdr_sync(h);
return v->line.l + 1;
}
int main(int argc, char **argv)
{
if (argc < 3) {
fprintf(stderr,"%s <bcf file> <num vars>\n", argv[0]);
return 1;
}
char *fname = argv[1];
uint32_t num_vars = atoi(argv[2]);
htsFile *fp = hts_open(fname,"rb");
bcf_hdr_t *hdr = bcf_hdr_read(fp);
bcf1_t *line = bcf_init1();
int32_t *gt_p = NULL;
uint32_t num_inds = bcf_hdr_nsamples(hdr);
int32_t i, j, k, ntmp = 0, int_i = 0, two_bit_i = 0, sum, t_sum = 0;
uint32_t num_ind_ints = 1 + ((num_inds - 1) / 16);
pri_queue q = priq_new(0);
priority p;
uint32_t *packed_ints = (uint32_t *) calloc(num_ind_ints,
sizeof(uint32_t));
FILE *gt_of = fopen("gt.tmp.packed","wb");
FILE *md_of = fopen("md.tmp.packed","w");
uint32_t *md_index = (uint32_t *) malloc(num_vars * sizeof(uint32_t));
uint32_t md_i = 0;
unsigned long t_bcf_read = 0,
t_bcf_dup = 0,
t_bcf_unpack = 0,
t_bcf_get_genotypes = 0,
t_bcf_hdr_nsamples = 0,
t_q = 0,
t_write = 0,
t_get_md = 0,
t_md_write = 0,
t_pack = 0;
for (i = 0; i < num_vars; ++i) {
sum = 0;
int_i = 0;
two_bit_i = 0;
int r = bcf_read(fp, hdr, line);
// Copy
bcf1_t *t_line = bcf_dup(line);
// Unpack
bcf_unpack(t_line, BCF_UN_ALL);
// Get metadata
size_t len = strlen(bcf_hdr_id2name(hdr, t_line->rid)) +
10 + // max length of pos
strlen(t_line->d.id) +
strlen(t_line->d.allele[0]) +
strlen(t_line->d.allele[1]) +
4; //tabs
char *md = (char *) malloc(len * sizeof(char));
sprintf(md, "%s\t%d\t%s\t%s\t%s",
bcf_hdr_id2name(hdr, t_line->rid),
t_line->pos + 1,
t_line->d.id,
t_line->d.allele[0],
t_line->d.allele[1]);
// Write metadata
md_i += strlen(md);
md_index[i] = md_i;
fprintf(md_of, "%s", md);
// Get gentotypes
uint32_t num_gts_per_sample = bcf_get_genotypes(hdr,
t_line,
>_p,
&ntmp);
num_gts_per_sample /= num_inds;
int32_t *gt_i = gt_p;
// Pack genotypes
for (j = 0; j < num_inds; ++j) {
uint32_t gt = 0;
for (k = 0; k < num_gts_per_sample; ++k) {
gt += bcf_gt_allele(gt_i[k]);
}
packed_ints[int_i] += gt << (30 - 2*two_bit_i);
two_bit_i += 1;
if (two_bit_i == 16) {
two_bit_i = 0;
int_i += 1;
}
sum += gt;
gt_i += num_gts_per_sample;
}
// Get a priority for the variant based on the sum and number of
// leading zeros
p.sum = sum;
uint32_t prefix_len = 0;
j = 0;
while ((j < num_ind_ints) && (packed_ints[j] == 0)){
prefix_len += 32;
j += 1;
}
if (j < num_ind_ints)
prefix_len += nlz1(packed_ints[j]);
// Push it into the q
p.len = prefix_len;
int *j = (int *) malloc (sizeof(int));
j[0] = i;
priq_push(q, j, p);
// Write to file
fwrite(packed_ints, sizeof(uint32_t), num_ind_ints,gt_of);
memset(packed_ints, 0, num_ind_ints*sizeof(uint32_t));
t_sum += sum;
bcf_destroy(t_line);
free(md);
}
fclose(gt_of);
fclose(md_of);
md_of = fopen("md.tmp.packed","r");
FILE *md_out = fopen("md.bim","w");
gt_of = fopen("gt.tmp.packed","rb");
FILE *s_gt_of = fopen("s.gt.tmp.packed","wb");
// Get variants in order and rewrite a variant-major sorted matrix
while ( priq_top(q, &p) != NULL ) {
int *d = priq_pop(q, &p);
uint32_t start = 0;
if (*d != 0)
start = md_index[*d - 1];
uint32_t len = md_index[*d] - start;
fseek(md_of, start*sizeof(char), SEEK_SET);
char buf[len+1];
fread(buf, sizeof(char), len, md_of);
buf[len] = '\0';
fseek(gt_of, (*d)*num_ind_ints*sizeof(uint32_t), SEEK_SET);
fread(packed_ints, sizeof(uint32_t), num_ind_ints, gt_of);
fwrite(packed_ints, sizeof(uint32_t), num_ind_ints,s_gt_of);
fprintf(md_out, "%s\n", buf);
}
fclose(md_out);
fclose(md_of);
fclose(gt_of);
fclose(s_gt_of);
/*
* In a packed-int variant-major matrix there will be a num_vars
* number of rows, and a num_inds number of values packed into
* num_inds_ints number of intergers. For examples, 16 rows of 16 values
* will be 16 ints, where each int encodes 16 values.
*
*/
uint32_t num_var_ints = 1 + ((num_vars - 1) / 16);
uint32_t *I_data = (uint32_t *)calloc(num_var_ints*16,sizeof(uint32_t));
uint32_t **I = (uint32_t **)malloc(16*sizeof(uint32_t*));
for (i = 0; i < 16; ++i)
I[i] = I_data + i*num_var_ints;
uint32_t I_i = 0, I_int_i = 0;
uint32_t v;
s_gt_of = fopen("s.gt.tmp.packed","rb");
FILE *rs_gt_of = fopen("r.s.gt.tmp.packed","wb");
// Write these to values to that this is a well-formed uncompressed
// packed int binary file (ubin) file
fwrite(&num_vars, sizeof(uint32_t), 1, rs_gt_of);
fwrite(&num_inds, sizeof(uint32_t), 1, rs_gt_of);
/*
* we need to loop over the columns in the v-major file.
* There are num_vars rows, and num_ind_ints 16-ind packed columns
*
* In this loop :
* i: cols in var-major form, rows in ind-major form
* j: rows in var-major form, cols in ind-major form
*/
uint32_t num_inds_to_write = num_inds;
for (i = 0; i < num_ind_ints; ++i) { // loop over each int col
for (j = 0; j < num_vars; ++j) { // loop over head row in that col
// skip to the value at the row/col
fseek(s_gt_of,
j*num_ind_ints*sizeof(uint32_t) + //row
i*sizeof(uint32_t), //col
SEEK_SET);
fread(&v, sizeof(uint32_t), 1, s_gt_of);
// one int corresponds to a col of 16 two-bit values
// two_bit_i will move across the cols
for (two_bit_i = 0; two_bit_i < 16; ++two_bit_i) {
I[two_bit_i][I_i] += ((v >> (30 - 2*two_bit_i)) & 3) <<
(30 - 2*I_int_i);
}
I_int_i += 1;
if (I_int_i == 16) {
I_i += 1;
I_int_i = 0;
}
}
// When we are at the end of the file, and the number of lines
// is not a factor of 16, only write out the lines that contain values
if (num_inds_to_write >= 16) {
fwrite(I_data,
sizeof(uint32_t),
num_var_ints*16,
rs_gt_of);
num_inds_to_write -= 16;
} else {
fwrite(I_data,
sizeof(uint32_t),
num_var_ints*num_inds_to_write,
rs_gt_of);
}
memset(I_data, 0, num_var_ints*16*sizeof(uint32_t));
I_int_i = 0;
I_i = 0;
}
fclose(s_gt_of);
fclose(rs_gt_of);
free(md_index);
free(packed_ints);
}
static void reheader_bcf(args_t *args, int is_compressed)
{
htsFile *fp = hts_open(args->fname, "r"); if ( !fp ) error("Failed to open: %s\n", args->fname);
bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to read the header: %s\n", args->fname);
kstring_t htxt = {0,0,0};
int hlen;
htxt.s = bcf_hdr_fmt_text(hdr, 1, &hlen);
htxt.l = hlen;
int i, nsamples = 0;
char **samples = NULL;
if ( args->samples_fname )
samples = hts_readlines(args->samples_fname, &nsamples);
if ( args->header_fname )
{
free(htxt.s); htxt.s = NULL; htxt.l = htxt.m = 0;
read_header_file(args->header_fname, &htxt);
}
if ( samples )
{
set_samples(samples, nsamples, &htxt);
for (i=0; i<nsamples; i++) free(samples[i]);
free(samples);
}
bcf_hdr_t *hdr_out = bcf_hdr_init("r");
bcf_hdr_parse(hdr_out, htxt.s);
if ( args->header_fname ) hdr_out = strip_header(hdr, hdr_out);
// write the header and the body
htsFile *fp_out = hts_open("-",is_compressed ? "wb" : "wbu");
bcf_hdr_write(fp_out, hdr_out);
bcf1_t *rec = bcf_init();
while ( bcf_read(fp, hdr, rec)==0 )
{
// sanity checking, this slows things down. Make it optional?
bcf_unpack(rec, BCF_UN_ALL);
if ( rec->rid >= hdr_out->n[BCF_DT_CTG] || strcmp(bcf_hdr_int2id(hdr,BCF_DT_CTG,rec->rid),bcf_hdr_int2id(hdr_out,BCF_DT_CTG,rec->rid)) )
error("The CHROM is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_CTG,rec->rid));
for (i=0; i<rec->d.n_flt; i++)
{
int id = rec->d.flt[i];
if ( id >= hdr_out->n[BCF_DT_ID] ) break;
if ( !bcf_hdr_idinfo_exists(hdr_out,BCF_HL_FLT,id) ) break;
if ( strcmp(hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key) )
error("FIXME: Broken FILTER ids: %s vs %s\n", hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key);
}
if ( i!=rec->d.n_flt )
error("The FILTER is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_ID,rec->d.flt[i]));
for (i=0; i<rec->n_info; i++)
{
int id = rec->d.info[i].key;
if ( id >= hdr_out->n[BCF_DT_ID] ) break;
if ( !hdr_out->id[BCF_DT_ID][id].key ) break;
if ( !bcf_hdr_idinfo_exists(hdr_out,BCF_HL_INFO,id) ) break;
if ( strcmp(hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key) )
error("FIXME: Broken INFO ids: %s vs %s\n", hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key);
}
if ( i!=rec->n_info )
error("The INFO tag is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_ID,rec->d.info[i].key));
for (i=0; i<rec->n_fmt; i++)
{
int id = rec->d.fmt[i].id;
if ( id >= hdr_out->n[BCF_DT_ID] ) break;
if ( !hdr_out->id[BCF_DT_ID][id].key ) break;
if ( !bcf_hdr_idinfo_exists(hdr_out,BCF_HL_FMT,id) ) break;
if ( strcmp(hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key) )
error("FIXME: Broken FORMAT ids: %s vs %s\n", hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key);
}
if ( i!=rec->n_fmt )
error("The FORMAT tag is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_ID,rec->d.fmt[i].id));
bcf_write(fp_out,hdr_out,rec);
}
bcf_destroy(rec);
free(htxt.s);
hts_close(fp_out);
hts_close(fp);
bcf_hdr_destroy(hdr_out);
bcf_hdr_destroy(hdr);
}
int main(int argc, char **argv)
{
if ( argc == 1 )
error("Usage : bed_annos -c config.json -O z -o output.vcf.gz input.vcf.gz");
int i;
for ( i = 1; i < argc; ) {
const char *a = argv[i++];
const char **var = 0;
if ( strcmp(a, "-c") == 0 )
var = &json_fname;
else if ( strcmp(a, "-O") == 0 )
var = &output_fname_type;
else if ( strcmp(a, "-o") == 0 )
var = &output_fname;
if ( var != 0 ) {
if ( i == argc )
error("Missing an argument after %s", a);
*var = argv[i++];
continue;
}
if ( input_fname == 0 ) {
input_fname = a;
continue;
}
error("Unknown argument : %s.", a);
}
struct vcfanno_config *con = vcfanno_config_init();
if ( vcfanno_load_config(con, json_fname) != 0 )
error("Failed to load configure file. %s : %s", json_fname, strerror(errno));
vcfanno_config_debug(con);
if ( con->beds.n_beds == 0)
error("No bed database specified.");
if ( input_fname == 0 && (!isatty(fileno(stdin))) )
input_fname = "-";
if ( input_fname == 0 )
error("No input file.");
int out_type = FT_VCF;
if ( output_fname_type != 0 ) {
switch (output_fname_type[0]) {
case 'b':
out_type = FT_BCF_GZ; break;
case 'u':
out_type = FT_BCF; break;
case 'z':
out_type = FT_VCF_GZ; break;
case 'v':
out_type = FT_VCF; break;
default :
error("The output type \"%d\" not recognised\n", out_type);
};
}
htsFile *fp = hts_open(input_fname, "r");
if ( fp == NULL )
error("Failed to open %s : %s.", input_fname, strerror(errno));
htsFormat type = *hts_get_format(fp);
if ( type.format != vcf && type.format != bcf )
error("Unsupported input format. %s", input_fname);
bcf_hdr_t *hdr = bcf_hdr_read(fp);
if ( hdr == NULL )
error("Failed to parse header.");
bcf_hdr_t *hdr_out = bcf_hdr_dup(hdr);
htsFile *fout = output_fname == 0 ? hts_open("-", hts_bcf_wmode(out_type)) : hts_open(output_fname, hts_bcf_wmode(out_type));
struct beds_options opts = { .beds_is_inited = 0,};
beds_options_init(&opts);
opts.hdr_out = hdr_out;
for ( i = 0; i < con->beds.n_beds; ++i ) {
beds_database_add(&opts, con->beds.files[i].fname, con->beds.files[i].columns);
}
bcf_hdr_write(fout, hdr_out);
bcf1_t *line = bcf_init();
while ( bcf_read(fp, hdr, line) == 0 ) {
anno_beds_core(&opts, line);
bcf_write(fout, hdr_out, line);
}
bcf_destroy(line);
bcf_hdr_destroy(hdr);
bcf_hdr_destroy(hdr_out);
beds_options_destroy(&opts);
hts_close(fp);
hts_close(fout);
return 0;
}
/**
* Gets records for the most recent position and fills up the buffer from file i.
* returns true if buffer is filled or it is not necessary to fill buffer.
* returns false if no more records are found to fill buffer
*/
void BCFSyncedReader::fill_buffer(int32_t i)
{
if (buffer[i].size()>=2)
return;
if (random_access)
{
int32_t pos1 = buffer[i].size()==0 ? 0 : bcf_get_pos1(buffer[i].front());
if (ftypes[i].format==bcf)
{
bcf1_t *v = get_bcf1_from_pool();
bool populated = false;
while (itrs[i] && bcf_itr_next(files[i], itrs[i], v)>=0)
{
populated = true;
bcf_unpack(v, BCF_UN_STR);
//check to ensure order
if (!buffer[i].empty())
{
if (!bcf_is_in_order(buffer[i].back(), v))
{
fprintf(stderr, "[E:%s:%d %s] VCF file not in order: %s\n", __FILE__, __LINE__, __FUNCTION__, file_names[i].c_str());
exit(1);
}
}
buffer[i].push_back(v);
insert_into_pq(i, v);
if (pos1==0)
{
pos1 = bcf_get_pos1(v);
}
if (bcf_get_pos1(v)!=pos1)
{
break;
}
v = get_bcf1_from_pool();
populated = false;
}
if (!populated)
store_bcf1_into_pool(v);
}
else if (ftypes[i].format==vcf)
{
while (itrs[i] && tbx_itr_next(files[i], tbxs[i], itrs[i], &s)>=0)
{
bcf1_t *v = get_bcf1_from_pool();
vcf_parse(&s, hdrs[i], v);
bcf_unpack(v, BCF_UN_STR);
//check to ensure order
if (!buffer[i].empty())
{
if (!bcf_is_in_order(buffer[i].back(), v))
{
fprintf(stderr, "[E:%s:%d %s] VCF file not in order: %s\n", __FILE__, __LINE__, __FUNCTION__, file_names[i].c_str());
exit(1);
}
}
buffer[i].push_back(v);
insert_into_pq(i, v);
if (pos1==0)
{
pos1 = bcf_get_pos1(v);
}
if (bcf_get_pos1(v)!=pos1)
{
break;
}
}
}
}
else
{
int32_t rid = buffer[i].size()==0 ? -1 : bcf_get_rid(buffer[i].front());
int32_t pos1 = buffer[i].size()==0 ? 0 : bcf_get_pos1(buffer[i].front());
bcf1_t *v = get_bcf1_from_pool();
bool populated = false;
while (bcf_read(files[i], hdrs[i], v)>=0)
{
populated = true;
bcf_unpack(v, BCF_UN_STR);
//check to ensure order
if (!buffer[i].empty())
{
if (!bcf_is_in_order(buffer[i].back(), v))
{
fprintf(stderr, "[E:%s:%d %s] VCF file not in order: %s\n", __FILE__, __LINE__, __FUNCTION__, file_names[i].c_str());
exit(1);
}
}
buffer[i].push_back(v);
insert_into_pq(i, v);
if (rid==-1)
{
rid = bcf_get_rid(v);
pos1 = bcf_get_pos1(v);
}
if (bcf_get_rid(v)!=rid || bcf_get_pos1(v)!=pos1)
{
break;
}
v = get_bcf1_from_pool();
populated = false;
}
if (!populated)
store_bcf1_into_pool(v);
}
}
