/*!
@abstract Sort an unsorted BAM file based on the chromosome order
and the leftmost position of an alignment
@param is_by_qname whether to sort by query name
@param fn name of the file to be sorted
@param prefix prefix of the temporary files (prefix.NNNN.bam are written)
@param fnout name of the final output file to be written
@param modeout sam_open() mode to be used to create the final output file
@param max_mem approxiate maximum memory (very inaccurate)
@return 0 for successful sorting, negative on errors
@discussion It may create multiple temporary subalignment files
and then merge them by calling bam_merge_core(). This function is
NOT thread safe.
*/
int bam_sort_core_ext(int is_by_qname, const char *fn, const char *prefix, const char *fnout, const char *modeout, size_t _max_mem, int n_threads)
{
int ret, i, n_files = 0;
size_t mem, max_k, k, max_mem;
bam_hdr_t *header;
samFile *fp;
bam1_t *b, **buf;
if (n_threads < 2) n_threads = 1;
g_is_by_qname = is_by_qname;
max_k = k = 0; mem = 0;
max_mem = _max_mem * n_threads;
buf = NULL;
fp = sam_open(fn, "r");
if (fp == NULL) {
fprintf(pysamerr, "[bam_sort_core] fail to open file %s\n", fn);
return -1;
}
header = sam_hdr_read(fp);
if (is_by_qname) change_SO(header, "queryname");
else change_SO(header, "coordinate");
// write sub files
for (;;) {
if (k == max_k) {
size_t kk, old_max = max_k;
max_k = max_k? max_k<<1 : 0x10000;
buf = (bam1_t**)realloc(buf, max_k * sizeof(bam1_t*));
for (kk = old_max; kk < max_k; ++kk) buf[kk] = NULL;
}
if (buf[k] == NULL) buf[k] = bam_init1();
b = buf[k];
if ((ret = sam_read1(fp, header, b)) < 0) break;
if (b->l_data < b->m_data>>2) { // shrink
b->m_data = b->l_data;
kroundup32(b->m_data);
b->data = (uint8_t*)realloc(b->data, b->m_data);
}
mem += sizeof(bam1_t) + b->m_data + sizeof(void*) + sizeof(void*); // two sizeof(void*) for the data allocated to pointer arrays
++k;
if (mem >= max_mem) {
n_files = sort_blocks(n_files, k, buf, prefix, header, n_threads);
mem = k = 0;
}
}
if (ret != -1)
fprintf(pysamerr, "[bam_sort_core] truncated file. Continue anyway.\n");
// write the final output
if (n_files == 0) { // a single block
ks_mergesort(sort, k, buf, 0);
write_buffer(fnout, modeout, k, buf, header, n_threads);
} else { // then merge
char **fns;
n_files = sort_blocks(n_files, k, buf, prefix, header, n_threads);
fprintf(pysamerr, "[bam_sort_core] merging from %d files...\n", n_files);
fns = (char**)calloc(n_files, sizeof(char*));
for (i = 0; i < n_files; ++i) {
fns[i] = (char*)calloc(strlen(prefix) + 20, 1);
sprintf(fns[i], "%s.%.4d.bam", prefix, i);
}
if (bam_merge_core2(is_by_qname, fnout, modeout, NULL, n_files, fns, MERGE_COMBINE_RG|MERGE_COMBINE_PG, NULL, n_threads) < 0) {
// Propagate bam_merge_core2() failure; it has already emitted a
// message explaining the failure, so no further message is needed.
return -1;
}
for (i = 0; i < n_files; ++i) {
unlink(fns[i]);
free(fns[i]);
}
free(fns);
}
// free
for (k = 0; k < max_k; ++k) bam_destroy1(buf[k]);
free(buf);
bam_hdr_destroy(header);
sam_close(fp);
return 0;
}
/*!
@abstract Sort an unsorted BAM file based on the chromosome order
and the leftmost position of an alignment
@param is_by_qname whether to sort by query name
@param fn name of the file to be sorted
@param prefix prefix of the output and the temporary files; upon
sucessess, prefix.bam will be written.
@param max_mem approxiate maximum memory (very inaccurate)
@discussion It may create multiple temporary subalignment files
and then merge them by calling bam_merge_core(). This function is
NOT thread safe.
*/
void bam_sort_core_ext(int is_by_qname, const char *fn, const char *prefix, size_t _max_mem, int is_stdout, int n_threads, int level, int sort_type)
{
int ret, i, n_files = 0;
size_t mem, max_k, k, max_mem;
bam_header_t *header;
bamFile fp;
bam1_t *b, **buf;
char *fnout = 0;
if (n_threads < 2) n_threads = 1;
g_is_by_qname = is_by_qname;
max_k = k = 0; mem = 0;
max_mem = _max_mem * n_threads;
buf = 0;
fp = strcmp(fn, "-")? bam_open(fn, "r") : bam_dopen(fileno(stdin), "r");
if (fp == 0) {
fprintf(stderr, "[bam_sort_core] fail to open file %s\n", fn);
return;
}
header = bam_header_read(fp);
if (is_by_qname) change_SO(header, "queryname");
else change_SO(header, "coordinate");
// write sub files
for (;;) {
if (k == max_k) {
size_t old_max = max_k;
max_k = max_k? max_k<<1 : 0x10000;
buf = realloc(buf, max_k * sizeof(void*));
memset(buf + old_max, 0, sizeof(void*) * (max_k - old_max));
}
if (buf[k] == 0) buf[k] = (bam1_t*)calloc(1, sizeof(bam1_t));
b = buf[k];
if ((ret = bam_read1(fp, b)) < 0) break;
if (b->data_len < b->m_data>>2) { // shrink
b->m_data = b->data_len;
kroundup32(b->m_data);
b->data = realloc(b->data, b->m_data);
}
mem += sizeof(bam1_t) + b->m_data + sizeof(void*) + sizeof(void*); // two sizeof(void*) for the data allocated to pointer arrays
++k;
if (mem >= max_mem) {
n_files = sort_blocks(n_files, k, buf, prefix, header, n_threads, sort_type);
mem = k = 0;
}
}
if (ret != -1)
fprintf(stderr, "[bam_sort_core] truncated file. Continue anyway.\n");
// output file name
fnout = calloc(strlen(prefix) + 20, 1);
if (is_stdout) sprintf(fnout, "-");
else sprintf(fnout, "%s.bam", prefix);
// write the final output
if (n_files == 0) { // a single block
char mode[8];
strcpy(mode, "w");
if (level >= 0) sprintf(mode + 1, "%d", level < 9? level : 9);
sort_aux_core(k, buf, sort_type);
#ifndef _PBGZF_USE
write_buffer(fnout, mode, k, buf, header, n_threads);
#else
write_buffer(fnout, mode, k, buf, header);
#endif
} else { // then merge
char **fns;
n_files = sort_blocks(n_files, k, buf, prefix, header, n_threads, sort_type);
fprintf(stderr, "[bam_sort_core] merging from %d files...\n", n_files);
fns = (char**)calloc(n_files, sizeof(char*));
for (i = 0; i < n_files; ++i) {
fns[i] = (char*)calloc(strlen(prefix) + 20, 1);
sprintf(fns[i], "%s.%.4d.bam", prefix, i);
}
#ifndef _PBGZF_USE
bam_merge_core2(is_by_qname, fnout, 0, n_files, fns, 0, 0, n_threads, level);
#else
bam_merge_core2(is_by_qname, fnout, 0, n_files, fns, 0, 0, level);
#endif
for (i = 0; i < n_files; ++i) {
unlink(fns[i]);
free(fns[i]);
}
free(fns);
}
free(fnout);
// free
for (k = 0; k < max_k; ++k) {
if (!buf[k]) continue;
free(buf[k]->data);
free(buf[k]);
}
free(buf);
bam_header_destroy(header);
bam_close(fp);
}
/*!
@abstract Sort an unsorted BAM file based on the chromosome order
and the leftmost position of an alignment
@param is_by_qname whether to sort by query name
@param fn name of the file to be sorted
@param prefix prefix of the output and the temporary files; upon
sucessess, prefix.bam will be written.
@param max_mem approxiate maximum memory (very inaccurate)
@discussion It may create multiple temporary subalignment files
and then merge them by calling bam_merge_core(). This function is
NOT thread safe.
*/
void bam_sort_core_ext(int is_by_qname, const char *fn, const char *prefix, size_t max_mem, int is_stdout)
{
int n, ret, k, i;
size_t mem;
bam_header_t *header;
bamFile fp;
bam1_t *b, **buf;
g_is_by_qname = is_by_qname;
n = k = 0; mem = 0;
fp = strcmp(fn, "-")? bam_open(fn, "r") : bam_dopen(fileno(stdin), "r");
if (fp == 0) {
fprintf(stderr, "[bam_sort_core] fail to open file %s\n", fn);
return;
}
header = bam_header_read(fp);
if (is_by_qname) change_SO(header, "queryname");
else change_SO(header, "coordinate");
buf = (bam1_t**)calloc(max_mem / BAM_CORE_SIZE, sizeof(bam1_t*));
// write sub files
for (;;) {
if (buf[k] == 0) buf[k] = (bam1_t*)calloc(1, sizeof(bam1_t));
b = buf[k];
if ((ret = bam_read1(fp, b)) < 0) break;
mem += ret;
++k;
if (mem >= max_mem) {
sort_blocks(n++, k, buf, prefix, header, 0);
mem = 0; k = 0;
}
}
if (ret != -1)
fprintf(stderr, "[bam_sort_core] truncated file. Continue anyway.\n");
if (n == 0) sort_blocks(-1, k, buf, prefix, header, is_stdout);
else { // then merge
char **fns, *fnout;
fprintf(stderr, "[bam_sort_core] merging from %d files...\n", n+1);
sort_blocks(n++, k, buf, prefix, header, 0);
fnout = (char*)calloc(strlen(prefix) + 20, 1);
if (is_stdout) sprintf(fnout, "-");
else sprintf(fnout, "%s.bam", prefix);
fns = (char**)calloc(n, sizeof(char*));
for (i = 0; i < n; ++i) {
fns[i] = (char*)calloc(strlen(prefix) + 20, 1);
sprintf(fns[i], "%s.%.4d.bam", prefix, i);
}
bam_merge_core(is_by_qname, fnout, 0, n, fns, 0, 0);
free(fnout);
for (i = 0; i < n; ++i) {
unlink(fns[i]);
free(fns[i]);
}
free(fns);
}
for (k = 0; (size_t)k < max_mem / BAM_CORE_SIZE; ++k) {
if (buf[k]) {
free(buf[k]->data);
free(buf[k]);
}
}
free(buf);
bam_header_destroy(header);
bam_close(fp);
}