int main(int argc, char* argv[]) {
FILE* F;
MSA *msa;
int *msa_gap_patterns = NULL;
HMM *hmm = NULL;
TreeNode *tree = NULL;
int i, input_format = SS, msa_idx, quiet_mode = FALSE,
ncats, nmsas, ncats_unspooled, indel_nseqs = -1;
String *msa_fname, *gff_fname;
List *gff_fname_list = NULL, *msa_fname_list = NULL,
*msa_length_list = NULL, *model_indels_str = NULL;
Matrix *traincounts = NULL;
Vector *begcounts = NULL, *statecounts = NULL;
CategoryMap *cm = NULL;
char c;
GapPatternMap *gpm = NULL;
GFF_Set *gff;
char *reverse_groups_tag = NULL;
while ((c = getopt(argc, argv, "i:g:c:m:M:R:I:n:t:P:G:qh")) != -1) {
switch(c) {
case 'i':
input_format = msa_str_to_format(optarg);
if (input_format == -1)
die("ERROR: bad alignment format.\n");
break;
case 'g':
gff_fname_list = get_arg_list(optarg);
break;
case 'c':
cm = cm_new_string_or_file(optarg);
break;
case 'm':
msa_fname_list = get_arg_list(optarg);
break;
case 'M':
msa_length_list = str_list_as_int(get_arg_list(optarg));
break;
case 'R':
reverse_groups_tag = optarg;
break;
case 'I':
model_indels_str = get_arg_list(optarg);
break;
case 'n':
indel_nseqs = get_arg_int(optarg);
break;
case 't':
if (optarg[0] == '(') /* in this case, assume topology given
at command line */
tree = tr_new_from_string(optarg);
else
tree = tr_new_from_file(phast_fopen(optarg, "r"));
break;
case 'q':
quiet_mode = TRUE;
break;
case 'h':
print_usage();
exit(0);
case '?':
die("ERROR: unrecognized option.\n\nType 'hmm_train -h' for usage.\n");
}
}
if (msa_fname_list == NULL)
die("ERROR: -m required. Type 'hmm_train -h' for usage.\n");
if (gff_fname_list == NULL)
die("ERROR: -g required in training mode. Type 'hmm_train -h' for usage.\n");
if (msa_length_list != NULL && msa_fname_list != NULL)
die("ERROR: -m and -M are mutually exclusive. Type 'hmm_train -h' for usage.\n");
if (model_indels_str != NULL && tree == NULL)
die("ERROR: -I requires -t. Type 'hmm_train -h' for usage.\n");
if (cm == NULL)
die("ERROR: category map required.\n");
set_seed(-1);
ncats = cm->ncats + 1;
ncats_unspooled = cm->unspooler != NULL ? cm->unspooler->nstates_unspooled :
ncats;
nmsas = (msa_length_list != NULL ? lst_size(msa_length_list) :
lst_size(msa_fname_list));
if (model_indels_str != NULL) {
if (tree == NULL)
die("ERROR: tree is NULL\n"); /*FIXME: indel_ncats broken */
gpm = gp_create_gapcats(cm, model_indels_str, tree, FALSE);
ncats = cm->ncats + 1; /* numbers will change */
ncats_unspooled = cm->unspooler == NULL ? ncats :
cm->unspooler->nstates_unspooled;
}
/* allocate memory for storage of "training paths" */
traincounts = mat_new(ncats_unspooled, ncats_unspooled);
statecounts = vec_new(ncats_unspooled);
begcounts = vec_new(ncats_unspooled);
mat_zero(traincounts);
vec_zero(statecounts);
vec_zero(begcounts);
/* create skeleton of new HMM. */
hmm = hmm_new_nstates(ncats_unspooled, 0, 0);
/* Main loop: consider each MSA in turn */
for (msa_idx = 0; msa_idx < nmsas; msa_idx++) {
if (msa_fname_list != NULL) {
msa_fname = (String*)lst_get_ptr(msa_fname_list, msa_idx);
F = phast_fopen(msa_fname->chars, "r");
if (!quiet_mode)
fprintf(stderr, "Reading alignment from %s ...\n",
F == stdin ? "stdin" : msa_fname->chars);
msa = msa_new_from_file(F, NULL);
phast_fclose(F);
}
else { /* only lengths of alignments specified */
msa = msa_new(NULL, NULL, 0, lst_get_int(msa_length_list, msa_idx), NULL);
/* just a shell in this case */
}
gff_fname = (String*)lst_get_ptr(gff_fname_list, msa_idx);
if (!quiet_mode)
fprintf(stderr, "Reading annotations from %s ...\n", gff_fname->chars);
gff = gff_read_set(phast_fopen(gff_fname->chars, "r"));
/* convert GFF to coordinate frame of alignment */
if (msa_length_list == NULL) {
if (!quiet_mode)
fprintf(stderr, "Mapping annotations to alignment ...\n");
msa_map_gff_coords(msa, gff, 1, 0, 0); /* assume seq 1 is ref */
}
if (model_indels_str != NULL) {
if (!quiet_mode)
fprintf(stderr, "Obtaining gap patterns ...\n");
msa_gap_patterns = smalloc(msa->length * sizeof(int));
gp_set_phylo_patterns(gpm, msa_gap_patterns, msa);
}
/* at this point, we don't actually need the alignment anymore;
if using ordered suff stats (likely with large data sets),
can free them now, to avoid running out of memory */
if (msa->ss != NULL) {
ss_free(msa->ss);
msa->ss = NULL;
}
if (reverse_groups_tag != NULL) {
if (!quiet_mode)
fprintf(stderr, "Reverse complementing features on negative strand (group by '%s') ...\n",
reverse_groups_tag);
/* we don't need to reverse complement the whole alignment --
just the gff and possibly the gap pattern array (pass a
NULL msa) */
gff_group(gff, reverse_groups_tag);
msa_reverse_compl_feats(NULL, gff, msa_gap_patterns);
}
if (!quiet_mode)
fprintf(stderr, "Labeling sites by category ...\n");
msa_label_categories(msa, gff, cm);
gff_free_set(gff);
if (model_indels_str != NULL) {
if (!quiet_mode)
fprintf(stderr, "Remapping categories according to gap patterns ...\n");
if (indel_nseqs > 0 && indel_nseqs != msa->nseqs) {
/* in this case, we'll simply reassign non-trivial gap
patterns randomly. This will achieve the desired
effect with minimal coding, as long as the number of
sites is not too small (the indel model is probably
useless anyway if the number is small) */
int pat, newpat;
int npatterns = 4 * indel_nseqs - 5;
int complex_allowed[cm->ncats+1];
List *no_complex_names, *no_complex_nums;
if (!quiet_mode)
fprintf(stderr, "(target number of sequences: %d)\n", indel_nseqs);
/* set up index indicating by cat no. whether complex gaps
are allowed */
for (i = 0; i < ncats; i++) complex_allowed[i] = 1;
no_complex_names = lst_new_ptr(10);
str_split(str_new_charstr(NO_COMPLEX), ",", no_complex_names);
no_complex_nums = cm_get_category_list(cm, no_complex_names, 1);
for (i = 0; i < lst_size(no_complex_nums); i++)
complex_allowed[lst_get_int(no_complex_nums, i)] = 0;
lst_free(no_complex_nums);
lst_free_strings(no_complex_names);
lst_free(no_complex_names);
/* now reassign all non-null numbers */
for (i = 0; i < msa->length; ) {
if ((pat = msa_gap_patterns[i]) != 0) {
if (complex_allowed[msa->categories[i]])
newpat = 1 + ((double)npatterns * unif_rand());
/* random number in interval [1, npatterns] */
else
newpat = 1 + ((double)(npatterns-1) * unif_rand());
/* random number in interval [1,npatterns-1]
(excludes complex gap pattern) */
for (; i < msa->length && msa_gap_patterns[i] == pat; i++)
msa_gap_patterns[i] = newpat; /* change for whole sequence */
}
else i++;
}
}
/* obtain gapped category number for each site */
for (i = 0; i < msa->length; i++)
if (gpm->cat_x_pattern_to_gapcat[msa->categories[i]] != NULL)
msa->categories[i] = gpm->cat_x_pattern_to_gapcat[msa->categories[i]][msa_gap_patterns[i]];
}
if (!quiet_mode)
fprintf(stderr, "Unspooling categories ...\n");
cm_spooled_to_unspooled(cm, msa->categories, msa->length);
if (!quiet_mode)
fprintf(stderr, "Collecting training data ...\n");
hmm_train_update_counts(traincounts, statecounts, begcounts,
msa->categories, msa->length,
ncats_unspooled);
if (msa_gap_patterns != NULL) sfree(msa_gap_patterns);
msa_free(msa);
}
/* now train HMM, using cumulative data */
hmm_train_from_counts(hmm, traincounts, NULL, statecounts, NULL,
begcounts, NULL);
/* if modeling indels, adjust begin transitions so probability is
distributed among different "gap pattern" states that all
correspond to the same ungapped state (category); this helps
avoid problems that occur when training on a few large sequences
(e.g., whole chromosomes) and then testing on many shorter ones */
if (model_indels_str != NULL) {
double tprob[gpm->ncats];
int nst[gpm->ncats]; /* total prob and number of states per
spooled, ungapped category */
for (i = 0; i < gpm->ncats; i++) tprob[i] = nst[i] = 0;
for (i = 0; i < hmm->nstates; i++) {
if (vec_get(hmm->begin_transitions, i) > 0)
/* have to go from unspooled space to spooled space, then to
ungapped space (HMM states correspond to unspooled,
gapped categories). Note that states with nonzero begin
probs shouldn't be conditioned on other states. */
tprob[gpm->gapcat_to_cat[cm_unspooled_to_spooled_cat(cm, i)]] +=
vec_get(hmm->begin_transitions, i);
nst[gpm->gapcat_to_cat[cm_unspooled_to_spooled_cat(cm, i)]]++;
}
for (i = 0; i < hmm->nstates; i++)
if (tprob[gpm->gapcat_to_cat[cm_unspooled_to_spooled_cat(cm, i)]] > 0)
vec_set(hmm->begin_transitions, i,
tprob[gpm->gapcat_to_cat[cm_unspooled_to_spooled_cat(cm, i)]] /
nst[gpm->gapcat_to_cat[cm_unspooled_to_spooled_cat(cm, i)]]);
/* (uniform prior) */
}
/* write trained HMM */
hmm_print(stdout, hmm);
if (!quiet_mode) fprintf(stderr, "Done.\n");
return 0;
}
/* checks to see if reference sequence looks okay wrt a given
list of features */
int ref_seq_okay(List *features, MSA *msa, int offset3,
int indel_strict, int splice_strict, List *problems) {
List *signals = NULL;
char *seq = NULL;
int seqalloc = 0;
int idx, retval = TRUE;
GFF_Feature *feat, *lastfeat_helper = NULL;
if (indel_strict) {
signals = lst_new_ptr(10);
str_split(str_new_charstr(SIGNALS), ",", signals);
}
for (idx = 0; idx < lst_size(features); idx++) {
int i, j, len, has_gaps = 0;
feat = lst_get_ptr(features, idx);
if (seqalloc <= feat->end - feat->start + 2) {
seqalloc = (feat->end - feat->start) * 2;
seq = srealloc(seq, seqalloc * sizeof(char));
}
for (i = feat->start - 1, len = 0; i < feat->end; i++) {
if (ss_get_char_pos(msa, i, 0, 0) != GAP_CHAR)
seq[len++] = ss_get_char_pos(msa, i, 0, 0);
else if (!has_gaps) has_gaps = 1;
}
seq[len] = '\0';
if (feat->strand == '-') msa_reverse_compl_seq(seq, len);
if (str_equals_charstr(feat->feature, GFF_START_TYPE) && strcmp(seq, "ATG") != 0) {
problem_add(problems, feat, BAD_REF_START, -1, -1);
retval = FALSE;
}
else if (str_equals_charstr(feat->feature, GFF_STOP_TYPE) &&
(feat->frame != 0 || !is_stop_codon(seq))) {
problem_add(problems, feat, BAD_REF_STOP, -1, -1);
retval = FALSE;
}
else if (str_starts_with_charstr(feat->feature, SPLICE_5) &&
!is_valid_5splice(seq, splice_strict)) {
problem_add(problems, feat, BAD_REF_5_SPLICE, -1, -1);
retval = FALSE;
}
else if (str_starts_with_charstr(feat->feature, SPLICE_3) &&
!is_valid_3splice(&seq[offset3], splice_strict)) {
problem_add(problems, feat, BAD_REF_3_SPLICE, -1, -1);
retval = FALSE;
}
else if (str_equals_charstr(feat->feature, GFF_CDS_TYPE)) {
for (i = (3 - feat->frame) % 3; i <= len - 3; i += 3) {
if (is_stop_codon(&seq[i])) {
problem_add(problems, feat, BAD_REF_ORF, -1, -1);
retval = FALSE;
break;
}
}
}
if (indel_strict) {
int strict_okay = TRUE;
List *signals = lst_new_ptr(10);
str_split(str_new_charstr(SIGNALS), ",", signals);
if (str_in_list(feat->feature, signals)) {
/* reject any signal feature with gaps in the ref seq, unless they
appear in a non-critical part of a splice site or in a
"prestart" feature */
if (has_gaps) {
if (str_starts_with_charstr(feat->feature, SPLICE_5)) {
if (ss_get_char_pos(msa, feat->start-1, 0, 0) == GAP_CHAR ||
ss_get_char_pos(msa, feat->start, 0, 0) == GAP_CHAR)
strict_okay = FALSE;
}
else if (str_starts_with_charstr(feat->feature, SPLICE_3)) {
if (ss_get_char_pos(msa, feat->end-1, 0, 0) == GAP_CHAR ||
ss_get_char_pos(msa, feat->end-2, 0, 0) == GAP_CHAR)
strict_okay = FALSE;
}
else if (!str_equals_charstr(feat->feature, "prestart"))
strict_okay = FALSE;
}
/* in addition, if two signals occur consec. with gaps and
only gaps between them, assume a violation of
--indel-strict */
if (lastfeat_helper != NULL && lastfeat_helper->end < feat->start-1) {
int allgaps = 1;
for (j = lastfeat_helper->end; allgaps && j < feat->start-1; j++)
/* note indexing: -1+1 for end and -1
for start */
if (ss_get_char_pos(msa, j, 0, 0) != GAP_CHAR) allgaps = 0;
if (allgaps)
strict_okay = FALSE;
}
lastfeat_helper = feat;
}
else lastfeat_helper = NULL;
/* also exclude CDS exons of length less than 6 in indel_strict
case -- these cause problems in exoniphy training because
start_codon is adjacent to cds5ss */
if (str_equals_charstr(feat->feature, GFF_CDS_TYPE) && len <= 6)
strict_okay = FALSE;
if (!strict_okay) {
problem_add(problems, feat, BAD_REF_INDEL_STRICT_FAIL, -1, -1);
retval = FALSE;
}
lst_free_strings(signals);
lst_free(signals);
}
}
if (seq != NULL) sfree(seq);
return retval;
}
int main(int argc, char *argv[]) {
char c;
int opt_idx;
GFF_Set *gff;
List *include = NULL;
char *groupby = "transcript_id", *exongroup_tag = NULL;
int unique = FALSE, sort = FALSE, simplebed = FALSE, fix_start_stop = FALSE,
add_utrs = FALSE, add_introns = FALSE, add_signals = FALSE;
enum {GFF, BED, GENEPRED, WIG} output_format = GFF;
FILE *discards_f = NULL, *groups_f = NULL;
struct option long_opts[] = {
{"output", 1, 0, 'o'},
{"include-only", 1, 0, 'i'},
{"include-groups", 1, 0, 'l'},
{"groupby", 1, 0, 'g'},
{"exongroup", 1, 0, 'e'},
{"add-utrs", 0, 0, 'U'},
{"add-introns", 0, 0, 'I'},
{"add-signals", 0, 0, 'S'},
{"fix-start-stop", 0, 0, 'f'},
{"unique", 0, 0, 'u'},
{"sort", 0, 0, 's'},
{"simplebed", 0, 0, 'b'},
{"discards", 1, 0, 'd'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
while ((c = (char)getopt_long(argc, argv, "o:i:l:g:e:d:UISfusbh", long_opts, &opt_idx)) != -1) {
switch (c) {
case 'o':
if (!strcmp("bed", optarg)) output_format = BED;
else if (!strcmp("genepred", optarg)) output_format = GENEPRED;
else if (!strcmp("wig", optarg)) output_format = WIG;
else if (strcmp("gff", optarg)) die("ERROR: bad output format.\n");
break;
case 'i':
include = get_arg_list(optarg);
break;
case 'l':
groups_f = phast_fopen(optarg, "r");
break;
case 'g':
groupby = optarg;
break;
case 'e':
exongroup_tag = optarg;
break;
case 'U':
add_utrs = TRUE;
break;
case 'I':
add_introns = TRUE;
break;
case 'S':
add_signals = TRUE;
break;
case 'f':
fix_start_stop = TRUE;
break;
case 'u':
unique = TRUE;
break;
case 'b':
simplebed = TRUE;
output_format = BED;
break;
case 'd':
discards_f = phast_fopen(optarg, "w+");
break;
case 's':
sort = TRUE;
break;
case 'h':
usage(argv[0]);
case '?':
die("Bad argument. Try '%s -h'.\n", argv[0]);
}
}
if (optind != argc - 1)
die("Input filename required. Try '%s -h'.\n", argv[0]);
set_seed(-1);
gff = gff_read_set(phast_fopen(argv[optind], "r"));
if (lst_size(gff->features) == 0) exit(0); /* helps avoid unexpected
behavior below */
/* filter by type */
if (include != NULL) gff_filter_by_type(gff, include, FALSE, discards_f);
/* group */
gff_group(gff, groupby);
/* utrs, introns, & signals */
if (add_utrs) gff_create_utrs(gff);
if (add_introns) gff_create_introns(gff);
if (add_signals) gff_create_signals(gff);
/* subgroup */
if (exongroup_tag != NULL) gff_exon_group(gff, exongroup_tag);
/* filter by group */
if (groups_f != NULL) {
String *s = str_new(STR_LONG_LEN);
List *groups = lst_new_ptr(10000);
str_slurp(s, groups_f);
str_split(s, NULL, groups);
gff_filter_by_group(gff, groups);
lst_free_strings(groups); lst_free(groups);
str_free(s);
}
/* sort */
if (sort) gff_sort(gff);
/* make unique */
if (unique) gff_remove_overlaps(gff, discards_f);
if (fix_start_stop) gff_fix_start_stop(gff);
if (output_format == BED)
gff_print_bed(stdout, gff, !simplebed);
else if (output_format == GENEPRED)
gff_print_genepred(stdout, gff);
else if (output_format == WIG)
wig_print(stdout, gff);
else
gff_print_set(stdout, gff);
gff_free_set(gff);
return 0;
}
int main(int argc, char *argv[]) {
char *msa_fname = NULL, *alph = "ACGT";
msa_format_type input_format = UNKNOWN_FORMAT;
char c;
int opt_idx, seed=-1;
String *optstr;
List *tmplist = NULL;
struct phyloFit_struct *pf;
FILE *infile;
struct option long_opts[] = {
{"msa", 1, 0, 'm'},
{"tree", 1, 0, 't'},
{"subst-mod", 1, 0, 's'},
{"msa-format", 1, 0, 'i'},
{"nrates", 1, 0, 'k'},
{"alpha", 1, 0, 'a'},
{"features", 1, 0, 'g'},
{"catmap", 1, 0, 'c'},
{"log", 1, 0, 'l'},
{"out-root", 1, 0, 'o'},
{"EM", 0, 0, 'E'},
{"error", 1, 0, 'e'},
{"precision", 1, 0, 'p'},
{"do-cats", 1, 0, 'C'},
{"non-overlapping", 0, 0, 'V'},
{"markov", 0, 0, 'N'},
{"reverse-groups", 1, 0, 'R'},
{"init-model", 1, 0, 'M'},
{"init-random", 0, 0, 'r'},
{"init-parsimony", 0, 0, 'y'},
{"print-parsimony", 1, 0, 'Y'},
{"lnl", 0, 0, 'L'},
{"scale-only", 0, 0, 'B'},
{"scale-subtree", 1, 0, 'S'},
{"estimate-freqs", 0, 0, 'F'},
{"sym-freqs", 0, 0, 'W'},
{"no-freqs", 0, 0, 'f'},
{"no-rates", 0, 0, 'n'},
{"no-opt", 1, 0, 'O'},
{"min-informative", 1, 0, 'I'},
{"gaps-as-bases", 0, 0, 'G'},
{"quiet", 0, 0, 'q'},
{"help", 0, 0, 'h'},
{"windows", 1, 0, 'w'},
{"windows-explicit", 1, 0, 'v'},
{"ancestor", 1, 0, 'A'},
{"post-probs", 0, 0, 'P'},
{"expected-subs", 0, 0, 'X'},
{"expected-total-subs", 0, 0, 'Z'},
{"expected-subs-col", 0, 0, 'J'},
{"column-probs", 0, 0, 'U'},
{"rate-constants", 1, 0, 'K'},
{"ignore-branches", 1, 0, 'b'},
{"clock", 0, 0, 'z'},
{"alt-model", 1, 0, 'd'},
{"label-branches", 1, 0, 0},
{"label-subtree", 1, 0, 0},
{"selection", 1, 0, 0},
{"bound", 1, 0, 'u'},
{"seed", 1, 0, 'D'},
{0, 0, 0, 0}
};
// NOTE: remaining shortcuts left: HjQx
pf = phyloFit_struct_new(0);
while ((c = (char)getopt_long(argc, argv, "m:t:s:g:c:C:i:o:k:a:l:w:v:M:p:A:I:K:S:b:d:O:u:Y:e:D:GVENRqLPXZUBFfnrzhWyJ", long_opts, &opt_idx)) != -1) {
switch(c) {
case 'm':
msa_fname = optarg;
break;
case 't':
if (optarg[0] == '(') /* in this case, assume topology given
at command line */
pf->tree = tr_new_from_string(optarg);
else
pf->tree = tr_new_from_file(phast_fopen(optarg, "r"));
break;
case 's':
pf->subst_mod = tm_get_subst_mod_type(optarg);
if (pf->subst_mod == UNDEF_MOD)
die("ERROR: illegal substitution model. Type \"phyloFit -h\" for usage.\n");
break;
case 'g':
pf->gff = gff_read_set(phast_fopen(optarg, "r"));
break;
case 'c':
pf->cm = cm_new_string_or_file(optarg);
break;
case 'C':
pf->cats_to_do_str = get_arg_list(optarg);
break;
case 'V':
pf->nonoverlapping = TRUE;
break;
case 'o':
pf->output_fname_root = optarg;
break;
case 'k':
pf->nratecats = get_arg_int_bounds(optarg, 0, INFTY);
break;
case 'a':
pf->alpha = get_arg_dbl(optarg);
break;
case 'R':
pf->reverse_group_tag = optarg;
break;
case 'i':
input_format = msa_str_to_format(optarg);
if (input_format == UNKNOWN_FORMAT)
die("ERROR: unrecognized alignment format. Type 'phyloFit -h' for usage.\n");
break;
case 'l':
if (!strcmp(optarg, "-"))
pf->logf = stderr;
else pf->logf = phast_fopen(optarg, "w+");
break;
case 'N':
pf->use_conditionals = 1;
break;
case 'w':
tmplist = get_arg_list(optarg);
if (lst_size(tmplist) != 2 ||
str_as_int(lst_get_ptr(tmplist, 0), &(pf->window_size)) != 0 ||
str_as_int(lst_get_ptr(tmplist, 1), &(pf->window_shift)) != 0)
die("ERROR: illegal arguments to --windows.\n");
lst_free_strings(tmplist);
lst_free(tmplist);
break;
case 'v':
tmplist = get_arg_list(optarg);
if (lst_size(tmplist) % 2 != 0)
die("ERROR: argument to --windows-explicit must be a list of even length.\n");
pf->window_coords = str_list_as_int(tmplist);
lst_free(tmplist);
break;
case 'E':
pf->use_em = TRUE;
break;
case 'e':
pf->error_fname=optarg;
break;
case 'p':
if (!strcmp(optarg, "LOW")) pf->precision = OPT_LOW_PREC;
else if (!strcmp(optarg, "MED")) pf->precision = OPT_MED_PREC;
else if (!strcmp(optarg, "HIGH")) pf->precision = OPT_HIGH_PREC;
else if (!strcmp(optarg, "VERY_HIGH")) pf->precision = OPT_VERY_HIGH_PREC;
else die("ERROR: --precision must be LOW, MED, or HIGH.\n\n");
break;
case 'M':
pf->input_mod = tm_new_from_file(phast_fopen(optarg, "r"), 1);
break;
case 'r':
pf->random_init = TRUE;
break;
case 'y':
pf->init_parsimony = TRUE;
break;
case 'Y':
pf->init_parsimony = TRUE;
pf->parsimony_cost_fname = optarg;
pf->parsimony_only=TRUE;
break;
case 'L':
pf->likelihood_only = TRUE;
break;
case 'q':
pf->quiet = TRUE;
break;
case 'P':
pf->do_bases = TRUE;
break;
case 'X':
pf->do_expected_nsubst = TRUE;
break;
case 'Z':
pf->do_expected_nsubst_tot = TRUE;
break;
case 'J':
pf->do_expected_nsubst_col = TRUE;
break;
case 'U':
pf->likelihood_only = TRUE; /* force -L */
pf->nsites_threshold = 0; /* also force this; typical use is
with small number of tuples, no
tuple_idx */
pf->do_column_probs = TRUE;
break;
case 'A':
pf->root_seqname = optarg;
break;
case 'I':
pf->nsites_threshold = get_arg_int(optarg);
break;
case 'G':
pf->gaps_as_bases = TRUE;
alph = "ACGT-";
break;
case 'B':
pf->estimate_scale_only = TRUE;
break;
case 'S':
pf->subtree_name = optarg;
break;
case 'F':
pf->estimate_backgd = TRUE;
break;
case 'W':
pf->estimate_backgd = TRUE;
pf->symfreq = TRUE;
break;
case 'f':
pf->no_freqs = TRUE;
break;
case 'n':
pf->no_rates = TRUE;
break;
case 'K':
tmplist = get_arg_list(optarg);
pf->rate_consts = str_list_as_dbl(tmplist);
pf->nratecats = lst_size(pf->rate_consts);
pf->use_em = 1;
lst_free_strings(tmplist); lst_free(tmplist);
break;
case 'b':
pf->ignore_branches = get_arg_list(optarg);
break;
case 'z':
pf->assume_clock = TRUE;
break;
case 'O':
if (pf->nooptstr == NULL)
pf->nooptstr = str_new_charstr(optarg);
else die("ERROR: no-opt argument can only be used once! parameters can be comma-separated list.");
break;
case 'd':
if (pf->alt_mod_str == NULL) {
pf->alt_mod_str = lst_new_ptr(1);
}
optstr = str_new_charstr(optarg);
lst_push_ptr(pf->alt_mod_str, optstr);
break;
case 0:
if (strcmp(long_opts[opt_idx].name, "label-branches") == 0 ||
strcmp(long_opts[opt_idx].name, "label-subtree") == 0) {
optstr = str_new_charstr(optarg);
if (pf->label_str == NULL) {
pf->label_str = lst_new_ptr(3);
pf->label_type = lst_new_int(3);
}
lst_push_ptr(pf->label_str, optstr);
lst_push_int(pf->label_type,
strcmp(long_opts[opt_idx].name, "label-branches") == 0 ?
BRANCH_TYPE : SUBTREE_TYPE);
}
else if (strcmp(long_opts[opt_idx].name, "selection") == 0) {
pf->selection = get_arg_dbl(optarg);
pf->use_selection = TRUE;
}
else {
die("ERROR: unknown option. Type 'phyloFit -h' for usage.\n");
}
break;
case 'u':
if (pf->bound_arg == NULL)
pf->bound_arg = lst_new_ptr(1);
optstr = str_new_charstr(optarg);
lst_push_ptr(pf->bound_arg, optstr);
break;
case 'D':
seed = get_arg_int_bounds(optarg, 1, INFTY);
break;
case 'h':
printf("%s", HELP);
exit(0);
case '?':
die("ERROR: illegal argument. Type 'phyloFit -h' for usage.\n");
}
}
set_seed(seed);
if (msa_fname == NULL) {
if (optind >= argc)
die("ERROR: missing alignment filename. Type 'phyloFit -h' for usage.\n");
msa_fname = argv[optind];
pf->msa_fname = msa_fname;
}
infile = phast_fopen(msa_fname, "r");
if (input_format == UNKNOWN_FORMAT)
input_format = msa_format_for_content(infile, 1);
if (pf->nonoverlapping && (pf->use_conditionals || pf->gff != NULL ||
pf->cats_to_do_str || input_format == SS))
die("ERROR: cannot use --non-overlapping with --markov, --features,\n--msa-format SS, or --do-cats.\n");
/* read alignment */
if (!pf->quiet) fprintf(stderr, "Reading alignment from %s ...\n", msa_fname);
if (input_format == MAF) {
pf->msa = maf_read(infile, NULL,
tm_order(pf->subst_mod) + 1,
NULL, pf->gff, pf->cm,
pf->nonoverlapping ? tm_order(pf->subst_mod) + 1 : -1,
FALSE, pf->reverse_group_tag, NO_STRIP, FALSE);
if (pf->gaps_as_bases)
msa_reset_alphabet(pf->msa, alph);
}
else
pf->msa = msa_new_from_file_define_format(infile,
input_format, alph);
/* set up for categories */
/* first label sites, if necessary */
pf->label_categories = (input_format != MAF);
run_phyloFit(pf);
if (pf->logf != NULL && pf->logf != stderr && pf->logf != stdout)
phast_fclose(pf->logf);
if (!pf->quiet) fprintf(stderr, "Done.\n");
sfree(pf);
return 0;
}
int main(int argc, char *argv[]) {
char c;
int opt_idx, node;
FILE *out_f = NULL, *msa_f, *mod_f;
char *out_root;
TreeModel *mod;
MSA *msa;
char out_fname[STR_MED_LEN];
struct option long_opts[] = {
{"refseq", 1, 0, 'r'},
{"msa-format", 1, 0, 'i'},
{"seqs", 1, 0, 's'},
{"exclude", 0, 0, 'x'},
{"no-probs", 0, 0, 'n'},
{"suff-stats", 0, 0, 'S'},
{"encode", 1, 0, 'e'},
{"keep-gaps", 0, 0, 'k'},
{"gibbs", 1, 0, 'G'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
/* arguments and defaults for options */
FILE *refseq_f = NULL;
msa_format_type msa_format = UNKNOWN_FORMAT;
int suff_stats = FALSE, exclude = FALSE, keep_gaps = FALSE, do_probs = TRUE;
List *seqlist = NULL;
PbsCode *code = NULL;
int gibbs_nsamples = -1;
while ((c = (char)getopt_long(argc, argv, "r:i:s:e:knxSh", long_opts, &opt_idx)) != -1) {
switch (c) {
case 'r':
refseq_f = phast_fopen(optarg, "r");
break;
case 'i':
msa_format = msa_str_to_format(optarg);
if (msa_format == UNKNOWN_FORMAT)
die("ERROR: unrecognized alignment format.\n");
break;
case 'S':
suff_stats = TRUE;
break;
case 'e':
code = pbs_new_from_file(phast_fopen(optarg, "r"));
break;
case 's':
seqlist = get_arg_list(optarg);
break;
case 'x':
exclude = TRUE;
break;
case 'n':
do_probs = FALSE;
break;
case 'k':
keep_gaps = TRUE;
break;
case 'G':
gibbs_nsamples = get_arg_int_bounds(optarg, 1, INFTY);
break;
case 'h':
printf("%s", HELP);
exit(0);
case '?':
die("Bad argument. Try 'prequel -h'.\n");
}
}
if (optind != argc - 3)
die("Three arguments required. Try 'prequel -h'.\n");
set_seed(-1);
if (!do_probs && (suff_stats || code != NULL))
die("ERROR: --no-probs can't be used with --suff-stats or --encode.\n");
msa_f = phast_fopen(argv[optind], "r");
if (msa_format == UNKNOWN_FORMAT)
msa_format = msa_format_for_content(msa_f, 1);
fprintf(stderr, "Reading alignment from %s...\n", argv[optind]);
if (msa_format == MAF) {
msa = maf_read(msa_f, refseq_f, 1, NULL, NULL, NULL, -1, !suff_stats, NULL,
NO_STRIP, FALSE);
/* (no need to store order if suff_stats mode) */
}
else
msa = msa_new_from_file_define_format(msa_f, msa_format, NULL);
if (msa->ss == NULL) {
fprintf(stderr, "Extracting sufficient statistics...\n");
ss_from_msas(msa, 1, TRUE, NULL, NULL, NULL, -1, 0);
}
else if (msa->ss->tuple_idx == NULL && !suff_stats)
die("ERROR: ordered representation of alignment required unless --suff-stats.\n");
mod_f = phast_fopen(argv[optind+1], "r");
out_root = argv[optind+2];
mod = tm_new_from_file(mod_f, 1);
/* MH prune just like in phastcons */
int old_nnodes = mod->tree->nnodes;
List *pruned_names = lst_new_ptr(msa->nseqs);
tm_prune(mod, msa, pruned_names);
if (lst_size(pruned_names) == (old_nnodes + 1) / 2)
die("ERROR: no match for leaves of tree in alignment (leaf names must match alignment names).\n");
if (lst_size(pruned_names) > 0) {
fprintf(stderr, "WARNING: pruned away leaves of tree with no match in alignment (");
int j;
for (j = 0; j < lst_size(pruned_names); j++)
fprintf(stderr, "%s%s", ((String*)lst_get_ptr(pruned_names, j))->chars, j < lst_size(pruned_names) - 1 ? ", " : ").\n");
}
lst_free_strings(pruned_names);
tr_name_ancestors(mod->tree);
if (mod->order != 0)
die("ERROR: Only single nucleotide models are supported.\n");
if (mod->nratecats > 1)
die("ERROR: Rate variation not supported.\n");
mod->tree_posteriors = tl_new_tree_posteriors(mod, msa, TRUE, FALSE,
FALSE, FALSE, FALSE, FALSE,
FALSE);
fprintf(stderr, "Computing posterior probabilities...\n");
if (gibbs_nsamples > 0)
die("ERROR: --gibbs not implemented yet.");
/* gb_sample_ancestral_seqs(mod, msa, mod->tree_posteriors, gibbs_nsamples); */
else
tl_compute_log_likelihood(mod, msa, NULL, NULL, -1, mod->tree_posteriors);
fprintf(stderr, "Reconstructing indels by parsimony...\n");
do_indels(msa, mod);
for (node = 0; node < mod->tree->nnodes; node++) {
int i, j;
TreeNode *n = lst_get_ptr(mod->tree->nodes, node);
if (n->lchild == NULL || n->rchild == NULL) continue;
if (seqlist != NULL) {
int in_list = str_in_list_charstr(n->name, seqlist);
if ((in_list && exclude) || (!in_list && !exclude))
continue;
}
fprintf(stderr, "Writing output for ancestral node '%s'...\n",
n->name);
if (suff_stats) {
if (out_f == NULL) {
sprintf(out_fname, "%s.stats", out_root);
out_f = phast_fopen(out_fname, "w+");
fprintf(out_f, "#count\t");
for (j = 0; j < mod->rate_matrix->size; j++)
fprintf(out_f, "p(%c)%c", mod->rate_matrix->states[j],
j == mod->rate_matrix->size - 1 ? '\n' : '\t');
}
for (i = 0; i < msa->ss->ntuples; i++) {
if (mod->tree_posteriors->base_probs[0][0][node][i] == -1)
continue; /* no base this node */
fprintf(out_f, "%.0f\t", msa->ss->counts[i]);
for (j = 0; j < mod->rate_matrix->size; j++) {
fprintf(out_f, "%f%c",
mod->tree_posteriors->base_probs[0][j][node][i],
j == mod->rate_matrix->size - 1 ? '\n' : '\t');
}
}
}
else if (code == NULL && do_probs) { /* ordinary sequence-by-sequence
output */
sprintf(out_fname, "%s.%s.probs", out_root, n->name);
out_f = phast_fopen(out_fname, "w+");
fprintf(out_f, "#");
for (j = 0; j < mod->rate_matrix->size; j++)
fprintf(out_f, "p(%c)%c", mod->rate_matrix->states[j],
j == mod->rate_matrix->size - 1 ? '\n' : '\t');
for (i = 0; i < msa->length; i++) {
if (mod->tree_posteriors->base_probs[0][0][node][msa->ss->tuple_idx[i]] == -1) {
/* no base */
if (keep_gaps) fprintf(out_f, "-\n");
/* otherwise do nothing */
}
else
for (j = 0; j < mod->rate_matrix->size; j++)
fprintf(out_f, "%f%c",
mod->tree_posteriors->base_probs[0][j][node][msa->ss->tuple_idx[i]],
j == mod->rate_matrix->size - 1 ? '\n' : '\t');
}
phast_fclose(out_f);
}
else if (code == NULL && !do_probs) { /* write point estimates
to FASTA file */
char *outseq = smalloc((msa->length + 1) * sizeof(char));
int len = 0;
for (i = 0; i < msa->length; i++) {
if (mod->tree_posteriors->base_probs[0][0][node][msa->ss->tuple_idx[i]] == -1) {
/* no base */
if (keep_gaps) outseq[len++] = GAP_CHAR;
/* otherwise do nothing */
}
else {
double maxprob = 0;
int maxidx = -1;
for (j = 0; j < mod->rate_matrix->size; j++) {
if (mod->tree_posteriors->base_probs[0][j][node][msa->ss->tuple_idx[i]] > maxprob) {
maxprob = mod->tree_posteriors->base_probs[0][j][node][msa->ss->tuple_idx[i]];
maxidx = j;
}
}
outseq[len++] = mod->rate_matrix->states[maxidx];
}
}
outseq[len] = '\0';
/* print in FASTA format */
sprintf(out_fname, "%s.%s.fa", out_root, n->name);
out_f = phast_fopen(out_fname, "w+");
print_seq_fasta(out_f, outseq, n->name, len);
phast_fclose(out_f);
sfree(outseq);
}
else { /* encoded sequence-by-sequence
output */
double error, tot_error = 0;
int ngaps = 0;
Vector *v;
unsigned *encoded;
/* first encode tuple by tuple */
v = vec_new(mod->rate_matrix->size);
encoded = smalloc(msa->ss->ntuples * sizeof(unsigned));
for (i = 0; i < msa->ss->ntuples; i++) {
if (mod->tree_posteriors->base_probs[0][0][node][i] == -1) {
encoded[i] = code->gap_code;
ngaps += msa->ss->counts[i];
}
else {
for (j = 0; j < mod->rate_matrix->size; j++)
vec_set(v, j, mod->tree_posteriors->base_probs[0][j][node][i]);
encoded[i] = pbs_get_index(code, v, &error);
tot_error += error * msa->ss->counts[i];
}
}
vec_free(v);
/* now write site by site */
sprintf(out_fname, "%s.%s.bin", out_root, n->name);
out_f = phast_fopen(out_fname, "w+");
for (i = 0; i < msa->length; i++) {
if (keep_gaps || encoded[msa->ss->tuple_idx[i]] != code->gap_code)
pbs_write_binary(code, encoded[msa->ss->tuple_idx[i]], out_f);
}
fprintf(stderr, "Average approximation error ('%s'): %f bits\n",
n->name, tot_error/(msa->length - ngaps));
sfree(encoded);
}
}
fprintf(stderr, "Done.\n");
return 0;
}
SEXP rph_gff_one_attribute(SEXP gffP, SEXP tagP) {
GFF_Set *gff = (GFF_Set*)EXTPTR_PTR(gffP);
GFF_Feature *f;
ListOfLists *lol;
List *l1, *l2;
int numtag, numval, i, j, k, resultLen, maxResultLen=10;
String *currStr, *tag, *currTag;
char **result;
SEXP rv;
SEXP rph_listOfLists_to_SEXP(ListOfLists *lol);
if (lst_size(gff->features) == 0) return R_NilValue;
gff_register_protect(gff);
result = smalloc(maxResultLen*sizeof(char*));
tag = str_new_charstr(CHARACTER_VALUE(tagP));
str_double_trim(tag);
lol = lol_new(lst_size(gff->features));
l1 = lst_new_ptr(10);
l2 = lst_new_ptr(10);
for (i=0; i < lst_size(gff->features); i++) {
checkInterruptN(i, 1000);
resultLen=0;
f = (GFF_Feature*) lst_get_ptr(gff->features, i);
numtag = str_split_with_quotes(f->attribute, ";", l1); //split tags
for (j=0; j < numtag; j++) {
currStr = (String*)lst_get_ptr(l1, j);
str_double_trim(currStr);
//first try gff version 3, see if we have tag=val format
numval = str_split_with_quotes(currStr, "=", l2);
if (numval == 2) {
currTag = (String*)lst_get_ptr(l2, 0);
str_double_trim(currTag);
if (str_equals(tag, currTag)) { // tag matches target, add all values to list
currStr = str_new_charstr(((String*)lst_get_ptr(l2, 1))->chars);
lst_free_strings(l2);
numval = str_split_with_quotes(currStr, ",", l2);
str_free(currStr);
for (k=0; k < numval; k++) {
currStr = lst_get_ptr(l2, k);
str_double_trim(currStr);
str_remove_quotes(currStr);
if (resultLen > maxResultLen) {
maxResultLen += 100;
result = srealloc(result, maxResultLen*sizeof(char*));
}
result[resultLen++] = copy_charstr(currStr->chars);
}
}
} else {
lst_free_strings(l2);
//gff version 2
//split into tag val val ... by whitespace unless enclosed in quotes
numval = str_split_with_quotes(currStr, NULL, l2);
if (numval > 1) {
currStr = (String*)lst_get_ptr(l2, 0);
str_double_trim(currStr);
if (str_equals(tag, currStr)) { //tag matches target, add all values to list
for (k=1; k < numval; k++) {
currStr = (String*)lst_get_ptr(l2, k);
str_double_trim(currStr);
str_remove_quotes(currStr);
if (resultLen > maxResultLen) {
maxResultLen += 100;
result = srealloc(result, maxResultLen*sizeof(char*));
}
result[resultLen++] = copy_charstr(currStr->chars);
}
}
}
lst_free_strings(l2);
}
}
if (resultLen == 0)
result[resultLen++] = copy_charstr(""); //empty string will be converted to NA later
lol_push_charvec(lol, result, resultLen, NULL);
for (j=0; j < resultLen; j++) sfree(result[j]);
}
PROTECT(rv = rph_listOfLists_to_SEXP(lol));
UNPROTECT(1);
return rv;
}
int main(int argc, char *argv[]) {
char c;
List *l;
int i, j, strand, bed_output = 0, backgd_nmods = -1, feat_nmods = -1,
winsize = -1, verbose = 0, max_nmods, memblocksize, old_nleaves,
refidx = 1, base_by_base = FALSE, windowWig = FALSE;
TreeModel **backgd_mods = NULL, **feat_mods = NULL;
HMM *backgd_hmm = NULL, *feat_hmm = NULL;
msa_format_type inform = UNKNOWN_FORMAT;
GFF_Set *features = NULL;
MSA *msa, *msa_compl=NULL;
double **backgd_emissions, **feat_emissions, **mem, **dummy_emissions,
*winscore_pos=NULL, *winscore_neg=NULL;
int *no_alignment=NULL;
List *pruned_names;
char *msa_fname;
FILE *infile;
int opt_idx;
struct option long_opts[] = {
{"background-mods", 1, 0, 'b'},
{"background-hmm", 1, 0, 'B'},
{"feature-mods", 1, 0, 'f'},
{"feature-hmm", 1, 0, 'F'},
{"features", 1, 0, 'g'},
{"window", 1, 0, 'w'},
{"window-wig", 1, 0, 'W'},
{"base-by-base", 0, 0, 'y'},
{"msa-format", 1, 0, 'i'},
{"refidx", 1, 0, 'r'},
{"output-bed", 0, 0, 'd'},
{"verbose", 0, 0, 'v'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
while ((c = getopt_long(argc, argv, "B:b:F:f:r:g:w:W:i:ydvh", long_opts, &opt_idx)) != -1) {
switch (c) {
case 'B':
backgd_hmm = hmm_new_from_file(phast_fopen(optarg, "r"));
break;
case 'b':
l = get_arg_list(optarg);
backgd_nmods = lst_size(l);
backgd_mods = smalloc(backgd_nmods * sizeof(void*));
for (i = 0; i < backgd_nmods; i++)
backgd_mods[i] = tm_new_from_file(phast_fopen(((String*)lst_get_ptr(l, i))->chars, "r"), 1);
lst_free_strings(l);
lst_free(l);
break;
case 'F':
feat_hmm = hmm_new_from_file(phast_fopen(optarg, "r"));
break;
case 'f':
l = get_arg_list(optarg);
feat_nmods = lst_size(l);
feat_mods = smalloc(feat_nmods * sizeof(void*));
for (i = 0; i < feat_nmods; i++)
feat_mods[i] = tm_new_from_file(phast_fopen(((String*)lst_get_ptr(l, i))->chars, "r"), 1);
lst_free_strings(l);
lst_free(l);
break;
case 'g':
features = gff_read_set(phast_fopen(optarg, "r"));
break;
case 'w':
winsize = get_arg_int(optarg);
if (winsize <= 0) die("ERROR: window size must be positive.\n");
break;
case 'W':
winsize = get_arg_int(optarg);
if (winsize <= 0) die("ERROR: window size must be positive.\n");
windowWig = TRUE;
break;
case 'y':
base_by_base = TRUE;
break;
case 'i':
inform = msa_str_to_format(optarg);
if (inform == UNKNOWN_FORMAT) die("Bad argument to -i.\n");
break;
case 'r':
refidx = get_arg_int_bounds(optarg, 0, INFTY);
break;
case 'd':
bed_output = 1;
break;
case 'h':
printf("%s", HELP);
exit(0);
case 'v':
verbose = 1;
break;
case '?':
die("Bad argument. Try '%s -h'.\n", argv[0]);
}
}
set_seed(-1);
if (backgd_mods == NULL || feat_mods == NULL)
die("ERROR: -b and -f required. Try '%s -h'.\n", argv[0]);
if (backgd_nmods == 1 && backgd_hmm == NULL)
backgd_hmm = hmm_create_trivial();
else if (backgd_hmm == NULL)
die("ERROR: -B required. Try '%s -h'.\n", argv[0]);
if (feat_nmods == 1 && feat_hmm == NULL)
feat_hmm = hmm_create_trivial();
else if (feat_hmm == NULL)
die("ERROR: -F required. Try '%s -h'.\n", argv[0]);
if ((winsize == -1 && features == NULL && !base_by_base) ||
(winsize != -1 && features != NULL) ||
(winsize != -1 && base_by_base) ||
(features != NULL && base_by_base))
die("ERROR: must specify exactly one of -g, -w, and -y. Try '%s -h'.\n", argv[0]);
if (backgd_hmm->nstates != backgd_nmods)
die("ERROR: number of states must equal number of tree models for background.\n");
if (feat_hmm->nstates != feat_nmods)
die("ERROR: number of states must equal number of tree models for features.\n");
if (features != NULL && lst_size(features->features) == 0)
die("ERROR: empty features file.\n");
if (base_by_base && (backgd_nmods > 1 || feat_nmods > 1))
die("ERROR: only single phylogenetic models (not HMMs) are supported with --base-by-base.\n");
if (optind != argc - 1)
die("ERROR: too few arguments. Try '%s -h'.\n", argv[0]);
if (verbose) fprintf(stderr, "Reading alignment ...\n");
msa_fname = argv[optind];
infile = phast_fopen(msa_fname, "r");
if (inform == UNKNOWN_FORMAT)
inform = msa_format_for_content(infile, 1);
if (inform == MAF)
msa = maf_read(infile, NULL, 1, NULL, NULL,
NULL, -1, TRUE, NULL, NO_STRIP, FALSE);
else
msa = msa_new_from_file_define_format(infile, inform, NULL);
if (msa_alph_has_lowercase(msa)) msa_toupper(msa);
msa_remove_N_from_alph(msa);
/* need ordered representation of alignment */
if (msa->seqs == NULL && (msa->ss == NULL || msa->ss->tuple_idx == NULL) )
die("ERROR: ordered sufficient statistics are required.\n");
pruned_names = lst_new_ptr(msa->nseqs);
for (i = 0; i < backgd_nmods; i++) {
old_nleaves = (backgd_mods[i]->tree->nnodes + 1) / 2;
tm_prune(backgd_mods[i], msa, pruned_names);
if (lst_size(pruned_names) >= old_nleaves)
die("ERROR: no match for leaves of tree in alignment (background model #%d)\n", i+1);
else if (lst_size(pruned_names) > 0) {
fprintf(stderr, "WARNING: pruned away leaves in background model (#%d) with no match in alignment (", i+1);
for (j = 0; j < lst_size(pruned_names); j++)
fprintf(stderr, "%s%s", ((String*)lst_get_ptr(pruned_names, j))->chars,
j < lst_size(pruned_names) - 1 ? ", " : ").\n");
}
lst_free_strings(pruned_names);
}
for (i = 0; i < feat_nmods; i++) {
old_nleaves = (feat_mods[i]->tree->nnodes + 1) / 2;
tm_prune(feat_mods[i], msa, pruned_names);
if (lst_size(pruned_names) >= old_nleaves)
die("ERROR: no match for leaves of tree in alignment (features model #%d)\n", i+1);
else if (lst_size(pruned_names) > 0) {
fprintf(stderr, "WARNING: pruned away leaves in features model (#%d) with no match in alignment (", i+1);
for (j = 0; j < lst_size(pruned_names); j++)
fprintf(stderr, "%s%s", ((String*)lst_get_ptr(pruned_names, j))->chars,
j < lst_size(pruned_names) - 1 ? ", " : ").\n");
}
lst_free_strings(pruned_names);
}
lst_free(pruned_names);
/* first have to subtract offset from features, if necessary */
if (msa->idx_offset != 0 && features != NULL) {
for (i = 0; i < lst_size(features->features); i++) {
GFF_Feature *f = lst_get_ptr(features->features, i);
f->start -= msa->idx_offset;
f->end -= msa->idx_offset;
}
}
/* convert to coord frame of alignment */
if (features != NULL && refidx != 0) {
if (verbose) fprintf(stderr, "Mapping coordinates ...\n");
msa_map_gff_coords(msa, features, refidx, 0, 0);
if (lst_size(features->features) == 0)
die("ERROR: no features within coordinate range of alignment.\n");
}
/* Make a reverse complemented copy of the alignment. The two
strands will be processed separately, to avoid problems with
overlapping features, etc. */
if (!base_by_base) { /* skip in base by base case */
if (verbose) fprintf(stderr, "Creating reverse complemented alignment ...\n");
msa_compl = msa_create_copy(msa, 0);
/* temporary workaround: make sure reverse complement not based on
sufficient stats */
if (msa_compl->seqs == NULL) ss_to_msa(msa_compl);
if (msa_compl->ss != NULL) {
ss_free(msa_compl->ss);
msa_compl->ss = NULL;
}
msa_reverse_compl(msa_compl);
}
/* allocate memory for computing scores */
backgd_emissions = smalloc(backgd_nmods * sizeof(void*));
for (i = 0; i < backgd_nmods; i++)
backgd_emissions[i] = smalloc(msa->length * sizeof(double));
feat_emissions = smalloc(feat_nmods * sizeof(void*));
for (i = 0; i < feat_nmods; i++)
feat_emissions[i] = smalloc(msa->length * sizeof(double));
max_nmods = max(backgd_nmods, feat_nmods);
dummy_emissions = smalloc(max_nmods * sizeof(void*));
mem = smalloc(max_nmods * sizeof(void*));
/* memory for forward algorithm -- each block must be as large as
the largest feature */
if (features != NULL) {
for (i = 0, memblocksize = -1; i < lst_size(features->features); i++) {
GFF_Feature *f = lst_get_ptr(features->features, i);
if (f->end - f->start + 1 > memblocksize)
memblocksize = f->end - f->start + 1;
}
}
else memblocksize = winsize; /* -1 if base-by-base mode */
if (memblocksize > 0)
for (i = 0; i < max_nmods; i++)
mem[i] = smalloc(memblocksize * sizeof(double));
if (winsize != -1) {
winscore_pos = smalloc(msa->length * sizeof(double));
winscore_neg = smalloc(msa->length * sizeof(double));
no_alignment = smalloc(msa->length * sizeof(int));
for (i = 0; i < msa->length; i++) {
winscore_pos[i] = winscore_neg[i] = NEGINFTY;
if (refidx == 0)
no_alignment[i] = FALSE;
else
no_alignment[i] = msa_missing_col(msa, refidx, i);
}
}
/* the rest will be repeated for each strand */
for (strand = 1; strand <= 2; strand++) {
MSA *thismsa = strand == 1 ? msa : msa_compl;
double *winscore = strand == 1 ? winscore_pos : winscore_neg;
if (base_by_base && strand == 2) break; /* don't do second pass in
base_by_base case */
if (verbose) fprintf(stderr, "Processing %c strand ...\n",
strand == 1 ? '+' : '-');
/* set up dummy categories array, so that emissions are only
computed where needed */
thismsa->categories = smalloc(thismsa->length * sizeof(int));
thismsa->ncats = 1;
if (winsize != -1) {
if (strand == 1)
for (i = 0; i < thismsa->length; i++)
thismsa->categories[i] = no_alignment[i] ? 0 : 1;
else
for (i = 0; i < thismsa->length; i++)
thismsa->categories[i] = no_alignment[thismsa->length - i - 1] ? 0 : 1;
}
else if (features != NULL) {
for (i = 0; i < thismsa->length; i++) thismsa->categories[i] = 0;
for (i = 0; i < lst_size(features->features); i++) {
GFF_Feature *f = lst_get_ptr(features->features, i);
if (f->start <= 0 || f->end <= 0) {
fprintf(stderr, "WARNING: feature out of range ('");
gff_print_feat(stderr, f);
fprintf(stderr, "')\n");
continue;
}
if (strand == 1 && f->strand != '-')
for (j = f->start - 1; j < f->end; j++)
thismsa->categories[j] = 1;
else if (strand == 2 && f->strand == '-')
for (j = thismsa->length - f->end;
j < thismsa->length - f->start + 1; j++)
thismsa->categories[j] = 1;
}
}
else { /* base-by-base scores */
for (i = 0; i < thismsa->length; i++) thismsa->categories[i] = 1;
}
if (thismsa->ss != NULL) ss_update_categories(thismsa);
/* compute emissions */
for (i = 0; i < backgd_nmods; i++) {
if (verbose)
fprintf(stderr, "Computing emissions for background model #%d ...\n", i+1);
tl_compute_log_likelihood(backgd_mods[i], thismsa,
backgd_emissions[i], NULL, 1, NULL);
}
for (i = 0; i < feat_nmods; i++) {
if (verbose)
fprintf(stderr, "Computing emissions for features model #%d ...\n", i+1);
tl_compute_log_likelihood(feat_mods[i], thismsa,
feat_emissions[i], NULL, 1, NULL);
}
/* now compute scores */
if (winsize != -1) { /* windows case */
int winstart;
if (verbose) fprintf(stderr, "Computing scores ...\n");
for (winstart = 0; winstart <= thismsa->length - winsize; winstart++) {
int centeridx = winstart + winsize/2;
if (strand == 2) centeridx = thismsa->length - centeridx - 1;
if (no_alignment[centeridx]) continue;
for (j = 0; j < feat_nmods; j++)
dummy_emissions[j] = &(feat_emissions[j][winstart]);
winscore[centeridx] = hmm_forward(feat_hmm, dummy_emissions,
winsize, mem);
if (winscore[centeridx] <= NEGINFTY) {
winscore[centeridx] = NEGINFTY;
continue;
}
for (j = 0; j < backgd_nmods; j++)
dummy_emissions[j] = &(backgd_emissions[j][winstart]);
winscore[centeridx] -= hmm_forward(backgd_hmm, dummy_emissions,
winsize, mem);
if (winscore[centeridx] < NEGINFTY) winscore[centeridx] = NEGINFTY;
}
}
else if (features != NULL) { /* features case */
if (verbose) fprintf(stderr, "Computing scores ...\n");
for (i = 0; i < lst_size(features->features); i++) {
GFF_Feature *f = lst_get_ptr(features->features, i);
int s, e;
if ((strand == 1 && f->strand == '-') ||
(strand == 2 && f->strand != '-') ||
f->start <= 0 || f->end <= 0 || f->end - f->start < 0)
continue;
/* effective coords */
if (f->strand == '-') {
s = thismsa->length - f->end + 1;
e = thismsa->length - f->start + 1;
}
else {
s = f->start;
e = f->end;
}
f->score_is_null = 0;
for (j = 0; j < feat_nmods; j++)
dummy_emissions[j] = &(feat_emissions[j][s-1]);
f->score = hmm_forward(feat_hmm, dummy_emissions, e - s + 1, mem);
if (f->score <= NEGINFTY) {
f->score = NEGINFTY;
continue;
}
for (j = 0; j < backgd_nmods; j++)
dummy_emissions[j] = &(backgd_emissions[j][s-1]);
f->score -= hmm_forward(backgd_hmm, dummy_emissions, e - s + 1, mem);
if (f->score < NEGINFTY) f->score = NEGINFTY;
}
}
}
if (verbose) fprintf(stderr, "Generating output ...\n");
if (winsize != -1 && windowWig == FALSE) { /* standard windows output */
for (i = 0, j = 0; i < msa->length; i++) {
if (no_alignment[i] == FALSE)
printf("%d\t%.3f\t%.3f\n", j + msa->idx_offset + 1, winscore_pos[i],
winscore_neg[i]);
if (ss_get_char_pos(msa, i, 0, 0) != GAP_CHAR) j++;
}
}
else if (windowWig == TRUE) { /* windows with wig output */
int last = NEGINFTY;
for (i = 0, j = 0; i < msa->length; i++) {
if (refidx == 0 || msa_get_char(msa, refidx-1, i) != GAP_CHAR) {
if (no_alignment[i] == FALSE && winscore_pos[i] > NEGINFTY) {
if (j > last + 1)
printf("fixedStep chrom=%s start=%d step=1\n",
refidx > 0 ? msa->names[refidx-1] : "alignment",
j + msa->idx_offset + 1);
printf("%.3f\n", winscore_pos[i]);
last = j;
}
j++;
}
}
}
else if (features != NULL) { /* features output */
/* return to coord frame of reference seq (also, replace offset) */
if (refidx != 0)
msa_map_gff_coords(msa, features, 0, refidx, msa->idx_offset);
else if (msa->idx_offset != 0) {
for (i = 0; i < lst_size(features->features); i++) {
GFF_Feature *f = lst_get_ptr(features->features, i);
f->start += msa->idx_offset;
f->end += msa->idx_offset;
}
}
if (bed_output)
gff_print_bed(stdout, features, FALSE);
else
gff_print_set(stdout, features);
}
else { /* base-by-base scores */
/* in this case, we can just output the difference between the emissions */
printf("fixedStep chrom=%s start=%d step=1\n",
refidx > 0 ? msa->names[refidx-1] : "alignment",
msa->idx_offset + 1);
for (i = 0, j = 0; i < msa->length; i++) {
if (refidx == 0 || msa_get_char(msa, refidx-1, i) != GAP_CHAR) {
printf("%.3f\n", feat_emissions[0][i] - backgd_emissions[0][i]);
j++;
}
}
}
if (verbose) fprintf(stderr, "\nDone.\n");
return 0;
}
int main(int argc, char *argv[]) {
FILE *STATSF;
char c;
int opt_idx, i, max_nrows;
String *line = str_new(STR_MED_LEN), *args = str_new(STR_MED_LEN);
List *fields = lst_new_ptr(5), *vectors = lst_new_ptr(1000),
*counts = lst_new_int(1000);
int dim = -1;
double error = -1;
PbsCode *code;
char comment[1000];
time_t t;
int have_data = TRUE;
/* argument variables and defaults */
int nrows = -1, nbytes = 1;
training_mode mode = FULL;
FILE *logf = NULL;
struct option long_opts[] = {
{"nrows", 1, 0, 'n'},
{"nbytes", 1, 0, 'b'},
{"no-greedy", 0, 0, 'G'},
{"no-train", 1, 0, 'x'},
{"log", 1, 0, 'l'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
set_seed(-1);
/* first capture arg list for comment in output */
for (i = 1; i < argc; i++) {
str_append_charstr(args, argv[i]);
if (i < argc - 1) str_append_char(args, ' ');
}
while ((c = (char)getopt_long(argc, argv, "n:b:l:Gxh", long_opts, &opt_idx)) != -1) {
switch (c) {
case 'n':
nrows = get_arg_int_bounds(optarg, 1, INFTY);
break;
case 'b':
nbytes = get_arg_int_bounds(optarg, 1, MAX_NBYTES);
break;
case 'G':
mode = NO_GREEDY;
break;
case 'x':
mode = NO_TRAIN;
dim = get_arg_int_bounds(optarg, 1, INFTY);
break;
case 'l':
logf = phast_fopen(optarg, "w+");
break;
case 'h':
printf("%s", HELP);
exit(0);
case '?':
die("Bad argument. Try 'pbsTrain -h'.\n");
}
}
if (mode == NO_TRAIN && optind == argc)
have_data = FALSE; /* data optional */
if (have_data) {
if (optind != argc - 1)
die("ERROR: Bad arguments. Try 'pbsTrain -h'.\n");
STATSF = phast_fopen(argv[optind], "r");
/* read stats */
while (str_readline(line, STATSF) != EOF) {
int count;
double prob, norm_const;
Vector *v;
str_trim(line);
if (line->length == 0 || line->chars[0] == '#') continue;
str_split(line, NULL, fields);
if (str_as_int(lst_get_ptr(fields, 0), &count) != 0)
die("ERROR: Bad count in stats file ('%s')\n", lst_get_ptr(fields, 0));
lst_push_int(counts, count);
if (dim == -1) dim = lst_size(fields) - 1;
else if (dim != lst_size(fields) - 1)
die("ERROR: Each probability vector must have the same dimension\n");
v = vec_new(dim);
for (i = 0; i < dim; i++) {
if (str_as_dbl(lst_get_ptr(fields, i+1), &prob) != 0 ||
prob < 0 || prob > 1)
die("ERROR: Bad probability in stats file ('%s')\n",
lst_get_ptr(fields, i+1));
vec_set(v, i, prob);
}
/* normalize to avoid problems from rounding errors */
norm_const = normalize_probs(v->data, dim);
if (fabs(1-norm_const) > 1e-2)
die("ERROR: Probabilities in stats file don't sum to one.\nOffending line: '%s'\n",
line->chars);
lst_push_ptr(vectors, v);
lst_free_strings(fields);
}
}
max_nrows = sxg_max_nrows(dim, ~(~0 << (8*nbytes)));
if (nrows == -1) nrows = max_nrows;
else if (nrows > max_nrows)
die("ERROR: nrows exceeds maximum of %d for nbytes = %d and dimension = %d\n",
max_nrows, nbytes, dim);
code = pbs_new(dim, nrows, nbytes);
if (mode != NO_TRAIN)
error = pbs_estimate_from_data(code, vectors, counts, logf, mode);
else if (have_data) { /* not training but need error */
int tot_count = 0;
double this_error;
error = 0;
for (i = 0; i < lst_size(vectors); i++) {
pbs_get_index(code, lst_get_ptr(vectors, i), &this_error);
error += this_error * lst_get_int(counts, i);
tot_count += lst_get_int(counts, i);
}
error /= tot_count;
}
/* generate comment */
t = time(NULL);
sprintf(comment, "# Code generated by pbsTrain, with argument(s) \"%s\"\n\
# %s\n\
# Average training error = %f bits\n",
args->chars, ctime(&t), error);
pbs_write(code, stdout, comment);
return 0;
}
int main(int argc, char *argv[]) {
/* variables for options, with defaults */
TreeNode *tree = NULL, *merge_tree = NULL, *extrapolate_tree = NULL;
Hashtable *rename_hash = NULL;
double scale_factor = 1;
List *prune_names = NULL, *label = NULL, *labelType = NULL;
int prune_all_but = FALSE, tree_only = FALSE, dissect = FALSE,
name_ancestors = FALSE, with_branch = FALSE, print_branchlen=FALSE,
inNewick=FALSE, no_branchlen = FALSE, print_distance_to_root = FALSE;
TreeModel *mod = NULL, *merge_mod = NULL;
char *reroot_name = NULL, *subtree_name =NULL, *get_subtree_name = NULL,
*node_distance_name = NULL;
/* other variables */
String *suffix, *optstr;
char c;
int i, opt_idx;
TreeNode *n;
struct option long_opts[] = {
{"scale", 1, 0, 's'},
{"extrapolate", 1, 0, 'e'},
{"prune", 1, 0, 'p'},
{"prune-all-but", 1, 0, 'P'},
{"get-subtree", 1, 0, 'g'},
{"merge", 1, 0, 'm'},
{"rename", 1, 0, 'r'},
{"tree-only", 0, 0, 't'},
{"no-branchlen", 0, 0, 'N'},
{"dissect", 0, 0, 'd'},
{"name-ancestors", 0, 0, 'a'},
{"reroot", 1, 0, 'R'},
{"with-branch", 1, 0, 'B'},
{"subtree", 1, 0, 'S'},
{"branchlen", 0, 0, 'b'},
{"newick", 0, 0, 'n'},
{"label-subtree", 1, 0, 'L'},
{"label-branches", 1, 0, 'l'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
while ((c = getopt_long(argc, argv, "s:p:P:g:m:r:R:B:S:D:l:L:adtNbnh",
long_opts, &opt_idx)) != -1) {
switch (c) {
case 's':
scale_factor = get_arg_dbl_bounds(optarg, 0, INFTY);
break;
case 'e':
if (!strcmp(optarg, "default")) {
optarg = smalloc(1000 * sizeof(char));
#if defined(__MINGW32__)
sprintf(optarg, "%s\\data\\exoniphy\\mammals\\cftr25_hybrid.nh",
PHAST_HOME);
#else
sprintf(optarg, "%s/data/exoniphy/mammals/cftr25_hybrid.nh",
PHAST_HOME);
#endif
}
extrapolate_tree = tr_new_from_file(phast_fopen(optarg, "r"));
break;
case 'p':
prune_names = get_arg_list(optarg);
break;
case 'P':
prune_names = get_arg_list(optarg);
prune_all_but = TRUE;
break;
case 'g':
get_subtree_name = optarg;
break;
case 'm':
suffix = str_new_charstr(optarg);
str_suffix(suffix, '.');
if (str_equals_charstr(suffix, "nh"))
merge_tree = tr_new_from_file(phast_fopen(optarg, "r"));
else {
merge_mod = tm_new_from_file(phast_fopen(optarg, "r"), 1);
merge_tree = merge_mod->tree;
}
break;
case 'r':
rename_hash = make_name_hash(optarg);
break;
case 't':
tree_only = TRUE;
break;
case 'N':
no_branchlen = TRUE;
tree_only = TRUE;
break;
case 'd':
dissect = TRUE;
break;
case 'b':
print_branchlen = TRUE;
break;
case 'D':
print_distance_to_root = TRUE;
node_distance_name = optarg;
break;
case 'R':
reroot_name = optarg;
break;
case 'B':
with_branch = TRUE;
break;
case 'a':
name_ancestors = TRUE;
break;
case 'S':
subtree_name = optarg;
break;
case 'n':
inNewick=TRUE;
break;
case 'L': //do the same for --label--subtree and --label-branches
case 'l':
if (label == NULL) {
label = lst_new_ptr(1);
labelType = lst_new_int(1);
}
optstr = str_new_charstr(optarg);
lst_push_ptr(label, optstr);
lst_push_int(labelType, (int)c);
break;
case 'h':
usage(argv[0]);
case '?':
die("Bad argument. Try '%s -h'.\n", argv[0]);
}
}
if (optind != argc - 1)
die("Input filename required. Try '%s -h'.\n", argv[0]);
if (merge_tree != NULL && extrapolate_tree != NULL)
die("ERROR: Can't use --merge and --extrapolate together");
set_seed(-1);
suffix = str_new_charstr(argv[optind]);
str_suffix(suffix, '.');
if (inNewick || str_equals_charstr(suffix, "nh")) {
tree = tr_new_from_file(phast_fopen(argv[optind], "r"));
tree_only = TRUE; /* can't output tree model in this case */
}
else {
mod = tm_new_from_file(phast_fopen(argv[optind], "r"), 1);
tree = mod->tree;
}
if (prune_names != NULL) {
tr_prune(&tree, prune_names, prune_all_but, NULL);
if (mod != NULL) mod->tree = tree; /* root may have changed */
}
if (get_subtree_name != NULL) {
n = tr_get_node(tree, get_subtree_name);
if (n == NULL) {
tr_name_ancestors(tree);
n = tr_get_node(tree, get_subtree_name);
if (n == NULL) {
die("ERROR: no node named '%s'.\n", subtree_name);
}
}
tr_prune_supertree(&tree, n);
if (mod != NULL) mod->tree = tree;
}
if (merge_tree != NULL) {
tree = tr_hybrid(tree, merge_tree);
if (mod != NULL) mod->tree = tree;
}
else if (extrapolate_tree != NULL) {
tr_scale_by_subtree(extrapolate_tree, tree);
tree = extrapolate_tree;
if (mod != NULL) mod->tree = tree;
}
if (scale_factor != 1) {
if (subtree_name == NULL)
tr_scale(tree, scale_factor);
else {
n = tr_get_node(tree, subtree_name);
if (n == NULL) die("ERROR: no node named '%s'.\n", subtree_name);
tr_scale_subtree(tree, n, scale_factor, with_branch);
}
}
if (name_ancestors)
tr_name_ancestors(tree);
if (rename_hash != NULL) {
char *newname;
for (i = 0; i < tree->nnodes; i++) {
n = lst_get_ptr(tree->nodes, i);
if (n->name != NULL && n->name[0] != '\0' &&
(newname = hsh_get(rename_hash, n->name)) != (char*)-1) {
strcpy(n->name, newname);
}
}
}
if (reroot_name != NULL) {
n = tr_get_node(tree, reroot_name);
if (n == NULL) die("ERROR: no node named '%s'.\n", reroot_name);
tr_reroot(tree, n, with_branch);
if (mod != NULL) mod->tree = with_branch ? n->parent : n;
tree = with_branch ? n->parent : n;
}
if (label != NULL) {
for (i=0; i < lst_size(label); i++) {
String *currstr = (String*)lst_get_ptr(label, i), *arg1, *labelVal;
List *tmplst = lst_new_ptr(10);
String *nodename;
int j;
str_split(currstr, ":", tmplst);
if (lst_size(tmplst) != 2)
die("ERROR: bad argument to --label-branches or --label-subtree.\n");
arg1 = lst_get_ptr(tmplst, 0);
labelVal = lst_get_ptr(tmplst, 1);
lst_clear(tmplst);
if (lst_get_int(labelType, i) == (int)'l') {
str_split(arg1, ",", tmplst);
for (j=0; j < lst_size(tmplst); j++) {
nodename = (String*)lst_get_ptr(tmplst, j);
tr_label_node(tree, nodename->chars, labelVal->chars);
}
lst_free_strings(tmplst);
} else if (lst_get_int(labelType, i) == (int)'L') {
int include_leading_branch = FALSE;
TreeNode *node;
nodename = arg1;
node = tr_get_node(tree, nodename->chars);
if (node == NULL && nodename->chars[nodename->length-1] == '+') {
nodename->chars[--nodename->length] = '\0';
node = tr_get_node(tree, nodename->chars);
include_leading_branch = TRUE;
}
tr_label_subtree(tree, nodename->chars, include_leading_branch,
labelVal->chars);
} else die("ERROR got label_type %c\n", lst_get_int(labelType, (char)i));
str_free(arg1);
str_free(labelVal);
lst_free(tmplst);
str_free(currstr);
}
lst_free(label);
lst_free(labelType);
}
if (dissect)
tr_print_nodes(stdout, tree);
if (print_branchlen)
printf("TOTAL_TREE_LEN: %f\n", tr_total_len(tree));
if (print_distance_to_root) {
TreeNode *node = tr_get_node(tree, node_distance_name);
if (node == NULL)
die("ERROR: no node named '%s'.\n", node_distance_name);
printf("length(root-%s): %f\n", node_distance_name,
tr_distance_to_root(node));
}
if (dissect==0 && print_branchlen==0 && print_distance_to_root==0) {
if (tree_only)
tr_print(stdout, tree, no_branchlen==FALSE);
else
tm_print(stdout, mod);
}
return 0;
}
int main(int argc, char *argv[]) {
FILE *prob_f;
char c;
int opt_idx, i, nlines = 0, ngaps = 0;
unsigned idx;
PbsCode *code;
List *fields = lst_new_ptr(10);
double error, tot_error = 0, prob;
Vector *v;
String *line = str_new(STR_MED_LEN);
struct option long_opts[] = {
{"discard-gaps", 0, 0, 'G'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
/* variables for options, with defaults */
int discard_gaps = FALSE;
set_seed(-1);
while ((c = (char)getopt_long(argc, argv, "Gh", long_opts, &opt_idx)) != -1) {
switch (c) {
case 'G':
discard_gaps = TRUE;
break;
case 'h':
printf("%s", HELP);
exit(0);
case '?':
die("Bad argument. Try 'pbsEncode -h'.\n");
}
}
if (optind != argc - 2)
die("Two arguments required. Try 'pbsEncode -h'.\n");
prob_f = phast_fopen(argv[optind], "r");
code = pbs_new_from_file(phast_fopen(argv[optind+1], "r"));
v = vec_new(code->sg->d);
while (str_readline(line, prob_f) != EOF) {
if (line->length == 0 || line->chars[0] == '#') continue;
str_split(line, NULL, fields);
if (lst_size(fields) == 1 && str_equals_charstr(lst_get_ptr(fields, 0), "-")) {
ngaps++;
if (!discard_gaps)
pbs_write_binary(code, code->gap_code, stdout);
}
else { /* ordinary prob vector */
if (lst_size(fields) != code->sg->d)
die("ERROR: number of columns must equal dimension of code (%d).\n",
code->sg->d);
for (i = 0; i < code->sg->d; i++) {
if (str_as_dbl(lst_get_ptr(fields, i), &prob) != 0 || prob < 0 || prob > 1)
die("ERROR: bad value ('%s')\n", lst_get_ptr(fields, i));
vec_set(v, i, prob);
}
idx = pbs_get_index(code, v, &error);
tot_error += error;
pbs_write_binary(code, idx, stdout);
nlines++;
}
lst_free_strings(fields);
}
fprintf(stderr, "Dimensions: %d\n\
Rows per dimension: %d\n\
Code size: %d\n\
Bytes per vector: %d\n\
Vectors processed: %d\n\
Gaps: %d%s\n\
Average approximation error: %f bits\n",
code->sg->d, code->sg->nrows, code->code_size, code->nbytes,
nlines, ngaps, discard_gaps ? " (discarded)" : "", tot_error/nlines);
return 0;
}
