/**********************************************************************
post_process()
adjust/normalize scores and p-values
**********************************************************************/
void post_process(CISML_T* cisml, ARRAYLST_T* motifs, BOOLEAN_T normalize_scores){
int m_index, seq_index;
MOTIF_AND_PSSM_T *combo;
for (m_index = 0; m_index < get_cisml_num_patterns(cisml); ++m_index) {
PATTERN_T* pattern = get_cisml_patterns(cisml)[m_index];
double maxscore = 1;
// FIXME: This should be done to the PSSM, not the individual scores!!!
// Normalize the scores to RMA format if necessary.
if (normalize_scores) {
int k;
combo = (MOTIF_AND_PSSM_T*)arraylst_get(m_index, motifs);
PSSM_T* pssm = combo->pssm_pair->pos_pssm;
for (k = 0; k < pssm->w; k++) {
double maxprob = -BIG; // These are scores, not probabilities!!!
int a;
for (a = 0; a < alph_size_core(pssm->alph); a++) {
double prob = get_matrix_cell(k, a, pssm->matrix);
if (maxprob < prob) maxprob = prob;
}
maxscore *= maxprob;
}
}
// adjust each scanned sequence
for (seq_index = 0; seq_index < get_pattern_num_scanned_sequences(pattern);
++seq_index) {
SCANNED_SEQUENCE_T* scanned_seq =
get_pattern_scanned_sequences(pattern)[seq_index];
// only adjust scores and p-values if more than one copy was scored
// num_scanned_positions is (mis-)used in ama to indicate the number of times
// a sequence identifier 0occured in the set
if (get_scanned_sequence_num_scanned_positions(scanned_seq) > 1L){
// take average score
if(has_scanned_sequence_score(scanned_seq)){
double avg_odds = get_scanned_sequence_score(scanned_seq) /
get_scanned_sequence_num_scanned_positions(scanned_seq);
set_scanned_sequence_score(scanned_seq, avg_odds);
}
// adjust the minimum p-value for multiple hypothesis testing
if(has_scanned_sequence_pvalue(scanned_seq)){
double corr_pvalue = 1.0 - pow(
1.0 - get_scanned_sequence_pvalue(scanned_seq),
get_scanned_sequence_num_scanned_positions(scanned_seq)
);
set_scanned_sequence_pvalue(scanned_seq, corr_pvalue);
}
}
// normalize if requested
if (normalize_scores) {
set_scanned_sequence_score(scanned_seq,
get_scanned_sequence_score(scanned_seq) / maxscore
);
}
}
}
}
/*************************************************************************
* Entry point for ama
*************************************************************************/
int main(int argc, char **argv) {
AMA_OPTIONS_T options;
ARRAYLST_T *motifs;
clock_t c0, c1; // measuring cpu_time
MOTIF_AND_PSSM_T *combo;
CISML_T *cisml;
PATTERN_T** patterns;
PATTERN_T *pattern;
FILE *fasta_file, *text_output, *cisml_output;
int i, seq_loading_num, seq_counter, unique_seqs, seq_len, scan_len, x1, x2, y1, y2;
char *seq_name, *path;
bool need_postprocessing, created;
SEQ_T *sequence;
RBTREE_T *seq_ids;
RBNODE_T *seq_node;
double *logcumback;
ALPH_T *alph;
// process the command
process_command_line(argc, argv, &options);
// load DNA motifs
motifs = load_motifs(&options);
// get the alphabet
if (arraylst_size(motifs) > 0) {
combo = (MOTIF_AND_PSSM_T*)arraylst_get(0, motifs);
alph = alph_hold(get_motif_alph(combo->motif));
} else {
alph = alph_dna();
}
// pick columns for GC operations
x1 = -1; x2 = -1; y1 = -1; y2 = -1;
if (alph_size_core(alph) == 4 && alph_size_pairs(alph) == 2) {
x1 = 0; // A
x2 = alph_complement(alph, x1); // T
y1 = (x2 == 1 ? 2 : 1); // C
y2 = alph_complement(alph, y1); // G
assert(x1 != x2 && y1 != y2 && x1 != y1 && x2 != y2 && x1 != y2 && x2 != y1);
}
// record starting time
c0 = clock();
// Create cisml data structure for recording results
cisml = allocate_cisml(PROGRAM_NAME, options.command_line, options.motif_filename, options.fasta_filename);
set_cisml_background_file(cisml, options.bg_filename);
// make a CISML pattern to hold scores for each motif
for (i = 0; i < arraylst_size(motifs); i++) {
combo = (MOTIF_AND_PSSM_T*)arraylst_get(i, motifs);
add_cisml_pattern(cisml, allocate_pattern(get_motif_id(combo->motif), ""));
}
// Open the FASTA file for reading.
fasta_file = NULL;
if (!open_file(options.fasta_filename, "r", false, "FASTA", "sequences", &fasta_file)) {
die("Couldn't open the file %s.\n", options.fasta_filename);
}
if (verbosity >= NORMAL_VERBOSE) {
if (options.last == 0) {
fprintf(stderr, "Using entire sequence\n");
} else {
fprintf(stderr, "Limiting sequence to last %d positions.\n", options.last);
}
}
//
// Read in all sequences and score with all motifs
//
seq_loading_num = 0; // keeps track on the number of sequences read in total
seq_counter = 0; // holds the index to the seq in the pattern
unique_seqs = 0; // keeps track on the number of unique sequences
need_postprocessing = false;
sequence = NULL;
logcumback = NULL;
seq_ids = rbtree_create(rbtree_strcasecmp,rbtree_strcpy,free,rbtree_intcpy,free);
while (read_one_fasta(alph, fasta_file, options.max_seq_length, &sequence)) {
++seq_loading_num;
seq_name = get_seq_name(sequence);
seq_len = get_seq_length(sequence);
scan_len = (options.last != 0 ? options.last : seq_len);
// red-black trees are only required if duplicates should be combined
if (options.combine_duplicates){
//lookup seq id and create new entry if required, return sequence index
seq_node = rbtree_lookup(seq_ids, get_seq_name(sequence), true, &created);
if (created) { // assign it a loading number
rbtree_set(seq_ids, seq_node, &unique_seqs);
seq_counter = unique_seqs;
++unique_seqs;
} else {
seq_counter = *((int*)rbnode_get(seq_node));
}
}
//
// Set up sequence-dependent background model and compute
// log cumulative probability of sequence.
// This needs the sequence in raw format.
//
if (options.sdbg_order >= 0)
logcumback = log_cumulative_background(alph, options.sdbg_order, sequence);
// Index the sequence, throwing away the raw format and ambiguous characters
index_sequence(sequence, alph, SEQ_NOAMBIG);
// Get the GC content of the sequence if binning p-values by GC
// and store it in the sequence object.
if (options.num_gc_bins > 1) {
ARRAY_T *freqs = get_sequence_freqs(sequence, alph);
set_total_gc_sequence(sequence, get_array_item(y1, freqs) + get_array_item(y2, freqs)); // f(C) + f(G)
free_array(freqs); // clean up
} else {
set_total_gc_sequence(sequence, -1); // flag ignore
}
// Scan with motifs.
for (i = 0; i < arraylst_size(motifs); i++) {
pattern = get_cisml_patterns(cisml)[i];
combo = (MOTIF_AND_PSSM_T*)arraylst_get(i, motifs);
if (verbosity >= HIGHER_VERBOSE) {
fprintf(stderr, "Scanning %s sequence with length %d "
"abbreviated to %d with motif %s with length %d.\n",
seq_name, seq_len, scan_len,
get_motif_id(combo->motif), get_motif_length(combo->motif));
}
SCANNED_SEQUENCE_T* scanned_seq = NULL;
if (!options.combine_duplicates || get_pattern_num_scanned_sequences(pattern) <= seq_counter) {
// Create a scanned_sequence record and save it in the pattern.
scanned_seq = allocate_scanned_sequence(seq_name, seq_name, pattern);
set_scanned_sequence_length(scanned_seq, scan_len);
} else {
// get existing sequence record
scanned_seq = get_pattern_scanned_sequences(pattern)[seq_counter];
set_scanned_sequence_length(scanned_seq, max(scan_len, get_scanned_sequence_length(scanned_seq)));
}
// check if scanned component of sequence has sufficient length for the motif
if (scan_len < get_motif_length(combo->motif)) {
// set score to zero and p-value to 1 if not set yet
if(!has_scanned_sequence_score(scanned_seq)){
set_scanned_sequence_score(scanned_seq, 0.0);
}
if(options.pvalues && !has_scanned_sequence_pvalue(scanned_seq)){
set_scanned_sequence_pvalue(scanned_seq, 1.0);
}
add_scanned_sequence_scanned_position(scanned_seq);
if (get_scanned_sequence_num_scanned_positions(scanned_seq) > 0L) {
need_postprocessing = true;
}
if (verbosity >= HIGH_VERBOSE) {
fprintf(stderr, "%s too short for motif %s. Score set to 0.\n",
seq_name, get_motif_id(combo->motif));
}
} else {
// scan the sequence using average/maximum motif affinity
ama_sequence_scan(alph, sequence, logcumback, combo->pssm_pair,
options.scoring, options.pvalues, options.last, scanned_seq,
&need_postprocessing);
}
} // All motifs scanned
free_seq(sequence);
if (options.sdbg_order >= 0) myfree(logcumback);
} // read sequences
fclose(fasta_file);
if (verbosity >= HIGH_VERBOSE) fprintf(stderr, "(%d) sequences read in.\n", seq_loading_num);
if (verbosity >= NORMAL_VERBOSE) fprintf(stderr, "Finished \n");
// if any sequence identifier was multiple times in the sequence set then
// postprocess of the data is required
if (need_postprocessing || options.normalize_scores) {
post_process(cisml, motifs, options.normalize_scores);
}
// output results
if (options.output_format == DIRECTORY_FORMAT) {
if (create_output_directory(options.out_dir, options.clobber, verbosity > QUIET_VERBOSE)) {
// only warn in higher verbose modes
fprintf(stderr, "failed to create output directory `%s' or already exists\n", options.out_dir);
exit(1);
}
path = make_path_to_file(options.out_dir, text_filename);
//FIXME check for errors: MEME doesn't either and we at least know we have a good directory
text_output = fopen(path, "w");
free(path);
path = make_path_to_file(options.out_dir, cisml_filename);
//FIXME check for errors
cisml_output = fopen(path, "w");
free(path);
print_cisml(cisml_output, cisml, true, NULL, false);
print_score(cisml, text_output);
fclose(cisml_output);
fclose(text_output);
} else if (options.output_format == GFF_FORMAT) {
print_score(cisml, stdout);
} else if (options.output_format == CISML_FORMAT) {
print_cisml(stdout, cisml, true, NULL, false);
} else {
die("Output format invalid!\n");
}
//
// Clean up.
//
rbtree_destroy(seq_ids);
arraylst_destroy(motif_and_pssm_destroy, motifs);
free_cisml(cisml);
rbtree_destroy(options.selected_motifs);
alph_release(alph);
// measure time
if (verbosity >= NORMAL_VERBOSE) { // starting time
c1 = clock();
fprintf(stderr, "cycles (CPU); %ld cycles\n", (long) c1);
fprintf(stderr, "elapsed CPU time: %f seconds\n", (float) (c1-c0) / CLOCKS_PER_SEC);
}
return 0;
}
/**********************************************************************
ama_sequence_scan()
Scan a given sequence with a specified motif using either
average motif affinity scoring or maximum one. In addition z-scores
may be calculated. Also the scan can be limited to only the end of
the passed sequences.
The motif has to be converted to odds in advance (in order
to speed up the scanning).
The result will be stored in the scanned_sequence parameter.
**********************************************************************/
void ama_sequence_scan(
ALPH_T* alph, // alphabet
SEQ_T *sequence, // the sequence to scan (IN)
double *logcumback, // cumulative bkg probability of sequence (IN)
PSSM_PAIR_T *pssm_pair, // the pos/neg pssms (IN)
int scoring, // AVG_ODDS or MAX_ODDS (IN)
BOOLEAN_T pvalues, // compute p-values (IN)
int last, // use only last <n> sequence positions
// or 0 if all positions should be used
SCANNED_SEQUENCE_T* scanned_seq,// the scanned sequence results (OUT)
BOOLEAN_T* need_postprocessing // Flag indicating the need for postprocessing (OUT)
)
{
assert(sequence != NULL);
assert(pssm_pair != NULL);
// FLAG indicates if sequence was suitable for motif matching
BOOLEAN_T isFeasible = true;
// Score the sequence.
double odds = score_sequence(alph, sequence, logcumback,
pssm_pair, scoring, last, &isFeasible);
// Compute the p-value of the AVG_ODDS score.
if (get_scanned_sequence_num_scanned_positions(scanned_seq) == 0L) {
set_scanned_sequence_score(scanned_seq, odds);
// sequence has not been scanned before
if (!isFeasible) {
if (verbosity >= NORMAL_VERBOSE) {
fprintf(stderr,"Sequence '%s' not suited for motif. P-value "
"set to 1.0!\n", get_scanned_sequence_accession(scanned_seq));
}
set_scanned_sequence_pvalue(scanned_seq, 1.0);
} else if (odds < 0.0){
if (verbosity >= NORMAL_VERBOSE) {
fprintf(stderr,"Sequence '%s' got invalid (negative) odds "
"score. P-value set to 1.0!\n",
get_scanned_sequence_accession(scanned_seq));
}
set_scanned_sequence_pvalue(scanned_seq, 1.0);
} else if (pvalues && scoring == AVG_ODDS) {
double pvalue = get_ama_pv(odds, get_scanned_sequence_length(scanned_seq),
get_total_gc_sequence(sequence), pssm_pair);
set_scanned_sequence_pvalue(scanned_seq, pvalue);
}
// scanned_position is used to keep track how often a sequence has been scored
// this feature is used in downstream gomo where a one2many homolog relationship
// is encoded through the same sequence identifier
add_scanned_sequence_scanned_position(scanned_seq);
} else {
// sequence has been scored before
if(!has_scanned_sequence_score(scanned_seq)) {
// no score set yet, so do
set_scanned_sequence_score(scanned_seq, odds);
} else {
// sum scores (take average later)
set_scanned_sequence_score(scanned_seq, odds +
get_scanned_sequence_score(scanned_seq));
}
if (!isFeasible) {
if (verbosity >= NORMAL_VERBOSE) {
fprintf(stderr,"Sequence '%s' not suited for motif. P-value set "
"to 1.0!\n", get_scanned_sequence_accession(scanned_seq));
}
if (!has_scanned_sequence_pvalue(scanned_seq)) {
set_scanned_sequence_pvalue(scanned_seq, 1.0);
}
} else if (odds < 0.0) {
if (verbosity >= NORMAL_VERBOSE) {
fprintf(stderr,"Sequence '%s' got invalid (negative) odds score. "
"P-value set to 1.0!\n", get_scanned_sequence_accession(scanned_seq));
}
if (!has_scanned_sequence_pvalue(scanned_seq)) {
set_scanned_sequence_pvalue(scanned_seq, 1.0);
}
} else if (pvalues && scoring == AVG_ODDS) {
double pvalue = get_ama_pv(odds, get_scanned_sequence_length(scanned_seq),
get_total_gc_sequence(sequence), pssm_pair);
if (!has_scanned_sequence_pvalue(scanned_seq)) {
set_scanned_sequence_pvalue(scanned_seq, pvalue);
} else {
// keep minimum p-value only
set_scanned_sequence_pvalue(scanned_seq, min(pvalue,
get_scanned_sequence_pvalue(scanned_seq)));
}
}
add_scanned_sequence_scanned_position(scanned_seq);
*need_postprocessing = true;
}
} // ama_sequence_scan
/*************************************************************************
* Entry point for ama
*************************************************************************/
int main(int argc, char *argv[]) {
int max_seq_length = MAX_SEQ;
STRING_LIST_T* selected_motifs = NULL;
double pseudocount = 0.01;
int output_format = CISML_FORMAT;
program_name = "ama";
int scoring = AVG_ODDS;
BOOLEAN_T pvalues = FALSE;
BOOLEAN_T normalize_scores = FALSE;
BOOLEAN_T combine_duplicates = FALSE;
int num_gc_bins = 1;
int sdbg_order = -1; // don't use sequence background
BOOLEAN_T scan_both_strands = TRUE;
ARRAY_T* pos_bg_freqs = NULL;
ARRAY_T* rev_bg_freqs = NULL;
clock_t c0, c1; /* measuring cpu_time */
CISML_T *cisml;
char * out_dir = NULL;
BOOLEAN_T clobber = FALSE;
int i;
int last = 0;
ALPH_T alph = INVALID_ALPH;
/**********************************************
* COMMAND LINE PROCESSING
**********************************************/
const int num_options = 16;
cmdoption const motif_scan_options[] = {
{ "max-seq-length", REQUIRED_VALUE },
{ "motif", REQUIRED_VALUE },
{ "motif-pseudo", REQUIRED_VALUE },
{ "rma", NO_VALUE },
{ "pvalues", NO_VALUE },
{ "sdbg", REQUIRED_VALUE },
{ "norc", NO_VALUE },
{ "cs", NO_VALUE },
{ "o-format", REQUIRED_VALUE },
{ "o", REQUIRED_VALUE },
{ "oc", REQUIRED_VALUE },
{ "scoring", REQUIRED_VALUE },
{ "verbosity", REQUIRED_VALUE },
{ "gcbins", REQUIRED_VALUE },
{ "last", REQUIRED_VALUE },
{ "version", NO_VALUE }
};
int option_index = 0;
// Define the usage message.
char usage[] = "USAGE: ama [options] <motif file> <sequence file> [<background file>]\n"
"\n"
" Options:\n"
" --sdbg <order>\t\t\tUse Markov background model of\n"
" \t\t\t\t\torder <order> derived from the sequence\n"
" \t\t\t\t\tto compute its likelihood ratios.\n"
" \t\t\t\t\tOverrides --pvalues, --gcbins and --rma;\n"
" \t\t\t\t\t<background file> is required unless\n"
" \t\t\t\t\t--sdbg is given.\n"
" --motif <id>\t\t\tUse only the motif identified by <id>.\n"
" \t\t\t\t\tThis option may be repeated.\n"
" --motif-pseudo <float>\t\tThe value <float> times the background\n"
" \t\t\t\t\tfrequency is added to the count of each\n"
" \t\t\t\t\tletter when creating the likelihood \n"
" \t\t\t\t\tratio matrix (default: %g).\n"
" --norc\t\t\t\tDisables the scanning of the reverse\n"
" \t\t\t\t\tcomplement strand.\n"
" --scoring [avg-odds|max-odds]\tIndicates whether the average or \n"
" \t\t\t\t\tthe maximum odds should be calculated\n"
" \t\t\t\t\t(default: avg-odds)\n"
" --rma\t\t\t\tScale motif scores to the range 0-1.\n"
" \t\t\t\t\t(Relative Motif Affinity).\n"
" \t\t\t\t\tMotif scores are scaled by the maximum\n"
" \t\t\t\t\tscore achievable by that PWM. (default:\n"
" \t\t\t\t\tmotif scores are not normalized)\n"
" --pvalues\t\t\t\tPrint p-value of avg-odds score in cisml\n"
" \t\t\t\t\toutput. Ignored for max-odds scoring.\n"
" \t\t\t\t\t(default: p-values are not printed)\n"
" --gcbins <bins>\t\t\tCompensate p-values for GC content of\n"
" \t\t\t\t\teach sequence using given number of \n"
" \t\t\t\t\tGC range bins. Recommended bins: 41.\n"
" \t\t\t\t\t(default: p-values are based on\n"
" \t\t\t\t\tfrequencies in background file)\n"
" --cs\t\t\t\tEnable combining sequences with same\n"
" \t\t\t\t\tidentifier by taking the average score\n"
" \t\t\t\t\tand the Sidac corrected p-value.\n"
" --o-format [gff|cisml]\t\tOutput file format (default: cisml)\n"
" \t\t\t\t\tignored if --o or --oc option used\n"
" --o <directory>\t\t\tOutput all available formats to\n"
" \t\t\t\t\t<directory>; give up if <directory>\n"
" \t\t\t\t\texists\n"
" --oc <directory>\t\t\tOutput all available formats to\n"
" \t\t\t\t\t<directory>; if <directory> exists\n"
" \t\t\t\t\toverwrite contents\n"
" --verbosity [1|2|3|4]\t\tControls amount of screen output\n"
" \t\t\t\t\t(default: %d)\n"
" --max-seq-length <int>\t\tSet the maximum length allowed for \n"
" \t\t\t\t\tinput sequences. (default: %d)\n"
" --last <int>\t\t\tUse only scores of (up to) last <n>\n"
" \t\t\t\t\tsequence positions to compute AMA.\n"
" --version \t\t\tPrint version and exit.\n"
"\n";
// Parse the command line.
if (simple_setopt(argc, argv, num_options, motif_scan_options) != NO_ERROR) {
die("Error processing command line options: option name too long.\n");
}
BOOLEAN_T setoutputformat = FALSE;
BOOLEAN_T setoutputdirectory = FALSE;
while (TRUE) {
int c = 0;
char* option_name = NULL;
char* option_value = NULL;
const char * message = NULL;
// Read the next option, and break if we're done.
c = simple_getopt(&option_name, &option_value, &option_index);
if (c == 0) {
break;
} else if (c < 0) {
(void) simple_getopterror(&message);
die("Error processing command line options (%s).\n", message);
} else if (strcmp(option_name, "max-seq-length") == 0) {
max_seq_length = atoi(option_value);
} else if (strcmp(option_name, "norc") == 0) {
scan_both_strands = FALSE;
} else if (strcmp(option_name, "cs") == 0) {
combine_duplicates = TRUE;
} else if (strcmp(option_name, "motif") == 0) {
if (selected_motifs == NULL) {
selected_motifs = new_string_list();
}
add_string(option_value, selected_motifs);
} else if (strcmp(option_name, "motif-pseudo") == 0) {
pseudocount = atof(option_value);
} else if (strcmp(option_name, "o-format") == 0) {
if (setoutputdirectory) {
if (verbosity >= NORMAL_VERBOSE)
fprintf(stderr, "output directory specified, ignoring --o-format\n");
} else {
setoutputformat = TRUE;
if (strcmp(option_value, "gff") == 0)
output_format = GFF_FORMAT;
else if (strcmp(option_value, "cisml") == 0)
output_format = CISML_FORMAT;
else {
if (verbosity >= NORMAL_VERBOSE)
fprintf(stderr, "Output format not known. Using standard instead (cisML).\n");
output_format = CISML_FORMAT;
}
}
} else if (strcmp(option_name, "o") == 0 || strcmp(option_name, "oc") == 0) {
setoutputdirectory = TRUE;
if (setoutputformat) {
if (verbosity >= NORMAL_VERBOSE)
fprintf(stderr, "output directory specified, ignoring --o-format\n");
}
clobber = strcmp(option_name, "oc") == 0;
out_dir = (char*) malloc (sizeof(char)*(strlen(option_value)+1));
strcpy(out_dir, option_value);
output_format = DIRECTORY_FORMAT;
} else if (strcmp(option_name, "verbosity") == 0) {
verbosity = atoi(option_value);
} else if (strcmp(option_name, "scoring") == 0) {
if (strcmp(option_value, "max-odds") == 0)
scoring = MAX_ODDS;
else if (strcmp(option_value, "avg-odds") == 0)
scoring = AVG_ODDS;
else if (strcmp(option_value, "sum-odds") == 0)
scoring = SUM_ODDS;
else
die("Specified scoring scheme not known.\n", message);
} else if (strcmp(option_name, "pvalues") == 0) {
pvalues = TRUE;
} else if (strcmp(option_name, "rma") == 0) {
normalize_scores = TRUE;
fprintf(stderr, "Normalizing motif scores using RMA method.\n");
} else if (strcmp(option_name, "gcbins") == 0) {
num_gc_bins = atoi(option_value);
pvalues = TRUE;
if (num_gc_bins <= 1) die("Number of bins in --gcbins must be greater than 1.\n", message);
} else if (strcmp(option_name, "sdbg") == 0) {
sdbg_order = atoi(option_value); // >=0 means use sequence bkg
}
else if (strcmp(option_name, "last") == 0) {
int i = 0;
if (option_value[0] == '-') ++i;
while (option_value[i] != '\0') {
if (!isdigit(option_value[i])) {
die("Specified parameter 'last' contains non-numeric characters.\n");
}
++i;
}
last = atoi(option_value);
if (errno != 0) {
die("Specified parameter 'last' could not be parsed as a number as:\n%s\n",strerror(errno));
}
if (last < 0) {
die("Specified parameter 'last' had negative value (%d) when only postive or zero values are allowed \n", last);
}
}
else if (strcmp(option_name, "version") == 0) {
fprintf(stdout, VERSION "\n");
exit(EXIT_SUCCESS);
}
}
// --sdbg overrides --pvalues and --gcbins and --rma
int req_args = 3;
if (sdbg_order >= 0) {
pvalues = FALSE;
normalize_scores = FALSE;
num_gc_bins = 1;
req_args = 2;
}
// Check all required arguments given
if (sdbg_order >= 0 && argc > option_index + req_args) {
die("<background file> cannot be given together with --sdbg.\n");
} else if (argc != option_index + req_args) {
fprintf(stderr, usage, pseudocount, verbosity, max_seq_length);
exit(EXIT_FAILURE);
}
// Get required arguments.
char* motif_filename = argv[option_index];
option_index++;
char* fasta_filename = argv[option_index];
option_index++;
char* bg_filename;
if (req_args == 3) { // required unless --sdbg given
bg_filename = argv[option_index];
option_index++;
} else {
bg_filename = "--uniform--"; // So PSSMs will use uniform background;
// we can multiply them out later.
}
// measure time
c0 = clock();
// Set up hash tables for computing reverse complement if doing --sdbg
if (sdbg_order >= 0) setup_hash_alph(DNAB);
// Create cisml data structure for recording results
cisml = allocate_cisml(program_name, motif_filename, fasta_filename);
set_cisml_background_file(cisml, bg_filename);
/**********************************************
* Read the motifs and background model.
**********************************************/
int num_motifs = 0;
MREAD_T *mread;
ARRAYLST_T *motifs;
PSSM_PAIR_T** pssm_pairs; // note pssm_pairs is an array of pointers
//this reads any meme file, xml, txt and html
mread = mread_create(motif_filename, OPEN_MFILE);
mread_set_bg_source(mread, bg_filename);
mread_set_pseudocount(mread, pseudocount);
motifs = mread_load(mread, NULL);
alph = mread_get_alphabet(mread);
pos_bg_freqs = mread_get_background(mread);
mread_destroy(mread);
num_motifs = arraylst_size(motifs);
// allocate memory for PSSM pairs
pssm_pairs = (PSSM_PAIR_T**)mm_malloc(sizeof(PSSM_PAIR_T*) * num_motifs);
if (verbosity >= NORMAL_VERBOSE)
fprintf(stderr, "Number of motifs in file %d.\n", num_motifs);
// make a CISML pattern to hold scores for each motif
PATTERN_T** patterns = NULL;
Resize(patterns, num_motifs, PATTERN_T*);
int motif_index;
for (motif_index = 0; motif_index < num_motifs; motif_index++) {
MOTIF_T* motif = (MOTIF_T*)arraylst_get(motif_index, motifs);
patterns[motif_index] = allocate_pattern(get_motif_id(motif), "");
add_cisml_pattern(cisml, patterns[motif_index]);
}
// make reverse complement motifs and background frequencies.
if (scan_both_strands == TRUE) {
add_reverse_complements(motifs);
assert(arraylst_size(motifs) == (2 * num_motifs));
rev_bg_freqs = allocate_array(get_array_length(pos_bg_freqs));
complement_dna_freqs(pos_bg_freqs, rev_bg_freqs);
}
/**************************************************************
* Convert motif matrices into log-odds matrices.
* Scale them.
* Compute the lookup tables for the PDF of scaled log-odds scores.
**************************************************************/
int ns = scan_both_strands ? 2 : 1; // number of strands
for (motif_index = 0; motif_index < num_motifs; motif_index++) {
MOTIF_T *motif, *motif_rc;
motif = (MOTIF_T*)arraylst_get(motif_index*ns, motifs);
if (scan_both_strands)
motif_rc = (MOTIF_T*)arraylst_get(motif_index*ns + 1, motifs);
else
motif_rc = NULL;
/*
* Note: If scanning both strands, we complement the motif frequencies
* but not the background frequencies so the motif looks the same.
* However, the given frequencies are used in computing the p-values
* since they represent the frequencies on the negative strands.
* (If we instead were to complement the input sequence, keeping the
* the motif fixed, we would need to use the complemented frequencies
* in computing the p-values. Is that any clearer?)
*/
double range = 300; // 100 is not very good; 1000 is great but too slow
PSSM_T* pos_pssm =
build_motif_pssm(
motif,
pos_bg_freqs,
pos_bg_freqs,
NULL, // Priors not used
0.0L, // alpha not used
range,
num_gc_bins,
TRUE
);
PSSM_T* neg_pssm = (scan_both_strands ?
build_motif_pssm(
motif_rc,
rev_bg_freqs,
pos_bg_freqs,
NULL, // Priors not used
0.0L, // alpha not used
range,
num_gc_bins,
TRUE
)
: NULL
);
pssm_pairs[motif_index] = create_pssm_pair(pos_pssm, neg_pssm);
}
// Open the FASTA file for reading.
FILE* fasta_file = NULL;
if (open_file(fasta_filename, "r", FALSE, "FASTA", "sequences", &fasta_file) == 0) {
die("Couldn't open the file %s.\n", fasta_filename);
}
if (verbosity >= NORMAL_VERBOSE) {
if (last == 0) {
fprintf(stderr, "Using entire sequence\n");
} else {
fprintf(stderr, "Limiting sequence to last %d positions.\n", last);
}
}
/**************************************************************
* Read in all sequences and score with all motifs
**************************************************************/
int seq_loading_num = 0; // keeps track on the number of sequences read in total
int seq_counter = 0; // holds the index to the seq in the pattern
int unique_seqs = 0; // keeps track on the number of unique sequences
BOOLEAN_T need_postprocessing = FALSE;
SEQ_T* sequence = NULL;
RBTREE_T* seq_ids = rbtree_create(rbtree_strcasecmp,NULL,free,rbtree_intcpy,free);
RBNODE_T* seq_node;
BOOLEAN_T created;
while (read_one_fasta(alph, fasta_file, max_seq_length, &sequence)) {
++seq_loading_num;
created = FALSE;
char* seq_name = get_seq_name(sequence);
int seq_len = get_seq_length(sequence);
int scan_len;
if (last != 0) {
scan_len = last;
} else {
scan_len = seq_len;
}
// red-black trees are only required if duplicates should be combined
if (combine_duplicates){
//lookup seq id and create new entry if required, return sequence index
char *tmp_id = mm_malloc(strlen(seq_name)+1); // required copy for rb-tree
strncpy(tmp_id,seq_name,strlen(seq_name)+1);
seq_node = rbtree_lookup(seq_ids, tmp_id, TRUE, &created);
if (created) {// assign it a loading number
rbtree_set(seq_ids, seq_node, &unique_seqs);
seq_counter = unique_seqs;
++unique_seqs;
} else {
seq_counter = *((int*)rbnode_get(seq_node));
}
}
//
// Set up sequence-dependent background model and compute
// log cumulative probability of sequence.
//
double *logcumback = NULL; // array of log cumulative probs.
if (sdbg_order >= 0) {
Resize(logcumback, seq_len+1, double);
char* raw_seq = get_raw_sequence(sequence);
BOOLEAN rc = FALSE;
double *a_cp = get_markov_from_sequence(raw_seq, alph_string(alph), rc, sdbg_order, 0);
log_cum_back(raw_seq, a_cp, sdbg_order, logcumback);
myfree(a_cp);
}
// Get the GC content of the sequence if binning p-values by GC
// and store it in the sequence object.
if (num_gc_bins > 1) {
ARRAY_T *freqs = get_sequence_freqs(sequence, alph);
set_total_gc_sequence(sequence,
get_array_item(1,freqs) + get_array_item(2,freqs)); // f(C) + f(G)
free_array(freqs); // clean up
} else {
set_total_gc_sequence(sequence, -1); // flag ignore
}
/**************************************************************
* Process all motifs.
**************************************************************/
int ns = scan_both_strands ? 2 : 1;
for (motif_index = 0; motif_index < num_motifs; motif_index++) {
PATTERN_T *pattern = patterns[motif_index];
MOTIF_T* motif = (MOTIF_T*)arraylst_get(ns*motif_index, motifs);
char* motif_id = (scan_both_strands ? get_motif_st_id(motif) : get_motif_id(motif));
if (verbosity >= HIGH_VERBOSE) {
fprintf(stderr, "Using motif %s of width %d.\n", motif_id, get_motif_length(motif));
}
if ((selected_motifs == NULL) || (have_string(get_motif_id(motif), selected_motifs) == TRUE)) {
if (verbosity >= HIGHER_VERBOSE) {
fprintf(stderr, "Scanning %s sequence with length %d "
"abbreviated to %d with motif %s with length %d.\n",
seq_name, seq_len, scan_len, motif_id, get_motif_length(motif));
}
SCANNED_SEQUENCE_T* scanned_seq = NULL;
if (!combine_duplicates || get_pattern_num_scanned_sequences(pattern) <= seq_counter){
// Create a scanned_sequence record and save it in the pattern.
scanned_seq = allocate_scanned_sequence(seq_name, seq_name, pattern);
set_scanned_sequence_length(scanned_seq, scan_len);
} else {
// get existing sequence record
scanned_seq = get_pattern_scanned_sequences(pattern)[seq_counter];
set_scanned_sequence_length(scanned_seq, max(scan_len, get_scanned_sequence_length(scanned_seq)));
}
// check if scanned component of sequence has sufficient length for the motif
if (scan_len < get_motif_length(motif)) {
// set score to zero and p-value to 1 if not set yet
if(!has_scanned_sequence_score(scanned_seq)){
set_scanned_sequence_score(scanned_seq, 0.0);
}
if(pvalues && !has_scanned_sequence_pvalue(scanned_seq)){
set_scanned_sequence_pvalue(scanned_seq, 1.0);
}
add_scanned_sequence_scanned_position(scanned_seq);
if (get_scanned_sequence_num_scanned_positions(scanned_seq) > 0L) need_postprocessing = TRUE;
if (verbosity >= HIGH_VERBOSE) fprintf(stderr, "%s too short for motif %s. Score set to 0!\n", seq_name, motif_id);
} else {
// scan the sequence using average/maximum motif affinity
ama_sequence_scan(alph, sequence, logcumback, pssm_pairs[motif_index], scoring,
pvalues, last, scanned_seq, &need_postprocessing);
}
} else {
if (verbosity >= HIGH_VERBOSE) fprintf(stderr, "Skipping motif %s.\n", motif_id);
}
} // All motifs parsed
free_seq(sequence);
if (sdbg_order >= 0) myfree(logcumback);
} // read sequences