int main(int argc, char **argv) {
char **base_paths = NULL;
char **paths = NULL;
int i;
int pcre_opts = PCRE_MULTILINE;
int study_opts = 0;
double time_diff;
pthread_t *workers = NULL;
int workers_len;
set_log_level(LOG_LEVEL_WARN);
work_queue = NULL;
work_queue_tail = NULL;
memset(&stats, 0, sizeof(stats));
root_ignores = init_ignore(NULL, "", 0);
out_fd = stdout;
#ifdef USE_PCRE_JIT
int has_jit = 0;
pcre_config(PCRE_CONFIG_JIT, &has_jit);
if (has_jit) {
study_opts |= PCRE_STUDY_JIT_COMPILE;
}
#endif
gettimeofday(&(stats.time_start), NULL);
parse_options(argc, argv, &base_paths, &paths);
log_debug("PCRE Version: %s", pcre_version());
#ifdef _WIN32
{
SYSTEM_INFO si;
GetSystemInfo(&si);
workers_len = si.dwNumberOfProcessors;
}
#else
workers_len = (int)sysconf(_SC_NPROCESSORS_ONLN);
#endif
if (opts.literal) {
workers_len--;
}
if (opts.workers) {
workers_len = opts.workers;
}
if (workers_len < 1) {
workers_len = 1;
}
log_debug("Using %i workers", workers_len);
done_adding_files = FALSE;
workers = ag_calloc(workers_len, sizeof(pthread_t));
if (pthread_cond_init(&files_ready, NULL)) {
die("pthread_cond_init failed!");
}
if (pthread_mutex_init(&print_mtx, NULL)) {
die("pthread_mutex_init failed!");
}
if (pthread_mutex_init(&stats_mtx, NULL)) {
die("pthread_mutex_init failed!");
}
if (pthread_mutex_init(&work_queue_mtx, NULL)) {
die("pthread_mutex_init failed!");
}
if (opts.casing == CASE_SMART) {
opts.casing = is_lowercase(opts.query) ? CASE_INSENSITIVE : CASE_SENSITIVE;
}
if (opts.literal) {
if (opts.casing == CASE_INSENSITIVE) {
/* Search routine needs the query to be lowercase */
char *c = opts.query;
for (; *c != '\0'; ++c) {
*c = (char)tolower(*c);
}
}
generate_alpha_skip(opts.query, opts.query_len, alpha_skip_lookup, opts.casing == CASE_SENSITIVE);
find_skip_lookup = NULL;
generate_find_skip(opts.query, opts.query_len, &find_skip_lookup, opts.casing == CASE_SENSITIVE);
if (opts.word_regexp) {
init_wordchar_table();
opts.literal_starts_wordchar = is_wordchar(opts.query[0]);
opts.literal_ends_wordchar = is_wordchar(opts.query[opts.query_len - 1]);
}
} else {
if (opts.casing == CASE_INSENSITIVE) {
pcre_opts |= PCRE_CASELESS;
}
if (opts.word_regexp) {
char *word_regexp_query;
ag_asprintf(&word_regexp_query, "\\b%s\\b", opts.query);
free(opts.query);
opts.query = word_regexp_query;
opts.query_len = strlen(opts.query);
}
compile_study(&opts.re, &opts.re_extra, opts.query, pcre_opts, study_opts);
}
if (opts.search_stream) {
search_stream(stdin, "");
} else {
for (i = 0; i < workers_len; i++) {
int rv = pthread_create(&(workers[i]), NULL, &search_file_worker, &i);
if (rv != 0) {
die("error in pthread_create(): %s", strerror(rv));
}
}
for (i = 0; paths[i] != NULL; i++) {
log_debug("searching path %s for %s", paths[i], opts.query);
symhash = NULL;
ignores *ig = init_ignore(root_ignores, "", 0);
search_dir(ig, base_paths[i], paths[i], 0);
cleanup_ignore(ig);
}
pthread_mutex_lock(&work_queue_mtx);
done_adding_files = TRUE;
pthread_cond_broadcast(&files_ready);
pthread_mutex_unlock(&work_queue_mtx);
for (i = 0; i < workers_len; i++) {
if (pthread_join(workers[i], NULL)) {
die("pthread_join failed!");
}
}
}
if (opts.stats) {
gettimeofday(&(stats.time_end), NULL);
time_diff = ((long)stats.time_end.tv_sec * 1000000 + stats.time_end.tv_usec) -
((long)stats.time_start.tv_sec * 1000000 + stats.time_start.tv_usec);
time_diff /= 1000000;
printf("%ld matches\n%ld files searched\n%ld bytes searched\n%f seconds\n", stats.total_matches, stats.total_files, stats.total_bytes, time_diff);
}
if (opts.pager) {
pclose(out_fd);
}
cleanup_options();
pthread_cond_destroy(&files_ready);
pthread_mutex_destroy(&work_queue_mtx);
pthread_mutex_destroy(&stats_mtx);
pthread_mutex_destroy(&print_mtx);
cleanup_ignore(root_ignores);
free(workers);
for (i = 0; paths[i] != NULL; i++) {
free(paths[i]);
free(base_paths[i]);
}
free(base_paths);
free(paths);
if (find_skip_lookup) {
free(find_skip_lookup);
}
return !opts.match_found;
}
int main(int argc, char **argv) {
char **base_paths = NULL;
char **paths = NULL;
int i;
int pcre_opts = PCRE_MULTILINE;
int study_opts = 0;
double time_diff;
worker_t *workers = NULL;
int workers_len;
int num_cores;
#ifdef KJK_BUILD
extern void setup_crash_handler(); /* in kjk_crash_handler.cpp */
setup_crash_handler();
#endif
set_log_level(LOG_LEVEL_WARN);
work_queue = NULL;
work_queue_tail = NULL;
memset(&stats, 0, sizeof(stats));
root_ignores = init_ignore(NULL, "", 0);
out_fd = stdout;
#ifdef USE_PCRE_JIT
int has_jit = 0;
pcre_config(PCRE_CONFIG_JIT, &has_jit);
if (has_jit) {
study_opts |= PCRE_STUDY_JIT_COMPILE;
}
#endif
gettimeofday(&(stats.time_start), NULL);
parse_options(argc, argv, &base_paths, &paths);
log_debug("PCRE Version: %s", pcre_version());
#ifdef _WIN32
{
SYSTEM_INFO si;
GetSystemInfo(&si);
num_cores = si.dwNumberOfProcessors;
}
#else
num_cores = (int)sysconf(_SC_NPROCESSORS_ONLN);
#endif
workers_len = num_cores;
if (opts.literal) {
workers_len--;
}
if (opts.workers) {
workers_len = opts.workers;
}
if (workers_len < 1) {
workers_len = 1;
}
log_debug("Using %i workers", workers_len);
done_adding_files = FALSE;
workers = (worker_t *) ag_calloc(workers_len, sizeof(worker_t));
if (pthread_cond_init(&files_ready, NULL)) {
die("pthread_cond_init failed!");
}
if (pthread_mutex_init(&print_mtx, NULL)) {
die("pthread_mutex_init failed!");
}
if (pthread_mutex_init(&stats_mtx, NULL)) {
die("pthread_mutex_init failed!");
}
if (pthread_mutex_init(&work_queue_mtx, NULL)) {
die("pthread_mutex_init failed!");
}
if (opts.casing == CASE_SMART) {
opts.casing = is_lowercase(opts.query) ? CASE_INSENSITIVE : CASE_SENSITIVE;
}
if (opts.literal) {
if (opts.casing == CASE_INSENSITIVE) {
/* Search routine needs the query to be lowercase */
char *c = opts.query;
for (; *c != '\0'; ++c) {
*c = (char)tolower(*c);
}
}
generate_alpha_skip(opts.query, opts.query_len, alpha_skip_lookup, opts.casing == CASE_SENSITIVE);
find_skip_lookup = NULL;
generate_find_skip(opts.query, opts.query_len, &find_skip_lookup, opts.casing == CASE_SENSITIVE);
if (opts.word_regexp) {
init_wordchar_table();
opts.literal_starts_wordchar = is_wordchar(opts.query[0]);
opts.literal_ends_wordchar = is_wordchar(opts.query[opts.query_len - 1]);
}
} else {
if (opts.casing == CASE_INSENSITIVE) {
pcre_opts |= PCRE_CASELESS;
}
if (opts.word_regexp) {
char *word_regexp_query;
ag_asprintf(&word_regexp_query, "\\b%s\\b", opts.query);
free(opts.query);
opts.query = word_regexp_query;
opts.query_len = strlen(opts.query);
}
compile_study(&opts.re, &opts.re_extra, opts.query, pcre_opts, study_opts);
}
if (opts.search_stream) {
// search_stream(stdin, "");
} else {
for (i = 0; i < workers_len; i++) {
workers[i].id = i;
int rv = pthread_create(&(workers[i].thread), NULL, &search_file_worker, &(workers[i].id));
if (rv != 0) {
die("error in pthread_create(): %s", strerror(rv));
}
#if defined(HAVE_PTHREAD_SETAFFINITY_NP) && defined(USE_CPU_SET)
if (opts.use_thread_affinity) {
cpu_set_t cpu_set;
CPU_ZERO(&cpu_set);
CPU_SET(i % num_cores, &cpu_set);
rv = pthread_setaffinity_np(workers[i].thread, sizeof(cpu_set), &cpu_set);
if (rv != 0) {
die("error in pthread_setaffinity_np(): %s", strerror(rv));
}
log_debug("Thread %i set to CPU %i", i, i);
} else {
log_debug("Thread affinity disabled.");
}
#else
log_debug("No CPU affinity support.");
#endif
}
for (i = 0; paths[i] != NULL; i++) {
log_debug("searching path %s for %s", paths[i], opts.query);
symhash = NULL;
ignores *ig = init_ignore(root_ignores, "", 0);
struct stat s;
s.st_dev = 0;
#ifndef _WIN32
/* The device is ignored if opts.one_dev is false, so it's fine
* to leave it at the default 0
*/
if (opts.one_dev && lstat(paths[i], &s) == -1) {
log_err("Failed to get device information for path %s. Skipping...", paths[i]);
}
#endif
search_dir(ig, base_paths[i], paths[i], 0, s.st_dev);
cleanup_ignore(ig);
}
pthread_mutex_lock(&work_queue_mtx);
done_adding_files = TRUE;
pthread_cond_broadcast(&files_ready);
pthread_mutex_unlock(&work_queue_mtx);
for (i = 0; i < workers_len; i++) {
if (pthread_join(workers[i].thread, NULL)) {
die("pthread_join failed!");
}
}
}
if (opts.stats) {
gettimeofday(&(stats.time_end), NULL);
time_diff = ((long)stats.time_end.tv_sec * 1000000 + stats.time_end.tv_usec) -
((long)stats.time_start.tv_sec * 1000000 + stats.time_start.tv_usec);
time_diff /= 1000000;
printf("%ld matches\n%ld files searched\n%ld bytes searched\n%f seconds\n", stats.total_matches, stats.total_files, stats.total_bytes, time_diff);
}
if (opts.pager) {
pclose(out_fd);
}
cleanup_options();
pthread_cond_destroy(&files_ready);
pthread_mutex_destroy(&work_queue_mtx);
pthread_mutex_destroy(&stats_mtx);
pthread_mutex_destroy(&print_mtx);
cleanup_ignore(root_ignores);
free(workers);
for (i = 0; paths[i] != NULL; i++) {
free(paths[i]);
free(base_paths[i]);
}
free(base_paths);
free(paths);
if (find_skip_lookup) {
free(find_skip_lookup);
}
return !opts.match_found;
}
/*************************************************************************
* Entry point for centrimo
*************************************************************************/
int main(int argc, char *argv[]) {
CENTRIMO_OPTIONS_T options;
SEQ_SITES_T seq_sites;
SITE_COUNTS_T counts;
int seqN, motifN, seqlen, db_i, motif_i, i;
double log_pvalue_thresh;
SEQ_T** sequences = NULL;
ARRAY_T* bg_freqs = NULL;
ARRAYLST_T *stats_list;
MOTIF_DB_T **dbs, *db;
MREAD_T *mread;
MOTIF_STATS_T *stats;
MOTIF_T *motif, *rev_motif;
PSSM_T *pos_pssm, *rev_pssm;
char *sites_path, *desc;
FILE *sites_file;
HTMLWR_T *html;
JSONWR_T *json;
// COMMAND LINE PROCESSING
process_command_line(argc, argv, &options);
// load the sequences
read_sequences(options.alphabet, options.seq_source, &sequences, &seqN);
seqlen = (seqN ? get_seq_length(sequences[0]) : 0);
// calculate a sequence background (unless other background is given)
if (!options.bg_source) {
bg_freqs = calc_bg_from_fastas(options.alphabet, seqN, sequences);
}
// load the motifs
motifN = 0;
dbs = mm_malloc(sizeof(MOTIF_DB_T*) * arraylst_size(options.motif_sources));
for (i = 0; i < arraylst_size(options.motif_sources); i++) {
char* db_source;
db_source = (char*)arraylst_get(i, options.motif_sources);
dbs[i] = read_motifs(i, db_source, options.bg_source, &bg_freqs,
options.pseudocount, options.selected_motifs, options.alphabet);
motifN += arraylst_size(dbs[i]->motifs);
}
log_pvalue_thresh = log(options.evalue_thresh) - log(motifN);
// Setup some things for double strand scanning
if (options.scan_both_strands == TRUE) {
// Set up hash tables for computing reverse complement
setup_hash_alph(DNAB);
setalph(0);
// Correct background by averaging on freq. for both strands.
average_freq_with_complement(options.alphabet, bg_freqs);
normalize_subarray(0, alph_size(options.alphabet, ALPH_SIZE), 0.0, bg_freqs);
calc_ambigs(options.alphabet, FALSE, bg_freqs);
}
// Create output directory
if (create_output_directory(options.output_dirname, options.allow_clobber,
(verbosity >= NORMAL_VERBOSE))) {
die("Couldn't create output directory %s.\n", options.output_dirname);
}
// open output files
sites_path = make_path_to_file(options.output_dirname, SITES_FILENAME);
sites_file = fopen(sites_path, "w");
free(sites_path);
// setup html monolith writer
json = NULL;
if ((html = htmlwr_create(get_meme_etc_dir(), TEMPLATE_FILENAME))) {
htmlwr_set_dest_name(html, options.output_dirname, HTML_FILENAME);
htmlwr_replace(html, "centrimo_data.js", "data");
json = htmlwr_output(html);
if (json == NULL) die("Template does not contain data section.\n");
} else {
DEBUG_MSG(QUIET_VERBOSE, "Failed to open html template file.\n");
}
if (json) {
// output some top level variables
jsonwr_str_prop(json, "version", VERSION);
jsonwr_str_prop(json, "revision", REVISION);
jsonwr_str_prop(json, "release", ARCHIVE_DATE);
jsonwr_str_array_prop(json, "cmd", argv, argc);
jsonwr_property(json, "options");
jsonwr_start_object_value(json);
jsonwr_dbl_prop(json, "motif-pseudo", options.pseudocount);
jsonwr_dbl_prop(json, "score", options.score_thresh);
jsonwr_dbl_prop(json, "ethresh", options.evalue_thresh);
jsonwr_lng_prop(json, "maxbin", options.max_window+1);
jsonwr_bool_prop(json, "norc", !options.scan_both_strands);
jsonwr_bool_prop(json, "noflip", options.no_flip);
jsonwr_end_object_value(json);
// output the description
desc = prepare_description(&options);
if (desc) {
jsonwr_str_prop(json, "job_description", desc);
free(desc);
}
// output size metrics
jsonwr_lng_prop(json, "seqlen", seqlen);
jsonwr_lng_prop(json, "tested", motifN);
// output the fasta db
jsonwr_property(json, "sequence_db");
jsonwr_start_object_value(json);
jsonwr_str_prop(json, "source", options.seq_source);
jsonwr_lng_prop(json, "count", seqN);
jsonwr_end_object_value(json);
// output the motif dbs
jsonwr_property(json, "motif_dbs");
jsonwr_start_array_value(json);
for (db_i = 0; db_i < arraylst_size(options.motif_sources); db_i++) {
db = dbs[db_i];
jsonwr_start_object_value(json);
jsonwr_str_prop(json, "source", db->source);
jsonwr_lng_prop(json, "count", arraylst_size(db->motifs));
jsonwr_end_object_value(json);
}
jsonwr_end_array_value(json);
// start the motif array
jsonwr_property(json, "motifs");
jsonwr_start_array_value(json);
}
/**************************************************************
* Tally the positions of the best sites for each of the
* selected motifs.
**************************************************************/
// prepare the sequence sites
memset(&seq_sites, 0, sizeof(SEQ_SITES_T));
// prepare the site counts
counts.allocated = ((2 * seqlen) - 1);
counts.sites = mm_malloc(sizeof(double) * counts.allocated);
// prepare the motifs stats list
stats_list = arraylst_create();
// prepare the other vars
motif = NULL; pos_pssm = NULL; rev_motif = NULL; rev_pssm = NULL;
for (db_i = 0; db_i < arraylst_size(options.motif_sources); db_i++) {
db = dbs[db_i];
for (motif_i = 0; motif_i < arraylst_size(db->motifs); motif_i++) {
motif = (MOTIF_T *) arraylst_get(motif_i, db->motifs);
DEBUG_FMT(NORMAL_VERBOSE, "Using motif %s of width %d.\n",
get_motif_id(motif), get_motif_length(motif));
// reset the counts
for (i = 0; i < counts.allocated; i++) counts.sites[i] = 0;
counts.total_sites = 0;
// create the pssm
pos_pssm = make_pssm(bg_freqs, motif);
// If required, do the same for the reverse complement motif.
if (options.scan_both_strands) {
rev_motif = dup_rc_motif(motif);
rev_pssm = make_pssm(bg_freqs, rev_motif);
}
// scan the sequences
for (i = 0; i < seqN; i++)
score_sequence(&options, sequences[i], pos_pssm, rev_pssm,
&seq_sites, &counts);
// DEBUG check that the sum of the sites is close to the site count
double sum_check = 0, sum_diff;
for (i = 0; i < counts.allocated; i++) sum_check += counts.sites[i];
sum_diff = counts.total_sites - sum_check;
if (sum_diff < 0) sum_diff = -sum_diff;
if (sum_diff > 0.1) {
fprintf(stderr, "Warning: site counts don't sum to accurate value! "
"%g != %ld", sum_check, counts.total_sites);
}
// output the plain text site counts
output_site_counts(sites_file, seqlen, db, motif, &counts);
// compute the best central window
stats = compute_stats(options.max_window, seqlen, db, motif, &counts);
// check if it passes the threshold
if (json && stats->log_adj_pvalue <= log_pvalue_thresh) {
output_motif_json(json, stats, &counts);
arraylst_add(stats, stats_list);
} else {
free(stats);
}
// Free memory associated with this motif.
free_pssm(pos_pssm);
free_pssm(rev_pssm);
destroy_motif(rev_motif);
}
}
if (json) jsonwr_end_array_value(json);
// finish writing sites
fclose(sites_file);
// finish writing html file
if (html) {
if (htmlwr_output(html) != NULL) {
die("Found another JSON replacement!\n");
}
htmlwr_destroy(html);
}
// write text file
output_centrimo_text(&options, motifN, stats_list);
// Clean up.
for (i = 0; i < seqN; ++i) {
free_seq(sequences[i]);
}
free(sequences);
for (i = 0; i < arraylst_size(options.motif_sources); i++) {
free_db(dbs[i]);
}
free(dbs);
free_array(bg_freqs);
free(counts.sites);
free(seq_sites.sites);
arraylst_destroy(free, stats_list);
cleanup_options(&options);
return 0;
}
