int main(int argc, char **argv) {
char **paths = NULL;
int i;
int pcre_opts = PCRE_MULTILINE;
int study_opts = 0;
double time_diff;
pthread_t *workers = NULL;
int workers_len;
set_log_level(LOG_LEVEL_WARN);
work_queue = NULL;
work_queue_tail = NULL;
memset(&stats, 0, sizeof(stats));
root_ignores = init_ignore(NULL);
out_fd = stdout;
#ifdef USE_PCRE_JIT
int has_jit = 0;
pcre_config(PCRE_CONFIG_JIT, &has_jit);
if (has_jit) {
study_opts |= PCRE_STUDY_JIT_COMPILE;
}
#endif
gettimeofday(&(stats.time_start), NULL);
parse_options(argc, argv, &paths);
log_debug("PCRE Version: %s", pcre_version());
if (opts.workers) {
workers_len = opts.workers;
} else {
/* Experiments show that two worker threads appear to be optimal, both
* on dual-core and quad-core systems. See
* http://geoff.greer.fm/2012/09/07/the-silver-searcher-adding-pthreads/.
* On single-core CPUs, more than one worker thread makes no sense.
*/
int ncpus = (int)sysconf(_SC_NPROCESSORS_ONLN);
workers_len = (ncpus >= 2) ? 2 : 1;
}
log_debug("Using %i workers", workers_len);
done_adding_files = FALSE;
workers = ag_calloc(workers_len, sizeof(pthread_t));
if (pthread_cond_init(&files_ready, NULL)) {
log_err("pthread_cond_init failed!");
exit(2);
}
if (pthread_mutex_init(&print_mtx, NULL)) {
log_err("pthread_mutex_init failed!");
exit(2);
}
if (pthread_mutex_init(&stats_mtx, NULL)) {
log_err("pthread_mutex_init failed!");
exit(2);
}
if (pthread_mutex_init(&work_queue_mtx, NULL)) {
log_err("pthread_mutex_init failed!");
exit(2);
}
if (opts.casing == CASE_SMART) {
opts.casing = contains_uppercase(opts.query) ? CASE_SENSITIVE : CASE_INSENSITIVE;
}
if (opts.literal) {
if (opts.casing == CASE_INSENSITIVE) {
/* Search routine needs the query to be lowercase */
char *c = opts.query;
for (; *c != '\0'; ++c) {
*c = (char) tolower(*c);
}
}
generate_skip_lookup(opts.query, opts.query_len, skip_lookup, opts.casing == CASE_SENSITIVE);
if (opts.word_regexp) {
init_wordchar_table();
opts.literal_starts_wordchar = is_wordchar(opts.query[0]);
opts.literal_ends_wordchar = is_wordchar(opts.query[opts.query_len - 1]);
}
} else {
if (opts.casing == CASE_INSENSITIVE) {
pcre_opts = pcre_opts | PCRE_CASELESS;
}
if (opts.word_regexp) {
char *word_regexp_query;
asprintf(&word_regexp_query, "\\b%s\\b", opts.query);
free(opts.query);
opts.query = word_regexp_query;
opts.query_len = strlen(opts.query);
}
compile_study(&opts.re, &opts.re_extra, opts.query, pcre_opts, study_opts);
}
if (opts.search_stream) {
search_stream(stdin, "");
} else {
for (i = 0; i < workers_len; i++) {
int ptc_rc = pthread_create(&(workers[i]), NULL, &search_file_worker,
NULL);
check_err(ptc_rc, "create worker thread");
}
for (i = 0; paths[i] != NULL; i++) {
log_debug("searching path %s for %s", paths[i], opts.query);
search_dir(root_ignores, paths[i], 0);
}
done_adding_files = TRUE;
pthread_cond_broadcast(&files_ready);
for (i = 0; i < workers_len; i++) {
if (pthread_join(workers[i], NULL)) {
log_err("pthread_join failed!");
exit(2);
}
}
}
if (opts.stats) {
gettimeofday(&(stats.time_end), NULL);
time_diff = ((long)stats.time_end.tv_sec * 1000000 + stats.time_end.tv_usec) -
((long)stats.time_start.tv_sec * 1000000 + stats.time_start.tv_usec);
time_diff /= 1000000;
printf("%ld matches\n%ld files searched\n%ld bytes searched\n%f seconds\n", stats.total_matches, stats.total_files, stats.total_bytes, time_diff);
}
if (opts.pager) {
pclose(out_fd);
}
pthread_cond_destroy(&files_ready);
pthread_mutex_destroy(&work_queue_mtx);
pthread_mutex_destroy(&stats_mtx);
pthread_mutex_destroy(&print_mtx);
cleanup_ignore(root_ignores);
free(workers);
free(paths);
return 0;
}
int main(int argc, char **argv) {
char **base_paths = NULL;
char **paths = NULL;
int i;
int pcre_opts = PCRE_MULTILINE;
int study_opts = 0;
double time_diff;
pthread_t *workers = NULL;
int workers_len;
set_log_level(LOG_LEVEL_WARN);
work_queue = NULL;
work_queue_tail = NULL;
memset(&stats, 0, sizeof(stats));
root_ignores = init_ignore(NULL, "", 0);
out_fd = stdout;
#ifdef USE_PCRE_JIT
int has_jit = 0;
pcre_config(PCRE_CONFIG_JIT, &has_jit);
if (has_jit) {
study_opts |= PCRE_STUDY_JIT_COMPILE;
}
#endif
gettimeofday(&(stats.time_start), NULL);
parse_options(argc, argv, &base_paths, &paths);
log_debug("PCRE Version: %s", pcre_version());
#ifdef _WIN32
{
SYSTEM_INFO si;
GetSystemInfo(&si);
workers_len = si.dwNumberOfProcessors;
}
#else
workers_len = (int)sysconf(_SC_NPROCESSORS_ONLN);
#endif
if (opts.literal) {
workers_len--;
}
if (opts.workers) {
workers_len = opts.workers;
}
if (workers_len < 1) {
workers_len = 1;
}
log_debug("Using %i workers", workers_len);
done_adding_files = FALSE;
workers = ag_calloc(workers_len, sizeof(pthread_t));
if (pthread_cond_init(&files_ready, NULL)) {
die("pthread_cond_init failed!");
}
if (pthread_mutex_init(&print_mtx, NULL)) {
die("pthread_mutex_init failed!");
}
if (pthread_mutex_init(&stats_mtx, NULL)) {
die("pthread_mutex_init failed!");
}
if (pthread_mutex_init(&work_queue_mtx, NULL)) {
die("pthread_mutex_init failed!");
}
if (opts.casing == CASE_SMART) {
opts.casing = is_lowercase(opts.query) ? CASE_INSENSITIVE : CASE_SENSITIVE;
}
if (opts.literal) {
if (opts.casing == CASE_INSENSITIVE) {
/* Search routine needs the query to be lowercase */
char *c = opts.query;
for (; *c != '\0'; ++c) {
*c = (char)tolower(*c);
}
}
generate_alpha_skip(opts.query, opts.query_len, alpha_skip_lookup, opts.casing == CASE_SENSITIVE);
find_skip_lookup = NULL;
generate_find_skip(opts.query, opts.query_len, &find_skip_lookup, opts.casing == CASE_SENSITIVE);
if (opts.word_regexp) {
init_wordchar_table();
opts.literal_starts_wordchar = is_wordchar(opts.query[0]);
opts.literal_ends_wordchar = is_wordchar(opts.query[opts.query_len - 1]);
}
} else {
if (opts.casing == CASE_INSENSITIVE) {
pcre_opts |= PCRE_CASELESS;
}
if (opts.word_regexp) {
char *word_regexp_query;
ag_asprintf(&word_regexp_query, "\\b%s\\b", opts.query);
free(opts.query);
opts.query = word_regexp_query;
opts.query_len = strlen(opts.query);
}
compile_study(&opts.re, &opts.re_extra, opts.query, pcre_opts, study_opts);
}
if (opts.search_stream) {
search_stream(stdin, "");
} else {
for (i = 0; i < workers_len; i++) {
int rv = pthread_create(&(workers[i]), NULL, &search_file_worker, &i);
if (rv != 0) {
die("error in pthread_create(): %s", strerror(rv));
}
}
for (i = 0; paths[i] != NULL; i++) {
log_debug("searching path %s for %s", paths[i], opts.query);
symhash = NULL;
ignores *ig = init_ignore(root_ignores, "", 0);
search_dir(ig, base_paths[i], paths[i], 0);
cleanup_ignore(ig);
}
pthread_mutex_lock(&work_queue_mtx);
done_adding_files = TRUE;
pthread_cond_broadcast(&files_ready);
pthread_mutex_unlock(&work_queue_mtx);
for (i = 0; i < workers_len; i++) {
if (pthread_join(workers[i], NULL)) {
die("pthread_join failed!");
}
}
}
if (opts.stats) {
gettimeofday(&(stats.time_end), NULL);
time_diff = ((long)stats.time_end.tv_sec * 1000000 + stats.time_end.tv_usec) -
((long)stats.time_start.tv_sec * 1000000 + stats.time_start.tv_usec);
time_diff /= 1000000;
printf("%ld matches\n%ld files searched\n%ld bytes searched\n%f seconds\n", stats.total_matches, stats.total_files, stats.total_bytes, time_diff);
}
if (opts.pager) {
pclose(out_fd);
}
cleanup_options();
pthread_cond_destroy(&files_ready);
pthread_mutex_destroy(&work_queue_mtx);
pthread_mutex_destroy(&stats_mtx);
pthread_mutex_destroy(&print_mtx);
cleanup_ignore(root_ignores);
free(workers);
for (i = 0; paths[i] != NULL; i++) {
free(paths[i]);
free(base_paths[i]);
}
free(base_paths);
free(paths);
if (find_skip_lookup) {
free(find_skip_lookup);
}
return !opts.match_found;
}
int main(int argc, char **argv) {
char **base_paths = NULL;
char **paths = NULL;
int i;
int pcre_opts = PCRE_MULTILINE;
int study_opts = 0;
double time_diff;
worker_t *workers = NULL;
int workers_len;
int num_cores;
#ifdef KJK_BUILD
extern void setup_crash_handler(); /* in kjk_crash_handler.cpp */
setup_crash_handler();
#endif
set_log_level(LOG_LEVEL_WARN);
work_queue = NULL;
work_queue_tail = NULL;
memset(&stats, 0, sizeof(stats));
root_ignores = init_ignore(NULL, "", 0);
out_fd = stdout;
#ifdef USE_PCRE_JIT
int has_jit = 0;
pcre_config(PCRE_CONFIG_JIT, &has_jit);
if (has_jit) {
study_opts |= PCRE_STUDY_JIT_COMPILE;
}
#endif
gettimeofday(&(stats.time_start), NULL);
parse_options(argc, argv, &base_paths, &paths);
log_debug("PCRE Version: %s", pcre_version());
#ifdef _WIN32
{
SYSTEM_INFO si;
GetSystemInfo(&si);
num_cores = si.dwNumberOfProcessors;
}
#else
num_cores = (int)sysconf(_SC_NPROCESSORS_ONLN);
#endif
workers_len = num_cores;
if (opts.literal) {
workers_len--;
}
if (opts.workers) {
workers_len = opts.workers;
}
if (workers_len < 1) {
workers_len = 1;
}
log_debug("Using %i workers", workers_len);
done_adding_files = FALSE;
workers = (worker_t *) ag_calloc(workers_len, sizeof(worker_t));
if (pthread_cond_init(&files_ready, NULL)) {
die("pthread_cond_init failed!");
}
if (pthread_mutex_init(&print_mtx, NULL)) {
die("pthread_mutex_init failed!");
}
if (pthread_mutex_init(&stats_mtx, NULL)) {
die("pthread_mutex_init failed!");
}
if (pthread_mutex_init(&work_queue_mtx, NULL)) {
die("pthread_mutex_init failed!");
}
if (opts.casing == CASE_SMART) {
opts.casing = is_lowercase(opts.query) ? CASE_INSENSITIVE : CASE_SENSITIVE;
}
if (opts.literal) {
if (opts.casing == CASE_INSENSITIVE) {
/* Search routine needs the query to be lowercase */
char *c = opts.query;
for (; *c != '\0'; ++c) {
*c = (char)tolower(*c);
}
}
generate_alpha_skip(opts.query, opts.query_len, alpha_skip_lookup, opts.casing == CASE_SENSITIVE);
find_skip_lookup = NULL;
generate_find_skip(opts.query, opts.query_len, &find_skip_lookup, opts.casing == CASE_SENSITIVE);
if (opts.word_regexp) {
init_wordchar_table();
opts.literal_starts_wordchar = is_wordchar(opts.query[0]);
opts.literal_ends_wordchar = is_wordchar(opts.query[opts.query_len - 1]);
}
} else {
if (opts.casing == CASE_INSENSITIVE) {
pcre_opts |= PCRE_CASELESS;
}
if (opts.word_regexp) {
char *word_regexp_query;
ag_asprintf(&word_regexp_query, "\\b%s\\b", opts.query);
free(opts.query);
opts.query = word_regexp_query;
opts.query_len = strlen(opts.query);
}
compile_study(&opts.re, &opts.re_extra, opts.query, pcre_opts, study_opts);
}
if (opts.search_stream) {
// search_stream(stdin, "");
} else {
for (i = 0; i < workers_len; i++) {
workers[i].id = i;
int rv = pthread_create(&(workers[i].thread), NULL, &search_file_worker, &(workers[i].id));
if (rv != 0) {
die("error in pthread_create(): %s", strerror(rv));
}
#if defined(HAVE_PTHREAD_SETAFFINITY_NP) && defined(USE_CPU_SET)
if (opts.use_thread_affinity) {
cpu_set_t cpu_set;
CPU_ZERO(&cpu_set);
CPU_SET(i % num_cores, &cpu_set);
rv = pthread_setaffinity_np(workers[i].thread, sizeof(cpu_set), &cpu_set);
if (rv != 0) {
die("error in pthread_setaffinity_np(): %s", strerror(rv));
}
log_debug("Thread %i set to CPU %i", i, i);
} else {
log_debug("Thread affinity disabled.");
}
#else
log_debug("No CPU affinity support.");
#endif
}
for (i = 0; paths[i] != NULL; i++) {
log_debug("searching path %s for %s", paths[i], opts.query);
symhash = NULL;
ignores *ig = init_ignore(root_ignores, "", 0);
struct stat s;
s.st_dev = 0;
#ifndef _WIN32
/* The device is ignored if opts.one_dev is false, so it's fine
* to leave it at the default 0
*/
if (opts.one_dev && lstat(paths[i], &s) == -1) {
log_err("Failed to get device information for path %s. Skipping...", paths[i]);
}
#endif
search_dir(ig, base_paths[i], paths[i], 0, s.st_dev);
cleanup_ignore(ig);
}
pthread_mutex_lock(&work_queue_mtx);
done_adding_files = TRUE;
pthread_cond_broadcast(&files_ready);
pthread_mutex_unlock(&work_queue_mtx);
for (i = 0; i < workers_len; i++) {
if (pthread_join(workers[i].thread, NULL)) {
die("pthread_join failed!");
}
}
}
if (opts.stats) {
gettimeofday(&(stats.time_end), NULL);
time_diff = ((long)stats.time_end.tv_sec * 1000000 + stats.time_end.tv_usec) -
((long)stats.time_start.tv_sec * 1000000 + stats.time_start.tv_usec);
time_diff /= 1000000;
printf("%ld matches\n%ld files searched\n%ld bytes searched\n%f seconds\n", stats.total_matches, stats.total_files, stats.total_bytes, time_diff);
}
if (opts.pager) {
pclose(out_fd);
}
cleanup_options();
pthread_cond_destroy(&files_ready);
pthread_mutex_destroy(&work_queue_mtx);
pthread_mutex_destroy(&stats_mtx);
pthread_mutex_destroy(&print_mtx);
cleanup_ignore(root_ignores);
free(workers);
for (i = 0; paths[i] != NULL; i++) {
free(paths[i]);
free(base_paths[i]);
}
free(base_paths);
free(paths);
if (find_skip_lookup) {
free(find_skip_lookup);
}
return !opts.match_found;
}
void parse_options(int argc, char **argv, char **paths[]) {
int ch;
int i;
int path_len = 0;
int useless = 0;
int group = 1;
int help = 0;
int version = 0;
int opt_index = 0;
const char *home_dir = getenv("HOME");
char *ignore_file_path = NULL;
int needs_query = 1;
init_options();
struct option longopts[] = {
{ "ackmate", no_argument, &(opts.ackmate), 1 },
{ "ackmate-dir-filter", required_argument, NULL, 0 },
{ "after", required_argument, NULL, 'A' },
{ "all-text", no_argument, NULL, 't' },
{ "all-types", no_argument, NULL, 'a' },
{ "before", required_argument, NULL, 'B' },
{ "break", no_argument, &(opts.print_break), 1 },
{ "case-sensitive", no_argument, NULL, 's' },
{ "color", no_argument, &(opts.color), 1 },
{ "column", no_argument, &(opts.column), 1 },
{ "context", optional_argument, NULL, 'C' },
{ "debug", no_argument, NULL, 'D' },
{ "depth", required_argument, NULL, 0 },
{ "file-search-regex", required_argument, NULL, 'G' },
{ "files-with-matches", no_argument, NULL, 'l' },
{ "files-without-matches", no_argument, NULL, 'L' },
{ "follow", no_argument, &(opts.follow_symlinks), 1 },
{ "group", no_argument, &(group), 1 },
{ "heading", no_argument, &(opts.print_heading), 1 },
{ "help", no_argument, NULL, 'h' },
{ "hidden", no_argument, &(opts.search_hidden_files), 1 },
{ "ignore", required_argument, NULL, 0 },
{ "ignore-case", no_argument, NULL, 'i' },
{ "invert-match", no_argument, &(opts.invert_match), 1 },
{ "literal", no_argument, NULL, 'Q' },
{ "match", no_argument, &useless, 0 },
{ "max-count", required_argument, NULL, 'm' },
{ "no-recurse", no_argument, NULL, 'n' },
{ "nobreak", no_argument, &(opts.print_break), 0 },
{ "nocolor", no_argument, &(opts.color), 0 },
{ "nofollow", no_argument, &(opts.follow_symlinks), 0 },
{ "nogroup", no_argument, &(group), 0 },
{ "noheading", no_argument, &(opts.print_heading), 0 },
{ "parallel", no_argument, &(opts.parallel), 1},
{ "path-to-agignore", required_argument, NULL, 'p'},
{ "print-long-lines", no_argument, &(opts.print_long_lines), 1 },
{ "recurse", no_argument, NULL, 'r' },
{ "search-binary", no_argument, &(opts.search_binary_files), 1 },
{ "search-files", no_argument, &(opts.search_stream), 0 },
{ "skip-vcs-ignores", no_argument, NULL, 'U' },
{ "smart-case", no_argument, NULL, 'S' },
{ "stats", no_argument, &(opts.stats), 1 },
{ "unrestricted", no_argument, NULL, 'u' },
{ "version", no_argument, &version, 1 },
{ "word-regexp", no_argument, NULL, 'w' },
{ "workers", required_argument, NULL, 0 },
{ NULL, 0, NULL, 0 }
};
if (argc < 2) {
usage();
exit(1);
}
/* stdin isn't a tty. something's probably being piped to ag */
if (!isatty(fileno(stdin))) {
opts.search_stream = 1;
}
/* If we're not outputting to a terminal. change output to:
* turn off colors
* print filenames on every line
*/
if (!isatty(fileno(stdout))) {
opts.color = 0;
group = 0;
}
while ((ch = getopt_long(argc, argv, "A:aB:C:DG:g:fhiLlm:np:QRrSsvVtuUw", longopts, &opt_index)) != -1) {
switch (ch) {
case 'A':
opts.after = atoi(optarg);
break;
case 'a':
opts.search_all_files = 1;
opts.search_binary_files = 1;
break;
case 'B':
opts.before = atoi(optarg);
break;
case 'C':
if (optarg) {
opts.context = atoi(optarg);
if (opts.context == 0 && errno == EINVAL) {
/* This arg must be the search string instead of the context length */
optind--;
opts.context = DEFAULT_CONTEXT_LEN;
}
}
else {
opts.context = DEFAULT_CONTEXT_LEN;
}
break;
case 'D':
set_log_level(LOG_LEVEL_DEBUG);
break;
case 'f':
opts.follow_symlinks = 1;
break;
case 'g':
needs_query = 0;
opts.match_files = 1;
/* Fall through and build regex */
case 'G':
compile_study(&opts.file_search_regex, &opts.file_search_regex_extra, optarg, 0, 0);
break;
case 'h':
help = 1;
break;
case 'i':
opts.casing = CASE_INSENSITIVE;
break;
case 'L':
opts.invert_match = 1;
/* fall through */
case 'l':
opts.print_filename_only = 1;
break;
case 'm':
opts.max_matches_per_file = atoi(optarg);
break;
case 'n':
opts.recurse_dirs = 0;
break;
case 'p':
opts.path_to_agignore = optarg;
break;
case 'Q':
opts.literal = 1;
break;
case 'R':
case 'r':
opts.recurse_dirs = 1;
break;
case 'S':
opts.casing = CASE_SMART;
break;
case 's':
opts.casing = CASE_SENSITIVE;
break;
case 't':
opts.search_all_files = 1;
break;
case 'u':
opts.search_binary_files = 1;
opts.search_all_files = 1;
opts.search_hidden_files = 1;
break;
case 'U':
opts.skip_vcs_ignores = 1;
break;
case 'v':
opts.invert_match = 1;
break;
case 'V':
version = 1;
break;
case 'w':
opts.word_regexp = 1;
break;
case 0: /* Long option */
if (strcmp(longopts[opt_index].name, "ackmate-dir-filter") == 0) {
compile_study(&opts.ackmate_dir_filter, &opts.ackmate_dir_filter_extra, optarg, 0, 0);
break;
}
else if (strcmp(longopts[opt_index].name, "depth") == 0) {
opts.max_search_depth = atoi(optarg);
break;
}
else if (strcmp(longopts[opt_index].name, "ignore") == 0) {
add_ignore_pattern(root_ignores, optarg);
break;
}
else if (strcmp(longopts[opt_index].name, "workers") == 0) {
opts.workers = atoi(optarg);
break;
}
/* Continue to usage if we don't recognize the option */
if (longopts[opt_index].flag != 0) {
break;
}
log_err("option %s does not take a value", longopts[opt_index].name);
default:
usage();
exit(1);
}
}
argc -= optind;
argv += optind;
if (help) {
usage();
exit(0);
}
if (version) {
print_version();
exit(0);
}
if (needs_query && argc == 0) {
log_err("What do you want to search for?");
exit(1);
}
if (home_dir && !opts.search_all_files) {
log_debug("Found user's home dir: %s", home_dir);
asprintf(&ignore_file_path, "%s/%s", home_dir, ignore_pattern_files[0]);
load_ignore_patterns(root_ignores, ignore_file_path);
free(ignore_file_path);
}
if (opts.context > 0) {
opts.before = opts.context;
opts.after = opts.context;
}
if (opts.ackmate) {
opts.color = 0;
opts.print_break = 1;
group = 1;
opts.search_stream = 0;
}
if (opts.parallel) {
opts.search_stream = 0;
}
if (opts.print_heading == 0 || opts.print_break == 0) {
goto skip_group;
}
if (group) {
opts.print_heading = 1;
opts.print_break = 1;
}
else {
opts.print_heading = 0;
opts.print_break = 0;
}
skip_group:;
if (opts.search_stream) {
opts.print_break = 0;
opts.print_heading = 0;
opts.print_line_numbers = 0;
}
if (needs_query) {
opts.query = strdup(argv[0]);
argc--;
argv++;
}
else {
opts.query = strdup(".");
}
opts.query_len = strlen(opts.query);
log_debug("Query is %s", opts.query);
if (opts.query_len == 0) {
log_err("Error: No query. What do you want to search for?");
exit(1);
}
if (!is_regex(opts.query)) {
opts.literal = 1;
}
char *path = NULL;
opts.paths_len = argc;
if (argc > 0) {
*paths = calloc(sizeof(char*), argc + 1);
for (i = 0; i < argc; i++) {
path = strdup(argv[i]);
path_len = strlen(path);
/* kill trailing slash */
if (path_len > 1 && path[path_len - 1] == '/') {
path[path_len - 1] = '\0';
}
(*paths)[i] = path;
}
(*paths)[i] = NULL;
/* Make sure we search these paths instead of stdin. */
opts.search_stream = 0;
}
else {
path = strdup(".");
*paths = malloc(sizeof(char*) * 2);
(*paths)[0] = path;
(*paths)[1] = NULL;
}
}
