int main(int argc, char **argv) { char **paths = NULL; int i; int pcre_opts = PCRE_MULTILINE; int study_opts = 0; double time_diff; pthread_t *workers = NULL; int workers_len; set_log_level(LOG_LEVEL_WARN); work_queue = NULL; work_queue_tail = NULL; memset(&stats, 0, sizeof(stats)); root_ignores = init_ignore(NULL); out_fd = stdout; #ifdef USE_PCRE_JIT int has_jit = 0; pcre_config(PCRE_CONFIG_JIT, &has_jit); if (has_jit) { study_opts |= PCRE_STUDY_JIT_COMPILE; } #endif gettimeofday(&(stats.time_start), NULL); parse_options(argc, argv, &paths); log_debug("PCRE Version: %s", pcre_version()); if (opts.workers) { workers_len = opts.workers; } else { /* Experiments show that two worker threads appear to be optimal, both * on dual-core and quad-core systems. See * http://geoff.greer.fm/2012/09/07/the-silver-searcher-adding-pthreads/. * On single-core CPUs, more than one worker thread makes no sense. */ int ncpus = (int)sysconf(_SC_NPROCESSORS_ONLN); workers_len = (ncpus >= 2) ? 2 : 1; } log_debug("Using %i workers", workers_len); done_adding_files = FALSE; workers = ag_calloc(workers_len, sizeof(pthread_t)); if (pthread_cond_init(&files_ready, NULL)) { log_err("pthread_cond_init failed!"); exit(2); } if (pthread_mutex_init(&print_mtx, NULL)) { log_err("pthread_mutex_init failed!"); exit(2); } if (pthread_mutex_init(&stats_mtx, NULL)) { log_err("pthread_mutex_init failed!"); exit(2); } if (pthread_mutex_init(&work_queue_mtx, NULL)) { log_err("pthread_mutex_init failed!"); exit(2); } if (opts.casing == CASE_SMART) { opts.casing = contains_uppercase(opts.query) ? CASE_SENSITIVE : CASE_INSENSITIVE; } if (opts.literal) { if (opts.casing == CASE_INSENSITIVE) { /* Search routine needs the query to be lowercase */ char *c = opts.query; for (; *c != '\0'; ++c) { *c = (char) tolower(*c); } } generate_skip_lookup(opts.query, opts.query_len, skip_lookup, opts.casing == CASE_SENSITIVE); if (opts.word_regexp) { init_wordchar_table(); opts.literal_starts_wordchar = is_wordchar(opts.query[0]); opts.literal_ends_wordchar = is_wordchar(opts.query[opts.query_len - 1]); } } else { if (opts.casing == CASE_INSENSITIVE) { pcre_opts = pcre_opts | PCRE_CASELESS; } if (opts.word_regexp) { char *word_regexp_query; asprintf(&word_regexp_query, "\\b%s\\b", opts.query); free(opts.query); opts.query = word_regexp_query; opts.query_len = strlen(opts.query); } compile_study(&opts.re, &opts.re_extra, opts.query, pcre_opts, study_opts); } if (opts.search_stream) { search_stream(stdin, ""); } else { for (i = 0; i < workers_len; i++) { int ptc_rc = pthread_create(&(workers[i]), NULL, &search_file_worker, NULL); check_err(ptc_rc, "create worker thread"); } for (i = 0; paths[i] != NULL; i++) { log_debug("searching path %s for %s", paths[i], opts.query); search_dir(root_ignores, paths[i], 0); } done_adding_files = TRUE; pthread_cond_broadcast(&files_ready); for (i = 0; i < workers_len; i++) { if (pthread_join(workers[i], NULL)) { log_err("pthread_join failed!"); exit(2); } } } if (opts.stats) { gettimeofday(&(stats.time_end), NULL); time_diff = ((long)stats.time_end.tv_sec * 1000000 + stats.time_end.tv_usec) - ((long)stats.time_start.tv_sec * 1000000 + stats.time_start.tv_usec); time_diff /= 1000000; printf("%ld matches\n%ld files searched\n%ld bytes searched\n%f seconds\n", stats.total_matches, stats.total_files, stats.total_bytes, time_diff); } if (opts.pager) { pclose(out_fd); } pthread_cond_destroy(&files_ready); pthread_mutex_destroy(&work_queue_mtx); pthread_mutex_destroy(&stats_mtx); pthread_mutex_destroy(&print_mtx); cleanup_ignore(root_ignores); free(workers); free(paths); return 0; }
int main(int argc, char **argv) { char **base_paths = NULL; char **paths = NULL; int i; int pcre_opts = PCRE_MULTILINE; int study_opts = 0; double time_diff; pthread_t *workers = NULL; int workers_len; set_log_level(LOG_LEVEL_WARN); work_queue = NULL; work_queue_tail = NULL; memset(&stats, 0, sizeof(stats)); root_ignores = init_ignore(NULL, "", 0); out_fd = stdout; #ifdef USE_PCRE_JIT int has_jit = 0; pcre_config(PCRE_CONFIG_JIT, &has_jit); if (has_jit) { study_opts |= PCRE_STUDY_JIT_COMPILE; } #endif gettimeofday(&(stats.time_start), NULL); parse_options(argc, argv, &base_paths, &paths); log_debug("PCRE Version: %s", pcre_version()); #ifdef _WIN32 { SYSTEM_INFO si; GetSystemInfo(&si); workers_len = si.dwNumberOfProcessors; } #else workers_len = (int)sysconf(_SC_NPROCESSORS_ONLN); #endif if (opts.literal) { workers_len--; } if (opts.workers) { workers_len = opts.workers; } if (workers_len < 1) { workers_len = 1; } log_debug("Using %i workers", workers_len); done_adding_files = FALSE; workers = ag_calloc(workers_len, sizeof(pthread_t)); if (pthread_cond_init(&files_ready, NULL)) { die("pthread_cond_init failed!"); } if (pthread_mutex_init(&print_mtx, NULL)) { die("pthread_mutex_init failed!"); } if (pthread_mutex_init(&stats_mtx, NULL)) { die("pthread_mutex_init failed!"); } if (pthread_mutex_init(&work_queue_mtx, NULL)) { die("pthread_mutex_init failed!"); } if (opts.casing == CASE_SMART) { opts.casing = is_lowercase(opts.query) ? CASE_INSENSITIVE : CASE_SENSITIVE; } if (opts.literal) { if (opts.casing == CASE_INSENSITIVE) { /* Search routine needs the query to be lowercase */ char *c = opts.query; for (; *c != '\0'; ++c) { *c = (char)tolower(*c); } } generate_alpha_skip(opts.query, opts.query_len, alpha_skip_lookup, opts.casing == CASE_SENSITIVE); find_skip_lookup = NULL; generate_find_skip(opts.query, opts.query_len, &find_skip_lookup, opts.casing == CASE_SENSITIVE); if (opts.word_regexp) { init_wordchar_table(); opts.literal_starts_wordchar = is_wordchar(opts.query[0]); opts.literal_ends_wordchar = is_wordchar(opts.query[opts.query_len - 1]); } } else { if (opts.casing == CASE_INSENSITIVE) { pcre_opts |= PCRE_CASELESS; } if (opts.word_regexp) { char *word_regexp_query; ag_asprintf(&word_regexp_query, "\\b%s\\b", opts.query); free(opts.query); opts.query = word_regexp_query; opts.query_len = strlen(opts.query); } compile_study(&opts.re, &opts.re_extra, opts.query, pcre_opts, study_opts); } if (opts.search_stream) { search_stream(stdin, ""); } else { for (i = 0; i < workers_len; i++) { int rv = pthread_create(&(workers[i]), NULL, &search_file_worker, &i); if (rv != 0) { die("error in pthread_create(): %s", strerror(rv)); } } for (i = 0; paths[i] != NULL; i++) { log_debug("searching path %s for %s", paths[i], opts.query); symhash = NULL; ignores *ig = init_ignore(root_ignores, "", 0); search_dir(ig, base_paths[i], paths[i], 0); cleanup_ignore(ig); } pthread_mutex_lock(&work_queue_mtx); done_adding_files = TRUE; pthread_cond_broadcast(&files_ready); pthread_mutex_unlock(&work_queue_mtx); for (i = 0; i < workers_len; i++) { if (pthread_join(workers[i], NULL)) { die("pthread_join failed!"); } } } if (opts.stats) { gettimeofday(&(stats.time_end), NULL); time_diff = ((long)stats.time_end.tv_sec * 1000000 + stats.time_end.tv_usec) - ((long)stats.time_start.tv_sec * 1000000 + stats.time_start.tv_usec); time_diff /= 1000000; printf("%ld matches\n%ld files searched\n%ld bytes searched\n%f seconds\n", stats.total_matches, stats.total_files, stats.total_bytes, time_diff); } if (opts.pager) { pclose(out_fd); } cleanup_options(); pthread_cond_destroy(&files_ready); pthread_mutex_destroy(&work_queue_mtx); pthread_mutex_destroy(&stats_mtx); pthread_mutex_destroy(&print_mtx); cleanup_ignore(root_ignores); free(workers); for (i = 0; paths[i] != NULL; i++) { free(paths[i]); free(base_paths[i]); } free(base_paths); free(paths); if (find_skip_lookup) { free(find_skip_lookup); } return !opts.match_found; }
int main(int argc, char **argv) { char **base_paths = NULL; char **paths = NULL; int i; int pcre_opts = PCRE_MULTILINE; int study_opts = 0; double time_diff; worker_t *workers = NULL; int workers_len; int num_cores; #ifdef KJK_BUILD extern void setup_crash_handler(); /* in kjk_crash_handler.cpp */ setup_crash_handler(); #endif set_log_level(LOG_LEVEL_WARN); work_queue = NULL; work_queue_tail = NULL; memset(&stats, 0, sizeof(stats)); root_ignores = init_ignore(NULL, "", 0); out_fd = stdout; #ifdef USE_PCRE_JIT int has_jit = 0; pcre_config(PCRE_CONFIG_JIT, &has_jit); if (has_jit) { study_opts |= PCRE_STUDY_JIT_COMPILE; } #endif gettimeofday(&(stats.time_start), NULL); parse_options(argc, argv, &base_paths, &paths); log_debug("PCRE Version: %s", pcre_version()); #ifdef _WIN32 { SYSTEM_INFO si; GetSystemInfo(&si); num_cores = si.dwNumberOfProcessors; } #else num_cores = (int)sysconf(_SC_NPROCESSORS_ONLN); #endif workers_len = num_cores; if (opts.literal) { workers_len--; } if (opts.workers) { workers_len = opts.workers; } if (workers_len < 1) { workers_len = 1; } log_debug("Using %i workers", workers_len); done_adding_files = FALSE; workers = (worker_t *) ag_calloc(workers_len, sizeof(worker_t)); if (pthread_cond_init(&files_ready, NULL)) { die("pthread_cond_init failed!"); } if (pthread_mutex_init(&print_mtx, NULL)) { die("pthread_mutex_init failed!"); } if (pthread_mutex_init(&stats_mtx, NULL)) { die("pthread_mutex_init failed!"); } if (pthread_mutex_init(&work_queue_mtx, NULL)) { die("pthread_mutex_init failed!"); } if (opts.casing == CASE_SMART) { opts.casing = is_lowercase(opts.query) ? CASE_INSENSITIVE : CASE_SENSITIVE; } if (opts.literal) { if (opts.casing == CASE_INSENSITIVE) { /* Search routine needs the query to be lowercase */ char *c = opts.query; for (; *c != '\0'; ++c) { *c = (char)tolower(*c); } } generate_alpha_skip(opts.query, opts.query_len, alpha_skip_lookup, opts.casing == CASE_SENSITIVE); find_skip_lookup = NULL; generate_find_skip(opts.query, opts.query_len, &find_skip_lookup, opts.casing == CASE_SENSITIVE); if (opts.word_regexp) { init_wordchar_table(); opts.literal_starts_wordchar = is_wordchar(opts.query[0]); opts.literal_ends_wordchar = is_wordchar(opts.query[opts.query_len - 1]); } } else { if (opts.casing == CASE_INSENSITIVE) { pcre_opts |= PCRE_CASELESS; } if (opts.word_regexp) { char *word_regexp_query; ag_asprintf(&word_regexp_query, "\\b%s\\b", opts.query); free(opts.query); opts.query = word_regexp_query; opts.query_len = strlen(opts.query); } compile_study(&opts.re, &opts.re_extra, opts.query, pcre_opts, study_opts); } if (opts.search_stream) { // search_stream(stdin, ""); } else { for (i = 0; i < workers_len; i++) { workers[i].id = i; int rv = pthread_create(&(workers[i].thread), NULL, &search_file_worker, &(workers[i].id)); if (rv != 0) { die("error in pthread_create(): %s", strerror(rv)); } #if defined(HAVE_PTHREAD_SETAFFINITY_NP) && defined(USE_CPU_SET) if (opts.use_thread_affinity) { cpu_set_t cpu_set; CPU_ZERO(&cpu_set); CPU_SET(i % num_cores, &cpu_set); rv = pthread_setaffinity_np(workers[i].thread, sizeof(cpu_set), &cpu_set); if (rv != 0) { die("error in pthread_setaffinity_np(): %s", strerror(rv)); } log_debug("Thread %i set to CPU %i", i, i); } else { log_debug("Thread affinity disabled."); } #else log_debug("No CPU affinity support."); #endif } for (i = 0; paths[i] != NULL; i++) { log_debug("searching path %s for %s", paths[i], opts.query); symhash = NULL; ignores *ig = init_ignore(root_ignores, "", 0); struct stat s; s.st_dev = 0; #ifndef _WIN32 /* The device is ignored if opts.one_dev is false, so it's fine * to leave it at the default 0 */ if (opts.one_dev && lstat(paths[i], &s) == -1) { log_err("Failed to get device information for path %s. Skipping...", paths[i]); } #endif search_dir(ig, base_paths[i], paths[i], 0, s.st_dev); cleanup_ignore(ig); } pthread_mutex_lock(&work_queue_mtx); done_adding_files = TRUE; pthread_cond_broadcast(&files_ready); pthread_mutex_unlock(&work_queue_mtx); for (i = 0; i < workers_len; i++) { if (pthread_join(workers[i].thread, NULL)) { die("pthread_join failed!"); } } } if (opts.stats) { gettimeofday(&(stats.time_end), NULL); time_diff = ((long)stats.time_end.tv_sec * 1000000 + stats.time_end.tv_usec) - ((long)stats.time_start.tv_sec * 1000000 + stats.time_start.tv_usec); time_diff /= 1000000; printf("%ld matches\n%ld files searched\n%ld bytes searched\n%f seconds\n", stats.total_matches, stats.total_files, stats.total_bytes, time_diff); } if (opts.pager) { pclose(out_fd); } cleanup_options(); pthread_cond_destroy(&files_ready); pthread_mutex_destroy(&work_queue_mtx); pthread_mutex_destroy(&stats_mtx); pthread_mutex_destroy(&print_mtx); cleanup_ignore(root_ignores); free(workers); for (i = 0; paths[i] != NULL; i++) { free(paths[i]); free(base_paths[i]); } free(base_paths); free(paths); if (find_skip_lookup) { free(find_skip_lookup); } return !opts.match_found; }
void parse_options(int argc, char **argv, char **paths[]) { int ch; int i; int path_len = 0; int useless = 0; int group = 1; int help = 0; int version = 0; int opt_index = 0; const char *home_dir = getenv("HOME"); char *ignore_file_path = NULL; int needs_query = 1; init_options(); struct option longopts[] = { { "ackmate", no_argument, &(opts.ackmate), 1 }, { "ackmate-dir-filter", required_argument, NULL, 0 }, { "after", required_argument, NULL, 'A' }, { "all-text", no_argument, NULL, 't' }, { "all-types", no_argument, NULL, 'a' }, { "before", required_argument, NULL, 'B' }, { "break", no_argument, &(opts.print_break), 1 }, { "case-sensitive", no_argument, NULL, 's' }, { "color", no_argument, &(opts.color), 1 }, { "column", no_argument, &(opts.column), 1 }, { "context", optional_argument, NULL, 'C' }, { "debug", no_argument, NULL, 'D' }, { "depth", required_argument, NULL, 0 }, { "file-search-regex", required_argument, NULL, 'G' }, { "files-with-matches", no_argument, NULL, 'l' }, { "files-without-matches", no_argument, NULL, 'L' }, { "follow", no_argument, &(opts.follow_symlinks), 1 }, { "group", no_argument, &(group), 1 }, { "heading", no_argument, &(opts.print_heading), 1 }, { "help", no_argument, NULL, 'h' }, { "hidden", no_argument, &(opts.search_hidden_files), 1 }, { "ignore", required_argument, NULL, 0 }, { "ignore-case", no_argument, NULL, 'i' }, { "invert-match", no_argument, &(opts.invert_match), 1 }, { "literal", no_argument, NULL, 'Q' }, { "match", no_argument, &useless, 0 }, { "max-count", required_argument, NULL, 'm' }, { "no-recurse", no_argument, NULL, 'n' }, { "nobreak", no_argument, &(opts.print_break), 0 }, { "nocolor", no_argument, &(opts.color), 0 }, { "nofollow", no_argument, &(opts.follow_symlinks), 0 }, { "nogroup", no_argument, &(group), 0 }, { "noheading", no_argument, &(opts.print_heading), 0 }, { "parallel", no_argument, &(opts.parallel), 1}, { "path-to-agignore", required_argument, NULL, 'p'}, { "print-long-lines", no_argument, &(opts.print_long_lines), 1 }, { "recurse", no_argument, NULL, 'r' }, { "search-binary", no_argument, &(opts.search_binary_files), 1 }, { "search-files", no_argument, &(opts.search_stream), 0 }, { "skip-vcs-ignores", no_argument, NULL, 'U' }, { "smart-case", no_argument, NULL, 'S' }, { "stats", no_argument, &(opts.stats), 1 }, { "unrestricted", no_argument, NULL, 'u' }, { "version", no_argument, &version, 1 }, { "word-regexp", no_argument, NULL, 'w' }, { "workers", required_argument, NULL, 0 }, { NULL, 0, NULL, 0 } }; if (argc < 2) { usage(); exit(1); } /* stdin isn't a tty. something's probably being piped to ag */ if (!isatty(fileno(stdin))) { opts.search_stream = 1; } /* If we're not outputting to a terminal. change output to: * turn off colors * print filenames on every line */ if (!isatty(fileno(stdout))) { opts.color = 0; group = 0; } while ((ch = getopt_long(argc, argv, "A:aB:C:DG:g:fhiLlm:np:QRrSsvVtuUw", longopts, &opt_index)) != -1) { switch (ch) { case 'A': opts.after = atoi(optarg); break; case 'a': opts.search_all_files = 1; opts.search_binary_files = 1; break; case 'B': opts.before = atoi(optarg); break; case 'C': if (optarg) { opts.context = atoi(optarg); if (opts.context == 0 && errno == EINVAL) { /* This arg must be the search string instead of the context length */ optind--; opts.context = DEFAULT_CONTEXT_LEN; } } else { opts.context = DEFAULT_CONTEXT_LEN; } break; case 'D': set_log_level(LOG_LEVEL_DEBUG); break; case 'f': opts.follow_symlinks = 1; break; case 'g': needs_query = 0; opts.match_files = 1; /* Fall through and build regex */ case 'G': compile_study(&opts.file_search_regex, &opts.file_search_regex_extra, optarg, 0, 0); break; case 'h': help = 1; break; case 'i': opts.casing = CASE_INSENSITIVE; break; case 'L': opts.invert_match = 1; /* fall through */ case 'l': opts.print_filename_only = 1; break; case 'm': opts.max_matches_per_file = atoi(optarg); break; case 'n': opts.recurse_dirs = 0; break; case 'p': opts.path_to_agignore = optarg; break; case 'Q': opts.literal = 1; break; case 'R': case 'r': opts.recurse_dirs = 1; break; case 'S': opts.casing = CASE_SMART; break; case 's': opts.casing = CASE_SENSITIVE; break; case 't': opts.search_all_files = 1; break; case 'u': opts.search_binary_files = 1; opts.search_all_files = 1; opts.search_hidden_files = 1; break; case 'U': opts.skip_vcs_ignores = 1; break; case 'v': opts.invert_match = 1; break; case 'V': version = 1; break; case 'w': opts.word_regexp = 1; break; case 0: /* Long option */ if (strcmp(longopts[opt_index].name, "ackmate-dir-filter") == 0) { compile_study(&opts.ackmate_dir_filter, &opts.ackmate_dir_filter_extra, optarg, 0, 0); break; } else if (strcmp(longopts[opt_index].name, "depth") == 0) { opts.max_search_depth = atoi(optarg); break; } else if (strcmp(longopts[opt_index].name, "ignore") == 0) { add_ignore_pattern(root_ignores, optarg); break; } else if (strcmp(longopts[opt_index].name, "workers") == 0) { opts.workers = atoi(optarg); break; } /* Continue to usage if we don't recognize the option */ if (longopts[opt_index].flag != 0) { break; } log_err("option %s does not take a value", longopts[opt_index].name); default: usage(); exit(1); } } argc -= optind; argv += optind; if (help) { usage(); exit(0); } if (version) { print_version(); exit(0); } if (needs_query && argc == 0) { log_err("What do you want to search for?"); exit(1); } if (home_dir && !opts.search_all_files) { log_debug("Found user's home dir: %s", home_dir); asprintf(&ignore_file_path, "%s/%s", home_dir, ignore_pattern_files[0]); load_ignore_patterns(root_ignores, ignore_file_path); free(ignore_file_path); } if (opts.context > 0) { opts.before = opts.context; opts.after = opts.context; } if (opts.ackmate) { opts.color = 0; opts.print_break = 1; group = 1; opts.search_stream = 0; } if (opts.parallel) { opts.search_stream = 0; } if (opts.print_heading == 0 || opts.print_break == 0) { goto skip_group; } if (group) { opts.print_heading = 1; opts.print_break = 1; } else { opts.print_heading = 0; opts.print_break = 0; } skip_group:; if (opts.search_stream) { opts.print_break = 0; opts.print_heading = 0; opts.print_line_numbers = 0; } if (needs_query) { opts.query = strdup(argv[0]); argc--; argv++; } else { opts.query = strdup("."); } opts.query_len = strlen(opts.query); log_debug("Query is %s", opts.query); if (opts.query_len == 0) { log_err("Error: No query. What do you want to search for?"); exit(1); } if (!is_regex(opts.query)) { opts.literal = 1; } char *path = NULL; opts.paths_len = argc; if (argc > 0) { *paths = calloc(sizeof(char*), argc + 1); for (i = 0; i < argc; i++) { path = strdup(argv[i]); path_len = strlen(path); /* kill trailing slash */ if (path_len > 1 && path[path_len - 1] == '/') { path[path_len - 1] = '\0'; } (*paths)[i] = path; } (*paths)[i] = NULL; /* Make sure we search these paths instead of stdin. */ opts.search_stream = 0; } else { path = strdup("."); *paths = malloc(sizeof(char*) * 2); (*paths)[0] = path; (*paths)[1] = NULL; } }