static void update_database (void)
{
int list = get_playlist (TRUE, TRUE);
if (list >= 0)
{
create_database (list);
schedule_search ();
}
else
destroy_database ();
if (results_list)
audgui_list_delete_rows (results_list, 0, audgui_list_row_count (results_list));
show_hide_widgets ();
}
static void search_cleanup (void)
{
hook_dissociate ("playlist add complete", add_complete_cb);
hook_dissociate ("playlist scan complete", scan_complete_cb);
hook_dissociate ("playlist update", playlist_update_cb);
if (search_source)
{
g_source_remove (search_source);
search_source = 0;
}
index_free_full (search_terms, (IndexFreeFunc) str_unref);
search_terms = NULL;
index_free (items);
items = NULL;
g_array_free (selection, TRUE);
selection = NULL;
destroy_added_table ();
destroy_database ();
}
int main(int argc, char** argv) {
s32 opt;
u32 loop_cnt = 0, purge_age = 0, seed;
u8 sig_loaded = 0, show_once = 0, no_statistics = 0,
display_mode = 0, has_fake = 0;
s32 oindex = 0;
u8 *wordlist = NULL, *output_dir = NULL;
u8 *sig_list_strg = NULL;
u8 *gtimeout_str = NULL;
u32 gtimeout = 0;
struct termios term;
struct timeval tv;
u64 st_time, en_time;
signal(SIGINT, ctrlc_handler);
signal(SIGWINCH, resize_handler);
signal(SIGPIPE, SIG_IGN);
SSL_library_init();
/* Options, options, and options */
static struct option long_options[] = {
{"auth", required_argument, 0, 'A' },
{"host", required_argument, 0, 'F' },
{"cookie", required_argument, 0, 'C' },
{"reject-cookies", required_argument, 0, 'N' },
{"header", required_argument, 0, 'H' },
{"user-agent", required_argument, 0, 'b' },
#ifdef PROXY_SUPPORT
{"proxy", required_argument, 0, 'J' },
#endif /* PROXY_SUPPORT */
{"max-depth", required_argument, 0, 'd' },
{"max-child", required_argument, 0, 'c' },
{"max-descendants", required_argument, 0, 'x' },
{"max-requests", required_argument, 0, 'r' },
{"max-rate", required_argument, 0, 'l'},
{"probability", required_argument, 0, 'p' },
{"seed", required_argument, 0, 'q' },
{"include", required_argument, 0, 'I' },
{"exclude", required_argument, 0, 'X' },
{"skip-param", required_argument, 0, 'K' },
{"skip-forms", no_argument, 0, 'O' },
{"include-domain", required_argument, 0, 'D' },
{"ignore-links", no_argument, 0, 'P' },
{"no-ext-fuzzing", no_argument, 0, 'Y' },
{"log-mixed-content", no_argument, 0, 'M' },
{"skip-error-pages", no_argument, 0, 'Z' },
{"log-external-urls", no_argument, 0, 'U' },
{"log-cache-mismatches", no_argument, 0, 'E' },
{"form-value", no_argument, 0, 'T' },
{"rw-wordlist", required_argument, 0, 'W' },
{"no-keyword-learning", no_argument, 0, 'L' },
{"mode", required_argument, 0, 'J' },
{"wordlist", required_argument, 0, 'S'},
{"trust-domain", required_argument, 0, 'B' },
{"max-connections", required_argument, 0, 'g' },
{"max-host-connections", required_argument, 0, 'm' },
{"max-fail", required_argument, 0, 'f' },
{"request-timeout", required_argument, 0, 't' },
{"network-timeout", required_argument, 0, 'w' },
{"idle-timeout", required_argument, 0, 'i' },
{"response-size", required_argument, 0, 's' },
{"discard-binary", required_argument, 0, 'e' },
{"output", required_argument, 0, 'o' },
{"help", no_argument, 0, 'h' },
{"quiet", no_argument, 0, 'u' },
{"verbose", no_argument, 0, 'v' },
{"scan-timeout", required_argument, 0, 'k'},
{"signatures", required_argument, 0, 'z'},
{"checks", no_argument, 0, 0},
{"checks-toggle", required_argument, 0, 0},
{"no-checks", no_argument, 0, 0},
{"fast", no_argument, 0, 0},
{"auth-form", required_argument, 0, 0},
{"auth-form-target", required_argument, 0, 0},
{"auth-user", required_argument, 0, 0},
{"auth-user-field", required_argument, 0, 0},
{"auth-pass", required_argument, 0, 0},
{"auth-pass-field", required_argument, 0, 0},
{"auth-verify-url", required_argument, 0, 0},
{0, 0, 0, 0 }
};
/* Come up with a quasi-decent random seed. */
gettimeofday(&tv, NULL);
seed = tv.tv_usec ^ (tv.tv_sec << 16) ^ getpid();
SAY("skipfish version " VERSION " by <*****@*****.**>\n");
while ((opt = getopt_long(argc, argv,
"+A:B:C:D:EF:G:H:I:J:K:LMNOPQR:S:T:UW:X:YZ"
"b:c:d:ef:g:hi:k:l:m:o:p:q:r:s:t:uvw:x:z:",
long_options, &oindex)) >= 0)
switch (opt) {
case 'A': {
u8* x = (u8*)strchr(optarg, ':');
if (!x) FATAL("Credentials must be in 'user:pass' form.");
*(x++) = 0;
auth_user = (u8*)optarg;
auth_pass = x;
auth_type = AUTH_BASIC;
break;
}
#ifdef PROXY_SUPPORT
case 'J': {
u8* x = (u8*)strchr(optarg, ':');
if (!x) FATAL("Proxy data must be in 'host:port' form.");
*(x++) = 0;
use_proxy = (u8*)optarg;
use_proxy_port = atoi((char*)x);
if (!use_proxy_port) FATAL("Incorrect proxy port number.");
break;
}
#endif /* PROXY_SUPPORT */
case 'F': {
u8* x = (u8*)strchr(optarg, '=');
u32 fake_addr;
if (!x) FATAL("Fake mappings must be in 'host=IP' form.");
*x = 0;
fake_addr = inet_addr((char*)x + 1);
if (fake_addr == (u32)-1)
FATAL("Could not parse IP address '%s'.", x + 1);
fake_host((u8*)optarg, fake_addr);
has_fake = 1;
break;
}
case 'H': {
u8* x = (u8*)strchr(optarg, '=');
if (!x) FATAL("Extra headers must be in 'name=value' form.");
*x = 0;
if (!strcasecmp(optarg, "Cookie"))
FATAL("Do not use -H to set cookies (try -C instead).");
SET_HDR((u8*)optarg, x + 1, &global_http_par);
break;
}
case 'C': {
u8* x = (u8*)strchr(optarg, '=');
if (!x) FATAL("Cookies must be in 'name=value' form.");
if (strchr(optarg, ';'))
FATAL("Split multiple cookies into separate -C options.");
*x = 0;
SET_CK((u8*)optarg, x + 1, &global_http_par);
break;
}
case 'D':
if (*optarg == '*') optarg++;
APPEND_FILTER(allow_domains, num_allow_domains, optarg);
break;
case 'K':
APPEND_FILTER(skip_params, num_skip_params, optarg);
break;
case 'B':
if (*optarg == '*') optarg++;
APPEND_FILTER(trust_domains, num_trust_domains, optarg);
break;
case 'I':
if (*optarg == '*') optarg++;
APPEND_FILTER(allow_urls, num_allow_urls, optarg);
break;
case 'X':
if (*optarg == '*') optarg++;
APPEND_FILTER(deny_urls, num_deny_urls, optarg);
break;
case 'T': {
u8* x = (u8*)strchr(optarg, '=');
if (!x) FATAL("Rules must be in 'name=value' form.");
*x = 0;
add_form_hint((u8*)optarg, x + 1);
break;
}
case 'N':
ignore_cookies = 1;
break;
case 'Y':
no_fuzz_ext = 1;
break;
case 'q':
if (sscanf(optarg, "0x%08x", &seed) != 1)
FATAL("Invalid seed format.");
srandom(seed);
break;
case 'Q':
suppress_dupes = 1;
break;
case 'P':
no_parse = 1;
break;
case 'M':
warn_mixed = 1;
break;
case 'U':
log_ext_urls = 1;
break;
case 'L':
dont_add_words = 1;
break;
case 'E':
pedantic_cache = 1;
break;
case 'O':
no_forms = 1;
break;
case 'R':
purge_age = atoi(optarg);
if (purge_age < 3) FATAL("Purge age invalid or too low (min 3).");
break;
case 'd':
max_depth = atoi(optarg);
if (max_depth < 2) FATAL("Invalid value '%s'.", optarg);
break;
case 'c':
max_children = atoi(optarg);
if (!max_children) FATAL("Invalid value '%s'.", optarg);
break;
case 'x':
max_descendants = atoi(optarg);
if (!max_descendants) FATAL("Invalid value '%s'.", optarg);
break;
case 'p':
crawl_prob = atoi(optarg);
if (!crawl_prob) FATAL("Invalid value '%s'.", optarg);
break;
case 'W':
if (wordlist)
FATAL("Only one -W parameter permitted (use -S to load supplemental dictionaries).");
if (!strcmp(optarg, "-")) wordlist = (u8*)"/dev/null";
else wordlist = (u8*)optarg;
break;
case 'S':
load_keywords((u8*)optarg, 1, 0);
break;
case 'z':
load_signatures((u8*)optarg);
sig_loaded = 1;
break;
case 'b':
if (optarg[0] == 'i') browser_type = BROWSER_MSIE; else
if (optarg[0] == 'f') browser_type = BROWSER_FFOX; else
if (optarg[0] == 'p') browser_type = BROWSER_PHONE; else
usage(argv[0]);
break;
case 'g':
max_connections = atoi(optarg);
if (!max_connections) FATAL("Invalid value '%s'.", optarg);
break;
case 'm':
max_conn_host = atoi(optarg);
if (!max_conn_host) FATAL("Invalid value '%s'.", optarg);
break;
case 'G':
max_guesses = atoi(optarg);
if (!max_guesses) FATAL("Invalid value '%s'.", optarg);
break;
case 'r':
max_requests = atoi(optarg);
if (!max_requests) FATAL("Invalid value '%s'.", optarg);
break;
case 'l':
max_requests_sec = atof(optarg);
if (!max_requests_sec) FATAL("Invalid value '%s'.", optarg);
break;
case 'f':
max_fail = atoi(optarg);
if (!max_fail) FATAL("Invalid value '%s'.", optarg);
break;
case 't':
resp_tmout = atoi(optarg);
if (!resp_tmout) FATAL("Invalid value '%s'.", optarg);
break;
case 'w':
rw_tmout = atoi(optarg);
if (!rw_tmout) FATAL("Invalid value '%s'.", optarg);
break;
case 'i':
idle_tmout = atoi(optarg);
if (!idle_tmout) FATAL("Invalid value '%s'.", optarg);
break;
case 's':
size_limit = atoi(optarg);
if (!size_limit) FATAL("Invalid value '%s'.", optarg);
break;
case 'o':
if (output_dir) FATAL("Multiple -o options not allowed.");
output_dir = (u8*)optarg;
rmdir(optarg);
if (mkdir(optarg, 0755))
PFATAL("Unable to create '%s'.", output_dir);
break;
case 'u':
no_statistics = 1;
break;
case 'v':
verbosity++;
break;
case 'e':
delete_bin = 1;
break;
case 'k':
if (gtimeout_str) FATAL("Multiple -k options not allowed.");
gtimeout_str = (u8*)optarg;
break;
case 'Z':
no_500_dir = 1;
break;
case 0:
if (!strcmp("checks", long_options[oindex].name ))
display_injection_checks();
if (!strcmp("checks-toggle", long_options[oindex].name ))
toggle_injection_checks((u8*)optarg, 1, 1);
if (!strcmp("no-checks", long_options[oindex].name ))
no_checks = 1;
if (!strcmp("signatures", long_options[oindex].name ))
load_signatures((u8*)optarg);
if(!strcmp("fast", long_options[oindex].name ))
toggle_injection_checks((u8*)"2,4,5,13,14,15,16", 0, 0);
if (!strcmp("auth-form", long_options[oindex].name ))
auth_form = (u8*)optarg;
if (!strcmp("auth-user", long_options[oindex].name ))
auth_user = (u8*)optarg;
if (!strcmp("auth-pass", long_options[oindex].name ))
auth_pass = (u8*)optarg;
if (!strcmp("auth-pass-field", long_options[oindex].name ))
auth_pass_field = (u8*)optarg;
if (!strcmp("auth-user-field", long_options[oindex].name ))
auth_user_field = (u8*)optarg;
if (!strcmp("auth-form-target", long_options[oindex].name ))
auth_form_target = (u8*)optarg;
if (!strcmp("auth-verify-url", long_options[oindex].name ))
auth_verify_url = (u8*)optarg;
break;
default:
usage(argv[0]);
}
#ifdef PROXY_SUPPORT
if (has_fake && use_proxy)
FATAL("-F and -J should not be used together.");
#endif /* PROXY_SUPPORT */
if (access(ASSETS_DIR "/index.html", R_OK))
PFATAL("Unable to access '%s/index.html' - wrong directory?", ASSETS_DIR);
srandom(seed);
if (optind == argc)
FATAL("Scan target not specified (try -h for help).");
if (!output_dir)
FATAL("Output directory not specified (try -h for help).");
if(verbosity && !no_statistics && isatty(2))
FATAL("Please use -v in combination with the -u flag or, "
"run skipfish while redirecting stderr to a file. ");
if (resp_tmout < rw_tmout)
resp_tmout = rw_tmout;
if (max_connections < max_conn_host)
max_connections = max_conn_host;
/* Parse the timeout string - format h:m:s */
if (gtimeout_str) {
int i = 0;
int m[3] = { 1, 60, 3600 };
u8* tok = (u8*)strtok((char*)gtimeout_str, ":");
while(tok && i <= 2) {
gtimeout += atoi((char*)tok) * m[i];
tok = (u8*)strtok(NULL, ":");
i++;
}
if(!gtimeout)
FATAL("Wrong timeout format, please use h:m:s (hours, minutes, seconds)");
DEBUG("* Scan timeout is set to %d seconds\n", gtimeout);
}
if (!wordlist) {
wordlist = (u8*)"/dev/null";
DEBUG("* No wordlist specified with -W: defaulting to /dev/null\n");
}
/* If no signature files have been specified via command-line: load
the default file */
if (!sig_loaded)
load_signatures((u8*)SIG_FILE);
load_keywords(wordlist, 0, purge_age);
/* Load the signatures list for the matching */
if (sig_list_strg) load_signatures(sig_list_strg);
/* Try to authenticate when the auth_user and auth_pass fields are set. */
if (auth_user && auth_pass) {
authenticate();
while (next_from_queue()) {
usleep(1000);
}
switch (auth_state) {
case ASTATE_DONE:
DEBUGC(L1, "*- Authentication succeeded!\n");
break;
default:
DEBUG("Auth state: %d\n", auth_state);
FATAL("Authentication failed (use -uv for more info)\n");
break;
}
}
/* Schedule all URLs in the command line for scanning. */
while (optind < argc) {
struct http_request *req;
/* Support @ notation for reading URL lists from files. */
if (argv[optind][0] == '@') {
read_urls((u8*)argv[optind++] + 1);
continue;
}
req = ck_alloc(sizeof(struct http_request));
if (parse_url((u8*)argv[optind], req, NULL))
FATAL("Scan target '%s' is not a valid absolute URL.", argv[optind]);
if (!url_allowed_host(req))
APPEND_FILTER(allow_domains, num_allow_domains,
__DFL_ck_strdup(req->host));
if (!url_allowed(req))
FATAL("URL '%s' explicitly excluded by -I / -X rules.", argv[optind]);
maybe_add_pivot(req, NULL, 2);
destroy_request(req);
optind++;
}
/* Char-by char stdin. */
tcgetattr(0, &term);
term.c_lflag &= ~ICANON;
tcsetattr(0, TCSANOW, &term);
fcntl(0, F_SETFL, O_NONBLOCK);
gettimeofday(&tv, NULL);
st_time = tv.tv_sec * 1000LL + tv.tv_usec / 1000;
#ifdef SHOW_SPLASH
if (!no_statistics) splash_screen();
#endif /* SHOW_SPLASH */
if (!no_statistics) SAY("\x1b[H\x1b[J");
else SAY(cLGN "[*] " cBRI "Scan in progress, please stay tuned...\n");
u64 refresh_time = 0;
/* Enter the crawler loop */
while ((next_from_queue() && !stop_soon) || (!show_once++)) {
u8 keybuf[8];
u64 end_time;
u64 run_time;
struct timeval tv_tmp;
gettimeofday(&tv_tmp, NULL);
end_time = tv_tmp.tv_sec * 1000LL + tv_tmp.tv_usec / 1000;
run_time = end_time - st_time;
if (gtimeout > 0 && run_time && run_time/1000 > gtimeout) {
DEBUG("* Stopping scan due to timeout\n");
stop_soon = 1;
}
req_sec = (req_count - queue_cur / 1.15) * 1000 / (run_time + 1);
if (no_statistics || ((loop_cnt++ % 100) && !show_once && idle == 0))
continue;
if (end_time > refresh_time) {
refresh_time = (end_time + 10);
if (clear_screen) {
SAY("\x1b[H\x1b[2J");
clear_screen = 0;
}
SAY(cYEL "\x1b[H"
"skipfish version " VERSION " by [email protected]\n\n"
cBRI " -" cPIN " %s " cBRI "-\n\n" cNOR,
allow_domains[0]);
if (!display_mode) {
http_stats(st_time);
SAY("\n");
database_stats();
} else {
http_req_list();
}
SAY(" \r");
}
if (fread(keybuf, 1, sizeof(keybuf), stdin) > 0) {
display_mode ^= 1;
clear_screen = 1;
}
}
gettimeofday(&tv, NULL);
en_time = tv.tv_sec * 1000LL + tv.tv_usec / 1000;
SAY("\n");
if (stop_soon)
SAY(cYEL "[!] " cBRI "Scan aborted by user, bailing out!" cNOR "\n");
term.c_lflag |= ICANON;
tcsetattr(0, TCSANOW, &term);
fcntl(0, F_SETFL, O_SYNC);
save_keywords((u8*)wordlist);
write_report(output_dir, en_time - st_time, seed);
#ifdef LOG_STDERR
SAY("\n== PIVOT DEBUG ==\n");
dump_pivots(0, 0);
SAY("\n== END OF DUMP ==\n\n");
#endif /* LOG_STDERR */
SAY(cLGN "[+] " cBRI "This was a great day for science!" cRST "\n\n");
#ifdef DEBUG_ALLOCATOR
if (!stop_soon) {
destroy_database();
destroy_signature_lists();
destroy_http();
destroy_signatures();
__TRK_report();
}
#endif /* DEBUG_ALLOCATOR */
fflush(0);
EVP_cleanup();
CRYPTO_cleanup_all_ex_data();
return 0;
}
/* main: this is supposedly required for a C program
* 1. Create threads to handle clients
* 2. Add clients from the file
* 3. Join threads and leave
* Returns 0 if everything was done correctly.
*/
int main(int argc, char** argv)
{
int num_queries, num_threads;
long i;
pthread_t * tids;
if(argc != 3)
{
fprintf(stderr,
"USAGE: %s [input_file_path] [num_threads]\n",
argv[0]);
exit(1);
}
// do preliminary work (database, queue, etc.)
setup_queue();
populate_database("TwitterDB.txt");
num_threads = atoi(argv[2]);
sem_init(&queue_access, 0, 1);
sem_init(&empty, 0, 0);
sem_init(&served, 0, 1);
sem_init(&full, 0, num_threads);
// create threads to handle clients
tids = calloc(num_threads, sizeof(pthread_t));
for(i = 0; i < num_threads; i++)
{
if(pthread_create(&tids[i], NULL, serveClients, (void *)(i + 1)))
{
perror("main: could not create create thread");
exit(1);
}
}
// populate queue
num_queries = buildQueue(argv[1]);
while(num_served != num_queries);
// cancel threads
for (i = 0; i < num_threads; i++)
{
if(pthread_cancel(tids[i]))
{
perror("main: could not join thread");
exit(1);
}
}
// clean up
destroy_database();
destroy_queue();
free(tids);
printf("Finished handling all clients\n");
return 0;
}
static void create_database (int list)
{
destroy_database ();
database = g_hash_table_new_full ((GHashFunc) str_hash, (GEqualFunc)
str_equal, NULL, (GDestroyNotify) item_free);
int entries = aud_playlist_entry_count (list);
for (int e = 0; e < entries; e ++)
{
char * title, * artist, * album;
Item * artist_item, * album_item, * title_item;
aud_playlist_entry_describe (list, e, & title, & artist, & album, TRUE);
if (! title)
{
str_unref (artist);
str_unref (album);
continue;
}
if (! artist)
artist = str_get (_("Unknown Artist"));
if (! album)
album = str_get (_("Unknown Album"));
artist_item = g_hash_table_lookup (database, artist);
if (! artist_item)
{
/* item_new() takes ownership of reference to artist */
artist_item = item_new (ARTIST, artist, NULL);
g_hash_table_insert (database, artist, artist_item);
}
else
str_unref (artist);
g_array_append_val (artist_item->matches, e);
album_item = g_hash_table_lookup (artist_item->children, album);
if (! album_item)
{
/* item_new() takes ownership of reference to album */
album_item = item_new (ALBUM, album, artist_item);
g_hash_table_insert (artist_item->children, album, album_item);
}
else
str_unref (album);
g_array_append_val (album_item->matches, e);
title_item = g_hash_table_lookup (album_item->children, title);
if (! title_item)
{
/* item_new() takes ownership of reference to title */
title_item = item_new (TITLE, title, album_item);
g_hash_table_insert (album_item->children, title, title_item);
}
else
str_unref (title);
g_array_append_val (title_item->matches, e);
}
}
