/*
* Main function.
* Reads command line arguments in the format:
* ./WordCounter pipe_name destination_log_file_name
* Where pipe_name is the name of FIFO pipe that the Listener should create and
* destination_log_file_name is the destination file name where the log should be written.
* This function should create the files queue and prepare the parameters to the Listener and
* WordCounter threads. Then, it should create these threads.
* After threads are created, this function should control them as follows:
* it should read input from user, and if the input line is "exit" (or "exit\n"), it should
* set the files queue as "finished". This should make the threads terminate (possibly only
* when the next connection is received).
* At the end the function should join the threads and exit.
*/
int main(int argc, char *argv[]) {
pthread_t wordcounter, listener;
char *pipe_file_name, *destination_log_file;
char input_buffer[STDIN_READ_BUFF_SIZE];
BoundedBuffer bounded_buff;
WordCounterData counter_data;
ListenerData listener_data;
// Check argument count
if (argc != 3) {
printf(INFO_USAGE);
return 1;
}
// Get arguments
pipe_file_name = argv[1];
destination_log_file = argv[2];
// Verify given file to readfrom exists
int result;
result = access (pipe_file_name, F_OK);
if (result != 0)
{
printf(INFO_EXSITANCE);
return 1;
}
// Initialize bounded buffer
bounded_buffer_init(&bounded_buff, MAX_BOUNDED_BUFF_CAPACITY);
// Initialize word counter data
counter_data.buff = &bounded_buff;
counter_data.log_file = fopen(destination_log_file, "w");
if (!counter_data.log_file) {
printf(ERR_CANT_OPEN, destination_log_file);
end_bounded_buffer(&bounded_buff);
terminate_bounded_buffer(&bounded_buff);
//Finish
return 1;
}
// Listener initialization
listener_data.buff = &bounded_buff;
listener_data.pipe = pipe_file_name;
printf(SHOW_LOG, destination_log_file);
// Start threads
pthread_create(&listener, NULL, run_listener, (void*)(&listener_data));
pthread_create(&wordcounter, NULL, run_wordcounter, (void*)(&counter_data));
// Read input in a loop until exit command is given
do {
// Get entered inout
fgets(input_buffer, STDIN_READ_BUFF_SIZE, stdin);
} while (strcmp(input_buffer, CMD_EXIT) != 0);
// Mark end
end_bounded_buffer(&bounded_buff);
// Join threads
pthread_join(wordcounter, NULL);
pthread_join(listener, NULL);
// Close log file
fclose(counter_data.log_file);
// Termination
terminate_bounded_buffer(&bounded_buff);
// END & EXIT
printf(EXIT_MSG);
return 0;
}
int crawl(char *start_url, int download_workers, int parse_workers, int queue_size,
char *(*_fetch_fn)(char *url), void (*_edge_fn)(char *from, char *to)) {
int i;
bounded_buffer_t url_queue;
unbounded_buffer_t page_queue;
hashset_t url_set;
bounded_buffer_init(&url_queue, queue_size);
unbounded_buffer_init(&page_queue);
hashset_init(&url_set, HASHSET_BUCKETS);
bounded_buffer_put(&url_queue, (void *)str_duplicate(start_url));
mutex_t done_mutex;
cond_t done_cond;
mutex_init(&done_mutex);
cond_init(&done_cond);
struct input_args in_args;
in_args.url_queue = &url_queue;
in_args.page_queue = &page_queue;
in_args.url_set = &url_set;
in_args.fetch = _fetch_fn;
in_args.edge = _edge_fn;
in_args.done_mutex = &done_mutex;
in_args.done_cond = &done_cond;
thread_t downloaders[download_workers];
thread_t parsers[parse_workers];
for (i = 0; i < download_workers; i++)
thread_create(&downloaders[i], downloader, (void *)&in_args);
for (i = 0; i < parse_workers; i++)
thread_create(&parsers[i], parser, (void *)&in_args);
while (1) {
mutex_lock(&done_mutex);
mutex_lock(&url_queue.mutex);
mutex_lock(&url_queue.worker_mutex);
mutex_lock(&page_queue.mutex);
mutex_lock(&page_queue.worker_mutex);
if (url_queue.count == 0 && url_queue.workers == 0 &&
page_queue.count == 0 && page_queue.workers == 0) {
url_queue.done = 1;
page_queue.done = 1;
cond_broadcast(&url_queue.empty);
cond_broadcast(&url_queue.fill);
cond_broadcast(&page_queue.fill);
mutex_unlock(&url_queue.mutex);
mutex_unlock(&url_queue.worker_mutex);
mutex_unlock(&page_queue.mutex);
mutex_unlock(&page_queue.worker_mutex);
mutex_unlock(&done_mutex);
break;
} else {
mutex_unlock(&url_queue.mutex);
mutex_unlock(&url_queue.worker_mutex);
mutex_unlock(&page_queue.mutex);
mutex_unlock(&page_queue.worker_mutex);
cond_wait(&done_cond, &done_mutex);
mutex_unlock(&done_mutex);
}
}
for (i = 0; i < download_workers; i++)
thread_join(downloaders[i], NULL);
for (i = 0; i < parse_workers; i++)
thread_join(parsers[i], NULL);
bounded_buffer_destroy(&url_queue);
unbounded_buffer_destroy(&page_queue);
hashset_destroy(&url_set);
return 0;
}
