void *
child(void *arg) {
printf("child\n");
done = 1;
Cond_signal(&c);
return NULL;
}
void download(node *n) {
//printf(" -->download called on %s.\n", n->link);
char *page = fetch_fn(n->link);
assert(page != NULL);
//printf("PAGE: %s\n", page);
// page producer
Mutex_lock(&p_m);
// page queue is unbounded, so no waiting
// do pushing
pages = push(page, strdup(n->link), pages);
page_count++;
printf("PUSH page: %s\n", n->link);
printf("--> new link_count: %d\n", link_count);
printf("--> curr page_count: %d\n", page_count);
Cond_signal(&p_fill);
Mutex_unlock(&p_m);
// will free page later when it comes off of the page queue
return;
}
void *
child(void *arg) {
printf("child\n");
Mutex_lock(&m);
Cond_signal(&c);
Mutex_unlock(&m);
return NULL;
}
void *downloader(void *arg) {
//printf("***downloader started!\n");
// loop forever
while(1) {
Mutex_lock(&l_m);
while(link_count == 0) {
//printf("***downloader waiting!\n");
// wait
Cond_wait(&l_fill, &l_m);
}
// make a new pointer to the top of the stack
node *next = links;
// pop the top of the stack (will need to free the node and the string later)
links = pop(links);
link_count--;
printf("POP link: %s\n", next->link);
printf("--> link_count: %d\n", link_count);
printf("--> page_count: %d\n", page_count);
// signal links empty
Cond_signal(&l_empty);
Mutex_unlock(&l_m);
// do actual processing
download(next);
// free node
free(next->link);
free(next);
//Mutex_lock(&l_m);
//link_count--;
//Mutex_unlock(&l_m);
}
return 0;
}
void do_tokenizing(char *orig, char *source) {
//printf("***tokenizing...\n------------------------\n%s\n----------------------\n", orig);
char *str = strdup(orig);
char *curr, *new_str, *word;
char *saveptr1, *saveptr2;
while( (curr = strstr(str, "link:")) != NULL) {
new_str = strdup(curr+1);
if(curr == str || (curr > str && is_blank_space(*(curr-1)))) {
curr = strtok_r(curr, " \n", &saveptr1);
assert(curr != NULL);
word = strtok_r(curr, ":", &saveptr2);
word = strtok_r(NULL, ":", &saveptr2);
//printf("found word: %s\n", word);
assert(word != NULL);
edge_fn(source, word);
// check hash set and, if not present, add
// link producer
Mutex_lock(&l_m);
while(link_count == max_link) {
// wait
Cond_wait(&l_empty, &l_m);
}
if(!contains(table, word)) {
// do pushing
printf("PUSH link: %s\n", word);
links = push(NULL, strdup(word), links);
link_count++;
Mutex_lock(&done_m);
total_work++;
Mutex_unlock(&done_m);
printf("--> new link_count: %d\n", link_count);
printf("--> curr page_count: %d\n", page_count);
// update table
add(table, word);
}
Cond_signal(&l_fill);
Mutex_unlock(&l_m);
}
free(str);
str = new_str;
}
free(str);
return;
}
void *parser(void *arg) {
/*
* 1) queue of links (parsers to downloaders): first, push start_url onto queue
* - fixed-size
* - parsers push links on, wait when full (need cv)
* - downloader wait when empty (need cv)
* - need mutex
*/
while(1) {
//printf("***parser started!\n");
Mutex_lock(&p_m);
while(page_count == 0) {
//printf("***parser waiting!\n");
// wait
Cond_wait(&p_fill, &p_m);
}
// make a new pointer to the top of the stack
node *next = pages;
// pop the top of the stack (will need to free the node and the string later)
pages = pop(pages);
page_count--;
printf("POP page: %s\n", next->link);
printf("--> new link_count: %d\n", link_count);
printf("--> curr page_count: %d\n", page_count);
// signal pages empty: may not need to do this since nobody will be waiting on
// pages to be reduced in size (unbounded)
// Cond_signal(&p_empty);
Mutex_unlock(&p_m);
// do actual processing
parse(next);
// free node
free(next->str);
free(next->link);
free(next);
//Mutex_lock(&p_m);
//page_count--;
//Mutex_unlock(&p_m);
//Mutex_lock(&p_m);
//Mutex_lock(&l_m);
Mutex_lock(&done_m);
total_work--;
printf("link count: %d\npage count: %d\n", link_count, page_count);
//if(page_count == 0 && link_count == 0) {
if(total_work == 0) {
//done = 1;
Cond_signal(&done_cv);
}
Mutex_unlock(&done_m);
//Mutex_unlock(&l_m);
//Mutex_unlock(&p_m);
}
return 0;
}
