void * child(void *arg) { printf("child\n"); done = 1; Cond_signal(&c); return NULL; }
void download(node *n) { //printf(" -->download called on %s.\n", n->link); char *page = fetch_fn(n->link); assert(page != NULL); //printf("PAGE: %s\n", page); // page producer Mutex_lock(&p_m); // page queue is unbounded, so no waiting // do pushing pages = push(page, strdup(n->link), pages); page_count++; printf("PUSH page: %s\n", n->link); printf("--> new link_count: %d\n", link_count); printf("--> curr page_count: %d\n", page_count); Cond_signal(&p_fill); Mutex_unlock(&p_m); // will free page later when it comes off of the page queue return; }
void * child(void *arg) { printf("child\n"); Mutex_lock(&m); Cond_signal(&c); Mutex_unlock(&m); return NULL; }
void *downloader(void *arg) { //printf("***downloader started!\n"); // loop forever while(1) { Mutex_lock(&l_m); while(link_count == 0) { //printf("***downloader waiting!\n"); // wait Cond_wait(&l_fill, &l_m); } // make a new pointer to the top of the stack node *next = links; // pop the top of the stack (will need to free the node and the string later) links = pop(links); link_count--; printf("POP link: %s\n", next->link); printf("--> link_count: %d\n", link_count); printf("--> page_count: %d\n", page_count); // signal links empty Cond_signal(&l_empty); Mutex_unlock(&l_m); // do actual processing download(next); // free node free(next->link); free(next); //Mutex_lock(&l_m); //link_count--; //Mutex_unlock(&l_m); } return 0; }
void do_tokenizing(char *orig, char *source) { //printf("***tokenizing...\n------------------------\n%s\n----------------------\n", orig); char *str = strdup(orig); char *curr, *new_str, *word; char *saveptr1, *saveptr2; while( (curr = strstr(str, "link:")) != NULL) { new_str = strdup(curr+1); if(curr == str || (curr > str && is_blank_space(*(curr-1)))) { curr = strtok_r(curr, " \n", &saveptr1); assert(curr != NULL); word = strtok_r(curr, ":", &saveptr2); word = strtok_r(NULL, ":", &saveptr2); //printf("found word: %s\n", word); assert(word != NULL); edge_fn(source, word); // check hash set and, if not present, add // link producer Mutex_lock(&l_m); while(link_count == max_link) { // wait Cond_wait(&l_empty, &l_m); } if(!contains(table, word)) { // do pushing printf("PUSH link: %s\n", word); links = push(NULL, strdup(word), links); link_count++; Mutex_lock(&done_m); total_work++; Mutex_unlock(&done_m); printf("--> new link_count: %d\n", link_count); printf("--> curr page_count: %d\n", page_count); // update table add(table, word); } Cond_signal(&l_fill); Mutex_unlock(&l_m); } free(str); str = new_str; } free(str); return; }
void *parser(void *arg) { /* * 1) queue of links (parsers to downloaders): first, push start_url onto queue * - fixed-size * - parsers push links on, wait when full (need cv) * - downloader wait when empty (need cv) * - need mutex */ while(1) { //printf("***parser started!\n"); Mutex_lock(&p_m); while(page_count == 0) { //printf("***parser waiting!\n"); // wait Cond_wait(&p_fill, &p_m); } // make a new pointer to the top of the stack node *next = pages; // pop the top of the stack (will need to free the node and the string later) pages = pop(pages); page_count--; printf("POP page: %s\n", next->link); printf("--> new link_count: %d\n", link_count); printf("--> curr page_count: %d\n", page_count); // signal pages empty: may not need to do this since nobody will be waiting on // pages to be reduced in size (unbounded) // Cond_signal(&p_empty); Mutex_unlock(&p_m); // do actual processing parse(next); // free node free(next->str); free(next->link); free(next); //Mutex_lock(&p_m); //page_count--; //Mutex_unlock(&p_m); //Mutex_lock(&p_m); //Mutex_lock(&l_m); Mutex_lock(&done_m); total_work--; printf("link count: %d\npage count: %d\n", link_count, page_count); //if(page_count == 0 && link_count == 0) { if(total_work == 0) { //done = 1; Cond_signal(&done_cv); } Mutex_unlock(&done_m); //Mutex_unlock(&l_m); //Mutex_unlock(&p_m); } return 0; }