/**
* List_Splice_Init - join two lists and reinitialise the emptied list.
* @list: the new list to add.
* @head: the place to add it in the first list.
*
* The list at @list is reinitialised
*/
static MV_INLINE void List_Splice_Init(List_Head *list,
List_Head *head)
{
if (!List_Empty(list)) {
__List_Splice(list, head);
MV_LIST_HEAD_INIT(list);
}
}
static MV_INLINE List_Head* List_GetLast(List_Head *head)
{
List_Head * one = NULL;
if ( List_Empty(head) ) return NULL;
one = head->prev;
List_Del(one);
return one;
}
/**
* List_Splice - join two lists
* @list: the new list to add.
* @head: the place to add it in the first list.
*/
static MV_INLINE void List_Splice(List_Head *list, List_Head *head)
{
if (!List_Empty(list))
__List_Splice(list, head);
}
void *parse (void * context)
{
void (*_edge_fn) (char *from, char *to) = context;
char *url, *page;
node_t *entry;
while (!areQueuesEmpty ())
{
//critical section: dequeue page from shared unbounded queue acting as consumer
//printf("in parser\n");
pthread_mutex_lock(&lock2);
while (List_Empty(queue_pages))
pthread_cond_wait (&fill_unbounded, &lock2);
entry = List_Get (queue_pages);
pthread_mutex_unlock(&lock2);
url = entry->key;
page = entry->val;
//printf("parsing %s\n",url);
//core logic for parsing a page
char *save_ptr1, *token;
char *delim = " \n\t";
regex_t regex;
char *expr = "^link:.*";
int reti = regcomp (®ex, expr, 0);
if (reti)
{
//printf (stderr, "Could not compile regex\n");
exit (1);
}
char *str1 = page;
int has_atleast_a_link = 0;
for (;; str1 = NULL)
{
token = strtok_r (str1, delim, &save_ptr1);
if (token != NULL)
{
if (!(regexec (®ex, token, 0, NULL, 0)) && strlen(token)>5)
{
//printf ("%s matched: %s len=%d\n", url,token, (int) strlen (token));
char *addr = strndup (token + 5, (int) strlen (token) - 5);
//critical section: insert into shared bounded queue acting as producer
pthread_mutex_lock (&lock1);
while (count == QUEUE_SIZE)
pthread_cond_wait (&empty, &lock1);
//check if url is not already visited
pthread_mutex_unlock (&lock1);
_edge_fn(url,addr);
pthread_mutex_lock (&lock1);
if(lookup_string(urls, addr)==NULL){
has_atleast_a_link = 1;
add_string(urls, addr);
put(addr);
pthread_mutex_lock(&glock);
sum_queue_lengths++;
pthread_mutex_unlock(&glock);
//_edge_fn(url,addr);
pthread_cond_signal (&fill);
}
pthread_mutex_unlock (&lock1);
}
}
else
break;
}
pthread_mutex_lock(&glock);
sum_queue_lengths--;
pthread_mutex_unlock(&glock);
/*
if (has_atleast_a_link == 0)
{
pthread_mutex_lock (&glock);
sum_queue_lengths--;
pthread_mutex_unlock (&glock);
}
*/
}
thr_exit ();
//printf("parser thread exiting\n");
return NULL;
}
