void * do_crawler(void *item)
{
char *url_ptr;
int clientfd ;
static int pages = 0;
static int error = 0;
char *buf = NULL;
urlq_t *url_list_head = NULL, *p, *p_pre;
char cur_dir[256];
hash_table *hash_in, *hash_out;
/* int tid = pthread_self();*/
int j = 0;
int i= 0;
int pos_found = 0;
char temp[256];
int status = 0;
static int a = 0;
static int b = 0;
static int c = 0;
static int d = 0;
static int e = 0;
static int f = 0;
static int g = 0;
while(1){
pthread_mutex_lock(&mutex);
while (urlqueue.head_ptr->next == NULL){
pthread_cond_wait(&ready, &mutex);
}
pthread_mutex_lock(&((threadpool_item *)item)->waitlock);
((threadpool_item *)item)->idle = 0;
pthread_mutex_unlock(&((threadpool_item *)item)->waitlock);
url_ptr = queue_pop(&urlqueue);
g++;
pthread_mutex_unlock(&mutex);
pthread_mutex_lock(&mutex);
/*if not visited, set flag = 1*/
if(has_visited(hash, url_ptr) == 1){
pthread_mutex_unlock(&mutex);
free(url_ptr);
pthread_mutex_lock(&((threadpool_item *)item)->waitlock);
((threadpool_item *)item)->idle =1;
pthread_mutex_unlock(&((threadpool_item *)item)->waitlock);
d++;
continue;
}
hash_out = has_url(hash, url_ptr);
e++;
pthread_mutex_unlock(&mutex);
if (hash_out == NULL){
printf("error\n");
getchar();
}
*temp = '\0';
cur_dir[0] = '\0';
strcpy(cur_dir, url_ptr);
j = strlen(cur_dir);
for (;cur_dir[j] != '/' && j != 0; j--) ;
if(j == 0)
cur_dir[j] = '\0';
else
cur_dir[j+1] = '\0';
for (i = 0; i < 3; i++){
if((clientfd = open_tcp("127.0.0.1", 80)) < 0){
close_fd(clientfd);
continue;
}
if( http_do_get(clientfd, rootdir, "127.0.0.1", url_ptr) < 0){
close_fd(clientfd);
continue;
}
if(recv_line(clientfd, temp) <= 0){
close_fd(clientfd);
continue;
}
if((status = http_response_status(temp)) == 4){
printf("%s error %d\n",url_ptr, error++);
pthread_mutex_lock(&mutex);
set_status(hash, url_ptr, 4);
set_webg_status(webg, hash_out, 4);
pthread_mutex_unlock(&mutex);
pthread_mutex_lock(&((threadpool_item *)item)->waitlock);
((threadpool_item *)item)->idle =1;
pthread_mutex_unlock(&((threadpool_item *)item)->waitlock);
close_fd(clientfd);
break;
}
buf = http_response_body(clientfd);
close_fd(clientfd);
break;
}
if (status == 4)
continue;
if(i == 3){
pthread_mutex_lock(&((threadpool_item *)item)->waitlock);
((threadpool_item *)item)->idle =1;
pthread_mutex_unlock(&((threadpool_item *)item)->waitlock);
close_fd(clientfd);
continue;
}
if (buf == NULL){
pthread_mutex_lock(&((threadpool_item *)item)->waitlock);
((threadpool_item *)item)->idle =1;
pthread_mutex_unlock(&((threadpool_item *)item)->waitlock);
continue;
}
printf("%s pages %d\n", url_ptr,pages++);
extract_link(buf, cur_dir, &url_list_head);
free(buf);
buf = NULL;
p = url_list_head->next;
p_pre = url_list_head;
while (p != NULL){
if(strcmp(url_ptr, p->url_ptr) == 0){
p_pre->next = p->next;
free(p->url_ptr);
free(p);
a++;
p = p_pre->next;
printf("a= %d, b= %d, c= %d, d= %d, e= %d, f= %d, g= %d\n", a,b,c,d,e,f,g);
continue;
}
pthread_mutex_lock(&mutex);
hash_in = has_url(hash, p->url_ptr);
if (hash_in != NULL ){
insert_edge(webg, hash_in, hash_out);
pthread_mutex_unlock(&mutex);
p_pre->next = p->next;
free(p->url_ptr);
free(p);
p = p_pre->next;
b++;
printf("a= %d, b= %d, c= %d, d= %d, e= %d, f= %d, g= %d\n", a,b,c,d,e,f,g);
continue;
}
else{
pos_found = insert_vertex(webg, hash_out, p->url_ptr);
insert_hash_item(hash, p->url_ptr, pos_found, 0);
pthread_mutex_unlock(&mutex);
c++;
p_pre = p;
p = p->next;
printf("a= %d, b= %d, c= %d, d= %d, e= %d, f= %d, g= %d\n", a,b,c,d,e,f,g);
}
}
if(p_pre != url_list_head){
pthread_mutex_lock(&mutex);
queue_push(&urlqueue, url_list_head->next, p_pre);
f++;
pthread_mutex_unlock(&mutex);
}
free(url_list_head);
p = p_pre = url_list_head = NULL;
pthread_mutex_lock(&((threadpool_item *)item)->waitlock);
((threadpool_item *)item)->idle = 1;
pthread_mutex_unlock(&((threadpool_item *)item)->waitlock);
/*printf("next time!\n");*/
}
/*printf("over!\n");*/
return NULL;
}
int main()
{
graph_link gl;
init_graph(&gl);
insert_vertex(&gl, 'A');
insert_vertex(&gl, 'B');
insert_vertex(&gl, 'C');
insert_vertex(&gl, 'D');
insert_vertex(&gl, 'E');
insert_vertex(&gl, 'F');
insert_vertex(&gl, 'G');
insert_vertex(&gl, 'H');
insert_vertex(&gl, 'I');
insert_vertex(&gl, 'J');
insert_vertex(&gl, 'K');
insert_vertex(&gl, 'L');
insert_vertex(&gl, 'M');
insert_edge(&gl, 'A', 'B');
insert_edge(&gl, 'A', 'C');
insert_edge(&gl, 'A', 'F');
insert_edge(&gl, 'A', 'L');
insert_edge(&gl, 'B', 'M');
insert_edge(&gl, 'L', 'J');
insert_edge(&gl, 'L', 'M');
insert_edge(&gl, 'J', 'M');
insert_edge(&gl, 'D', 'E');
insert_edge(&gl, 'G', 'H');
insert_edge(&gl, 'G', 'I');
insert_edge(&gl, 'G', 'K');
insert_edge(&gl, 'H', 'K');
printf("\n");
show_graph(&gl);
printf("Depth First Search(DFS) all nodes of the graph: \n");
depth_first_search(&gl, 'D');
printf("Nul\n");
printf("Breadth First Search(BFS) all nodes of the graph: \n");
breadth_first_search(&gl, 'A');
printf("Nul\n");
printf("Non connect graph DFS: \n");
components(&gl);
printf("Nul\n");
//int v = get_first_neighbor(&gl, 'A');
//printf("The first neighbor node of 'A' is: %d\n", v);
//int v1 = get_next_neighbor(&gl, 'B', 'E');
//printf("The next neighbor node of 'B' and 'E' is: %d\n", v1);
//printf("\n");
//delete_edge(&gl, 'B', 'C');
//show_graph(&gl);
//delete_vertex(&gl, 'C');
destroy_graph(&gl);
}
int interface(TNode *graph) {
int opt, v1, v2, c;
printf("Grafo:\n");
print_graph(graph);
help();
while (1) {
printf("> ");
scanf(" %d", &opt);
switch (opt) {
case 0:
help();
break;
case 1:
printf("Grafo:\n");
print_graph(graph);
break;
case 2:
printf("Inserir No. Digite um identificador para o no: \n>> ");
scanf(" %d", &v1);
graph = insert_node(graph, v1);
break;
case 3:
printf("Inserir Aresta. Digite os identificadores dos nos e o custo: \n>> ");
scanf(" %d %d %d", &v1, &v2, &c);
insert_edge(graph, v1, v2, c);
break;
case 4:
printf("Remover No. Digite um identificador para o no: \n>> ");
scanf(" %d", &v1);
graph = remove_node(graph, v1);
break;
case 5:
printf("Remover Aresta. Digite os identificadores dos nos: \n>> ");
scanf(" %d %d", &v1, &v2);
remove_edge(graph, v1, v2);
break;
case 6:
printf("Arvore geradora minima\n");
if (is_connected(graph)) {
TNode *k = kruskal(graph);
save_file(k, "avm.txt");
printf("Resultado salvo em avm.txt\n");
free_nodes(k);
} else {
printf("Grafo desconexo\n");
}
break;
case 7:
if (is_connected(graph)) {
printf("Caminho Minimo. Digite o identificador do no: \n>> ");
scanf(" %d", &v1);
int ret = dijkstra(graph, v1);
if (ret)
printf("Resultado salvo em cmc.txt\n");
else
printf("No nao existe!\n");
} else {
printf("Grafo desconexo\n");
}
break;
case 8:
printf("Salvando arquivo saida.txt\n");
save_file(graph, "saida.txt");
break;
case 9:
free_nodes(graph);
return 0;
}
}
}
int main(int argc, char *argv[]){
int source, dest, min_node, max_node;
int *communities2nodes;
int method, quantity_communities;
Queue nodes;
Graph graph;
FILE *arq;
min_node = MAX_WEIGHT;
max_node = -1;
if(argc >= 3){
arq = fopen(argv[1], "r");
if(arq == NULL)
printf("Erro, nao foi possivel abrir o arquivo\n");
else
while((fscanf(arq,"%d %d\n", &source, &dest)) !=EOF){
if(source < min_node)
min_node = source;
if(dest < min_node)
min_node = dest;
if(source > max_node)
max_node = source;
if(dest > max_node)
max_node = dest;
enqueue(&nodes, source);
enqueue(&nodes, dest);
}
fclose(arq);
graph.numbers_nodes = max_node - min_node + 1;
allocate_memory(&graph);
while(not empty(&nodes)){
source = dequeue(&nodes);
dest = dequeue(&nodes);
insert_edge(&graph, source - min_node, dest - min_node, 1);
}
quantity_communities = atoi(argv[2]);
if(argc < 4){
communities2nodes = calculate_communities_repeated_square(&graph, quantity_communities);
imprime(communities2nodes, graph.numbers_nodes, min_node);
communities2nodes = calculate_communities_floyd_warshall(&graph, quantity_communities);
imprime(communities2nodes, graph.numbers_nodes, min_node);
communities2nodes = calculate_communities_johnson_queue(&graph, quantity_communities);
imprime(communities2nodes, graph.numbers_nodes, min_node);
communities2nodes = calculate_communities_johnson_array(&graph, quantity_communities);
imprime(communities2nodes, graph.numbers_nodes, min_node);
communities2nodes = calculate_communities_nbfs(&graph, quantity_communities);
imprime(communities2nodes, graph.numbers_nodes, min_node);
} else{
method = atoi(argv[3]);
switch(method){
case 0:
communities2nodes = calculate_communities_repeated_square(&graph, quantity_communities);
break;
case 1:
communities2nodes = calculate_communities_floyd_warshall(&graph, quantity_communities);
break;
case 2:
communities2nodes = calculate_communities_johnson_queue(&graph, quantity_communities);
break;
case 3:
communities2nodes = calculate_communities_johnson_array(&graph, quantity_communities);
break;
case 4:
communities2nodes = calculate_communities_nbfs(&graph, quantity_communities);
break;
default:
printf("Metodo Invalido");
return 0;
};
}
imprime(communities2nodes, graph.numbers_nodes, min_node);
}
return 0;
}
int main () {
struct digraph *G = new_digraph (5);
struct digraph *tc_G;
/* make a funny digraph */
insert_edge (G, 0, 1);
insert_edge (G, 1, 2);
//insert_edge (G, 2, 3);
insert_edge (G, 3, 4);
insert_edge (G, 4, 0);
/* is the digraph connected? */
if (is_connected (G)) printf ("connected!\n"); else printf ("not connected!\n");
/* print the adjacency matrix */
printf ("adjacency matrix for G:\n");
print_adj_matrix (G);
/* find the transitive closure */
tc_G = Warshall (G);
/* print the adjacency matrix for the transitive closure */
printf ("adjacency matrix for transitive closure of G:\n");
print_adj_matrix (tc_G);
//return 0;
/* make a funny DAG */
struct digraph *H = new_digraph (7);
insert_edge (H, 3, 4);
insert_edge (H, 4, 2);
insert_edge (H, 3, 5);
insert_edge (H, 5, 2);
insert_edge (H, 2, 6);
insert_edge (H, 5, 3);
insert_edge (H, 6, 1);
insert_edge (H, 6, 0);
printf ("adjacency matrix for H:\n");
print_adj_matrix (H);
if (is_connected (H)) printf ("connected!\n"); else printf ("not connected!\n");
topological_sort (H, 3);
//return 0;
struct digraph *tc_H = Warshall (H);
printf ("transitive closure of H is:\n");
print_adj_matrix (tc_H);
return 0;
}
/* sets the edges for a series of parallel segments along two edges starting
* from segment i, segment j. It is assumed that the edge should be from
* segment i to segment j - the dependency is appropriately propagated
*/
static void
set_parallel_edges (segment* seg1, segment* seg2, int dir1, int dir2, int hops,
maze* mp)
{
int x;
channel* chan;
channel* nchan;
segment* prev1;
segment* prev2;
if (seg1->isVert)
chan = chanSearch(mp->vchans, seg1);
else
chan = chanSearch(mp->hchans, seg1);
insert_edge(chan->G, seg1->ind_no, seg2->ind_no);
for (x=1;x<=hops;x++) {
prev1 = next_seg(seg1, dir1);
prev2 = next_seg(seg2, dir2);
if(!seg1->isVert) {
nchan = chanSearch(mp->vchans, prev1);
if(prev1->comm_coord==seg1->p.p1) {
if(seg1->l1==B_UP) {
if(edge_exists(chan->G, seg1->ind_no, seg2->ind_no))
insert_edge(nchan->G, prev2->ind_no, prev1->ind_no);
else
insert_edge(nchan->G, prev1->ind_no, prev2->ind_no);
}
else {
if(edge_exists(chan->G, seg1->ind_no, seg2->ind_no))
insert_edge(nchan->G, prev1->ind_no, prev2->ind_no);
else
insert_edge(nchan->G, prev2->ind_no, prev1->ind_no);
}
}
else {
if(seg1->l2==B_UP) {
if(edge_exists(chan->G, seg1->ind_no, seg2->ind_no))
insert_edge(nchan->G,prev1->ind_no, prev2->ind_no);
else
insert_edge(nchan->G,prev2->ind_no, prev1->ind_no);
}
else {
if(edge_exists(chan->G, seg1->ind_no, seg2->ind_no))
insert_edge(nchan->G, prev2->ind_no, prev1->ind_no);
else
insert_edge(nchan->G, prev1->ind_no, prev2->ind_no);
}
}
}
else {
nchan = chanSearch(mp->hchans, prev1);
if(prev1->comm_coord==seg1->p.p1) {
if(seg1->l1==B_LEFT) {
if(edge_exists(chan->G, seg1->ind_no, seg2->ind_no))
insert_edge(nchan->G, prev1->ind_no, prev2->ind_no);
else
insert_edge(nchan->G, prev2->ind_no, prev1->ind_no);
}
else {
if(edge_exists(chan->G, seg1->ind_no, seg2->ind_no))
insert_edge(nchan->G, prev2->ind_no, prev1->ind_no);
else
insert_edge(nchan->G, prev1->ind_no, prev2->ind_no);
}
}
else {
if(seg1->l2==B_LEFT) {
if(edge_exists(chan->G, seg1->ind_no, seg2->ind_no))
insert_edge(nchan->G, prev2->ind_no, prev1->ind_no);
else
insert_edge(nchan->G, prev1->ind_no, prev2->ind_no);
}
else {
if(edge_exists(chan->G, seg1->ind_no, seg2->ind_no))
insert_edge(nchan->G, prev1->ind_no, prev2->ind_no);
else
insert_edge(nchan->G, prev2->ind_no, prev1->ind_no);
}
}
}
chan = nchan;
seg1 = prev1;
seg2 = prev2;
}
}
