int run_page_rank (FILE *f, int argc, char **argv) { char line[128]; PageRank *pr = page_rank_new (); int seed; double *r; seed = page_rank_find_node (pr, "-"); page_rank_add_edge (pr, "-", "rillian"); for (;;) { char *from; char *to; char *color; if (fgets (line, sizeof(line), f) == NULL) break; if (get_graph_line (line, &from, &to, &color)) page_rank_add_edge (pr, from, to); } r = page_rank_compute (pr, seed, 0.15); #if 0 print_page_rank (pr, r); #endif g_free (r); return 0; }
PageRankScorerError page_rank_scorer_new(PageRankScorer **prs, PageDB *db) { PageRankScorer *p = *prs = malloc(sizeof(*p)); if (!p) return page_rank_scorer_error_memory; p->error = error_new(); if (p->error == 0) { free(p); return page_rank_scorer_error_memory; } p->page_db = db; if (page_rank_new(&p->page_rank, db->path, 1000) != 0) { page_rank_scorer_set_error(p, page_rank_scorer_error_internal, __func__); page_rank_scorer_add_error(p, "initializing PageRank"); page_rank_scorer_add_error(p, p? p->error->message: "NULL"); return p->error->code; } page_rank_scorer_set_persist(p, PAGE_RANK_SCORER_PERSIST); page_rank_scorer_set_use_content_scores(p, PAGE_RANK_SCORER_USE_CONTENT_SCORES); return 0; }
/* Checks the accuracy of the PageRank computation */ void test_page_rank(CuTest *tc) { printf("%s\n", __func__); /* Compute the PageRank score of the following graph * +-->2---+ * | | | * | v v * 1-->5<--3 * ^ ^ | * | | | * +---4<--+ * * The link matrix L[i,j], where L[i,j] = 1 means 'i' links to 'j' is: * * +- -+ * | 0 1 0 0 1 | * | 0 0 1 0 1 | * L = | 0 0 0 1 1 | * | 1 0 0 0 1 | * | 0 0 0 0 0 | * +- -+ * * Since page "5" has no outbound links, it is assumed it links to every other page: * * +- -+ * | 0 1 0 0 1 | * | 0 0 1 0 1 | * L = | 0 0 0 1 1 | * | 1 0 0 0 1 | * | 1 1 1 1 1 | * +- -+ * * The out degree is: * * deg = {2, 2, 2, 2, 5} * * Dividing each row with the out degree and transposing we get the matrix: * * M[i, j] = L[j, i]/deg[j] * * we get: * * +- -+ * | 0 0 0 0.5 0.2 | * | 0.5 0 0 0 0.2 | * M = | 0 0.5 0 0 0.2 | * | 0 0 0.5 0 0.2 | * | 0.5 0.5 0.5 0.5 0.2 | * +- -+ * * If 'd' is the damping then the PageRank 'PR' is: * * 1 - d * PR = ----- + (d * M)*PR * N * * For d=0.85 the numerical solution is: * * PR(1) = PR(2) = PR(3) = PR(4) = 0.15936255 * PR(5) = 0.3625498 */ char test_dir[] = "test-pagedb-XXXXXX"; mkdtemp(test_dir); PageDB *db; int ret = page_db_new(&db, test_dir); CuAssert(tc, db!=0? db->error->message: "NULL", ret == 0); db->persist = 0; char *urls[5] = {"1", "2", "3", "4", "5" }; LinkInfo links_1[] = {{"2", 0.1}, {"5", 0.1}}; LinkInfo links_2[] = {{"3", 0.1}, {"5", 0.1}}; LinkInfo links_3[] = {{"4", 0.1}, {"5", 0.1}}; LinkInfo links_4[] = {{"1", 0.1}, {"5", 0.1}}; LinkInfo *links[5] = { links_1, links_2, links_3, links_4, 0 }; int n_links[5] = {2, 2, 2, 2, 0}; for (int i=0; i<5; ++i) { CrawledPage *cp = crawled_page_new(urls[i]); for (int j=0; j<n_links[i]; ++j) crawled_page_add_link(cp, links[i][j].url, links[i][j].score); cp->score = i/5.0; crawled_page_set_hash64(cp, i); PageInfoList *pil; CuAssert(tc, db->error->message, page_db_add(db, cp, &pil) == 0); page_info_list_delete(pil); crawled_page_delete(cp); } // Without content scores // ------------------------------------------------------------------------ PageDBLinkStream *st; CuAssert(tc, db->error->message, page_db_link_stream_new(&st, db) == 0); st->only_diff_domain = 0; PageRank *pr; ret = page_rank_new(&pr, test_dir, 5); CuAssert(tc, pr!=0? pr->error->message: "NULL", ret == 0); pr->precision = 1e-6; CuAssert(tc, pr->error->message, page_rank_compute(pr, st, page_db_link_stream_next, page_db_link_stream_reset) == 0); page_db_link_stream_delete(st); uint64_t idx; float *score; float scores[5] = {0.15936255, 0.15936255, 0.15936255, 0.15936255, 0.3625498}; for (int i=0; i<5; ++i) { CuAssert(tc, db->error->message, page_db_get_idx(db, page_db_hash(urls[i]), &idx) == 0); CuAssertPtrNotNull(tc, score = mmap_array_idx(pr->value1, idx)); CuAssertDblEquals(tc, scores[i], *score, 1e-6); } CHECK_DELETE(tc, pr->error->message, page_rank_delete(pr)); // With content scores, damping = 0 // ------------------------------------------------------------------------ CuAssert(tc, db->error->message, page_db_link_stream_new(&st, db) == 0); st->only_diff_domain = 0; ret = page_rank_new(&pr, test_dir, 5); CuAssert(tc, pr!=0? pr->error->message: "NULL", ret == 0); pr->precision = 1e-6; pr->damping = 0.0; CuAssert(tc, db->error->message, page_db_get_scores(db, &pr->scores) == 0); CuAssert(tc, pr->error->message, page_rank_compute(pr, st, page_db_link_stream_next, page_db_link_stream_reset) == 0); page_db_link_stream_delete(st); float total_score = 0.0; for (int i=0; i<5; ++i) total_score += scores[i] = *((float*)mmap_array_idx(pr->scores, i)); for (int i=0; i<5; ++i) { CuAssert(tc, db->error->message, page_db_get_idx(db, page_db_hash(urls[i]), &idx) == 0); CuAssertPtrNotNull(tc, score = mmap_array_idx(pr->value1, idx)); CuAssertDblEquals(tc, scores[idx]/total_score, *score, 1e-6); } CHECK_DELETE(tc, pr->scores->error->message, mmap_array_delete(pr->scores)); CHECK_DELETE(tc, pr->error->message, page_rank_delete(pr)); // With content scores, damping = 0.5 // ------------------------------------------------------------------------ CuAssert(tc, db->error->message, page_db_link_stream_new(&st, db) == 0); st->only_diff_domain = 0; ret = page_rank_new(&pr, test_dir, 5); CuAssert(tc, pr!=0? pr->error->message: "NULL", ret == 0); pr->precision = 1e-6; pr->damping = 0.5; CuAssert(tc, db->error->message, page_db_get_scores(db, &pr->scores) == 0); CuAssert(tc, pr->error->message, page_rank_compute(pr, st, page_db_link_stream_next, page_db_link_stream_reset) == 0); page_db_link_stream_delete(st); float expected_pr[] = { 0.06386554621848739, 0.08739495798319329, 0.1647058823529412, 0.25546218487394956, 0.4285714285714286 }; for (int i=0; i<5; ++i) { CuAssert(tc, db->error->message, page_db_get_idx(db, page_db_hash(urls[i]), &idx) == 0); CuAssertPtrNotNull(tc, score = mmap_array_idx(pr->value1, idx)); CuAssertDblEquals(tc, expected_pr[i], *score, 1e-6); } CHECK_DELETE(tc, pr->scores->error->message, mmap_array_delete(pr->scores)); CHECK_DELETE(tc, pr->error->message, page_rank_delete(pr)); page_db_delete(db); }