static int te_calc_num_ret(const EPI *epi, const REL_INFO *rel_info, const RESULTS *results, const TREC_MEAS *tm, TREC_EVAL *eval) { RES_RELS res_rels; /* Can't just use results, since epi->only_judged_docs may be set */ if (UNDEF == te_form_res_rels(epi, rel_info, results, &res_rels)) return (UNDEF); eval->values[tm->eval_index].value = (double) res_rels.num_ret; return (1); }
static int te_calc_set_map (const EPI *epi, const REL_INFO *rel_info, const RESULTS *results, const TREC_MEAS *tm, TREC_EVAL *eval) { RES_RELS res_rels; if (UNDEF == te_form_res_rels (epi, rel_info, results, &res_rels)) return (UNDEF); if (res_rels.num_ret && res_rels.num_rel) eval->values[tm->eval_index].value = ((double) res_rels.num_rel_ret * (double) res_rels.num_rel_ret) / ((double) res_rels.num_ret * (double) res_rels.num_rel); return (1); }
static int te_calc_gm_ndcg_cut(const EPI *epi, const REL_INFO *rel_info, const RESULTS *results, const TREC_MEAS *tm, TREC_EVAL *eval) { long *cutoffs = (long *) tm->meas_params->param_values; long cutoff_index = 0; RES_RELS res_rels; double gain, sum; double ideal_dcg; /* ideal discounted cumulative gain */ double ndcg; long cur_lvl, lvl_count; long i; if (UNDEF == te_form_res_rels(epi, rel_info, results, &res_rels)) { return (UNDEF); } sum = 0.0; for (i = 0; i < res_rels.num_ret; i++) { if (i == cutoffs[cutoff_index]) { /* Calculate previous cutoff threshold. Note i guaranteed to be positive by init_meas */ eval->values[tm->eval_index + cutoff_index].value = sum; if (++cutoff_index == tm->meas_params->num_params) { break; } if (epi->debug_level > 0) { printf("ndcg_cut: cutoff %ld dcg %6.4f\n", i, sum); } } gain = res_rels.results_rel_list[i]; if (gain > 0) { /* Note: i+2 since doc i has rank i+1 */ sum += gain / log2((double) (i + 2)); if (epi->debug_level > 1) { printf("ndcg_cut:%ld %3.1f %6.4f\n", i, gain, sum); } } } /* calculate values for those cutoffs not achieved */ while (cutoff_index < tm->meas_params->num_params) { eval->values[tm->eval_index + cutoff_index].value = sum; if (epi->debug_level > 0) { printf("ndcg_cut: cutoff %ld dcg %6.4f\n", cutoffs[cutoff_index], sum); } cutoff_index++; } /* Calculate ideal discounted cumulative gain for this topic, and normalize previous sum by it */ cutoff_index = 0; cur_lvl = res_rels.num_rel_levels - 1; lvl_count = 0; ideal_dcg = 0.0; for (i = 0; 1; i++) { lvl_count++; while (cur_lvl > 0 && lvl_count > res_rels.rel_levels[cur_lvl]) { cur_lvl--; lvl_count = 1; } if (cur_lvl == 0) { break; } if (i == cutoffs[cutoff_index]) { /* Calculate previous cutoff threshold. Note i guaranteed to be positive by init_meas */ if (ideal_dcg > 0.0) { ndcg = eval->values[tm->eval_index + cutoff_index].value / ideal_dcg; eval->values[tm->eval_index + cutoff_index].value = (double) log((double) (MAX(ndcg, MIN_GEO_MEAN))); } if (epi->debug_level > 0) { printf("ndcg_cut: cutoff %ld idcg %6.4f\n", i, ideal_dcg); } if (++cutoff_index == tm->meas_params->num_params) { break; } } gain = cur_lvl; ideal_dcg += gain / (double) log2((double) (i + 2)); if (epi->debug_level > 0) { printf("ndcg_cut:%ld %ld %3.1f %6.4f\n", i, cur_lvl, gain, ideal_dcg); } } /* calculate values for those cutoffs not achieved */ while (cutoff_index < tm->meas_params->num_params) { if (ideal_dcg > 0.0) { ndcg = eval->values[tm->eval_index + cutoff_index].value / ideal_dcg; eval->values[tm->eval_index + cutoff_index].value = (double) log( (double) (MAX(ndcg, MIN_GEO_MEAN))); } if (epi->debug_level > 0) { printf("ndcg_cut: cutoff %ld idcg %6.4f\n", cutoffs[cutoff_index], ideal_dcg); } cutoff_index++; } return (1); }
static int te_calc_ndcg_rel(const EPI *epi, const REL_INFO *rel_info, const RESULTS *results, const TREC_MEAS *tm, TREC_EVAL *eval) { RES_RELS res_rels; double results_gain, results_dcg; double ideal_gain, ideal_dcg; double sum = 0.0; long num_rel_ret = 0; long num_rel = 0; long cur_level, num_at_level; long i; GAINS gains; if (UNDEF == te_form_res_rels(epi, rel_info, results, &res_rels)) return (UNDEF); if (UNDEF == setup_gains(tm, &res_rels, &gains)) return (UNDEF); results_dcg = 0.0; ideal_dcg = 0.0; cur_level = gains.num_gains - 1; ideal_gain = (cur_level >= 0) ? gains.rel_gains[cur_level].gain : 0.0; num_at_level = 0; for (i = 0; i < res_rels.num_ret && ideal_gain > 0.0; i++) { /* Calculate change in results dcg */ results_gain = get_gain(res_rels.results_rel_list[i], &gains); if (results_gain != 0) /* Note: i+2 since doc i has rank i+1 */ results_dcg += results_gain / log2((double) (i + 2)); /* Calculate change in ideal dcg */ num_at_level++; while (cur_level >= 0 && num_at_level > gains.rel_gains[cur_level].num_at_level) { num_at_level = 1; cur_level--; ideal_gain = (cur_level >= 0) ? gains.rel_gains[cur_level].gain : 0.0; } if (ideal_gain > 0.0) { num_rel++; ideal_dcg += ideal_gain / log2((double) (i + 2)); } /* Average will include this point if rel */ if (results_gain > 0) { sum += results_dcg / ideal_dcg; num_rel_ret++; } if (epi->debug_level > 0) printf("ndcg_rel: %ld %ld %3.1f %6.4f %3.1f %6.4f %6.4f\n", i, cur_level, results_gain, results_dcg, ideal_gain, ideal_dcg, sum); } if (i < res_rels.num_ret) { while (i < res_rels.num_ret) { /* Calculate change in results dcg */ results_gain = get_gain(res_rels.results_rel_list[i], &gains); if (results_gain != 0) results_dcg += results_gain / log2((double) (i + 2)); /* Average will include this point if rel */ if (results_gain > 0) { sum += results_dcg / ideal_dcg; num_rel_ret++; } if (epi->debug_level > 0) printf("ndcg_rel: %ld %ld %3.1f %6.4f %3.1f %6.4f %6.4f\n", i, cur_level, results_gain, results_dcg, 0.0, ideal_dcg, sum); i++; } } while (ideal_gain > 0.0) { /* Calculate change in ideal dcg */ num_at_level++; while (cur_level >= 0 && num_at_level > gains.rel_gains[cur_level].num_at_level) { num_at_level = 1; cur_level--; ideal_gain = (cur_level >= 0) ? gains.rel_gains[cur_level].gain : 0.0; } if (ideal_gain > 0.0) { num_rel++; ideal_dcg += ideal_gain / log2((double) (i + 2)); } if (epi->debug_level > 0) printf("ndcg_rel: %ld %ld %3.1f %6.4f %3.1f %6.4f\n", i, cur_level, 0.0, results_dcg, ideal_gain, ideal_dcg); i++; } sum += ((double) (num_rel - num_rel_ret)) * results_dcg / ideal_dcg; if (epi->debug_level > 0) printf("ndcg_rel: %ld %ld %6.4f %6.4f %6.4f\n", i, cur_level, results_dcg, ideal_dcg, sum); if (sum > 0.0) eval->values[tm->eval_index].value = sum / num_rel; Free(gains.rel_gains); return (1); }
static int te_calc_gm_bpref (const EPI *epi, const REL_INFO *rel_info, const RESULTS *results, const TREC_MEAS *tm, TREC_EVAL *eval) { RES_RELS res_rels; long j; long nonrel_so_far, rel_so_far, pool_unjudged_so_far; long num_nonrel = 0; double bpref = 0.0; if (UNDEF == te_form_res_rels (epi, rel_info, results, &res_rels)) return (UNDEF); for (j = 0; j < epi->relevance_level; j++) num_nonrel += res_rels.rel_levels[j]; /* Calculate judgement based measures (dependent on only judged docs; no assumption of non-relevance if not judged) */ /* Binary Preference measures; here expressed as all docs with a higher value of rel are to be preferred. Optimize by keeping track of nonrel seen so far */ nonrel_so_far = 0; rel_so_far = 0; pool_unjudged_so_far = 0; for (j = 0; j < res_rels.num_ret; j++) { if (res_rels.results_rel_list[j] == RELVALUE_NONPOOL) /* document not in pool. Skip */ continue; if (res_rels.results_rel_list[j] == RELVALUE_UNJUDGED) { /* document in pool but unjudged. */ pool_unjudged_so_far++; continue; } if (res_rels.results_rel_list[j] >= 0 && res_rels.results_rel_list[j] < epi->relevance_level) nonrel_so_far++; else { /* Judged Rel doc */ rel_so_far++; /* Add fraction of correct preferences. */ /* Special case nonrel_so_far == 0 to avoid division by 0 */ if (nonrel_so_far > 0) { bpref += 1.0 - (((double) MIN (nonrel_so_far, res_rels.num_rel)) / (double) MIN (num_nonrel, res_rels.num_rel)); } else bpref += 1.0; } } if (res_rels.num_rel) bpref /= res_rels.num_rel; /* Original measure value is constrained to be greater than MIN_GEO_MEAN (for time being .00001, since trec_eval prints to four significant digits) */ eval->values[tm->eval_index].value = (double) log ((double)(MAX (bpref, MIN_GEO_MEAN))); if (epi->debug_level > 1) printf ("gm_bpref: bpref %6.4f, gm_bpref %6.4f", bpref, eval->values[tm->eval_index].value); return (1); }