static int te_calc_num_ret(const EPI *epi, const REL_INFO *rel_info,
const RESULTS *results, const TREC_MEAS *tm, TREC_EVAL *eval) {
RES_RELS res_rels;
/* Can't just use results, since epi->only_judged_docs may be set */
if (UNDEF == te_form_res_rels(epi, rel_info, results, &res_rels))
return (UNDEF);
eval->values[tm->eval_index].value = (double) res_rels.num_ret;
return (1);
}
static int
te_calc_set_map (const EPI *epi, const REL_INFO *rel_info,
const RESULTS *results, const TREC_MEAS *tm, TREC_EVAL *eval)
{
RES_RELS res_rels;
if (UNDEF == te_form_res_rels (epi, rel_info, results, &res_rels))
return (UNDEF);
if (res_rels.num_ret && res_rels.num_rel)
eval->values[tm->eval_index].value =
((double) res_rels.num_rel_ret *
(double) res_rels.num_rel_ret) /
((double) res_rels.num_ret *
(double) res_rels.num_rel);
return (1);
}
static int te_calc_gm_ndcg_cut(const EPI *epi, const REL_INFO *rel_info,
const RESULTS *results, const TREC_MEAS *tm, TREC_EVAL *eval) {
long *cutoffs = (long *) tm->meas_params->param_values;
long cutoff_index = 0;
RES_RELS res_rels;
double gain, sum;
double ideal_dcg; /* ideal discounted cumulative gain */
double ndcg;
long cur_lvl, lvl_count;
long i;
if (UNDEF == te_form_res_rels(epi, rel_info, results, &res_rels)) {
return (UNDEF);
}
sum = 0.0;
for (i = 0; i < res_rels.num_ret; i++) {
if (i == cutoffs[cutoff_index]) {
/* Calculate previous cutoff threshold.
Note i guaranteed to be positive by init_meas */
eval->values[tm->eval_index + cutoff_index].value = sum;
if (++cutoff_index == tm->meas_params->num_params) {
break;
}
if (epi->debug_level > 0) {
printf("ndcg_cut: cutoff %ld dcg %6.4f\n", i, sum);
}
}
gain = res_rels.results_rel_list[i];
if (gain > 0) {
/* Note: i+2 since doc i has rank i+1 */
sum += gain / log2((double) (i + 2));
if (epi->debug_level > 1) {
printf("ndcg_cut:%ld %3.1f %6.4f\n", i, gain, sum);
}
}
}
/* calculate values for those cutoffs not achieved */
while (cutoff_index < tm->meas_params->num_params) {
eval->values[tm->eval_index + cutoff_index].value = sum;
if (epi->debug_level > 0) {
printf("ndcg_cut: cutoff %ld dcg %6.4f\n", cutoffs[cutoff_index],
sum);
}
cutoff_index++;
}
/* Calculate ideal discounted cumulative gain for this topic, and
normalize previous sum by it */
cutoff_index = 0;
cur_lvl = res_rels.num_rel_levels - 1;
lvl_count = 0;
ideal_dcg = 0.0;
for (i = 0; 1; i++) {
lvl_count++;
while (cur_lvl > 0 && lvl_count > res_rels.rel_levels[cur_lvl]) {
cur_lvl--;
lvl_count = 1;
}
if (cur_lvl == 0) {
break;
}
if (i == cutoffs[cutoff_index]) {
/* Calculate previous cutoff threshold.
Note i guaranteed to be positive by init_meas */
if (ideal_dcg > 0.0) {
ndcg = eval->values[tm->eval_index + cutoff_index].value
/ ideal_dcg;
eval->values[tm->eval_index + cutoff_index].value =
(double) log((double) (MAX(ndcg, MIN_GEO_MEAN)));
}
if (epi->debug_level > 0) {
printf("ndcg_cut: cutoff %ld idcg %6.4f\n", i, ideal_dcg);
}
if (++cutoff_index == tm->meas_params->num_params) {
break;
}
}
gain = cur_lvl;
ideal_dcg += gain / (double) log2((double) (i + 2));
if (epi->debug_level > 0) {
printf("ndcg_cut:%ld %ld %3.1f %6.4f\n", i, cur_lvl, gain,
ideal_dcg);
}
}
/* calculate values for those cutoffs not achieved */
while (cutoff_index < tm->meas_params->num_params) {
if (ideal_dcg > 0.0) {
ndcg = eval->values[tm->eval_index + cutoff_index].value
/ ideal_dcg;
eval->values[tm->eval_index + cutoff_index].value = (double) log(
(double) (MAX(ndcg, MIN_GEO_MEAN)));
}
if (epi->debug_level > 0) {
printf("ndcg_cut: cutoff %ld idcg %6.4f\n", cutoffs[cutoff_index],
ideal_dcg);
}
cutoff_index++;
}
return (1);
}
static int te_calc_ndcg_rel(const EPI *epi, const REL_INFO *rel_info,
const RESULTS *results, const TREC_MEAS *tm, TREC_EVAL *eval) {
RES_RELS res_rels;
double results_gain, results_dcg;
double ideal_gain, ideal_dcg;
double sum = 0.0;
long num_rel_ret = 0;
long num_rel = 0;
long cur_level, num_at_level;
long i;
GAINS gains;
if (UNDEF == te_form_res_rels(epi, rel_info, results, &res_rels))
return (UNDEF);
if (UNDEF == setup_gains(tm, &res_rels, &gains))
return (UNDEF);
results_dcg = 0.0;
ideal_dcg = 0.0;
cur_level = gains.num_gains - 1;
ideal_gain = (cur_level >= 0) ? gains.rel_gains[cur_level].gain : 0.0;
num_at_level = 0;
for (i = 0; i < res_rels.num_ret && ideal_gain > 0.0; i++) {
/* Calculate change in results dcg */
results_gain = get_gain(res_rels.results_rel_list[i], &gains);
if (results_gain != 0)
/* Note: i+2 since doc i has rank i+1 */
results_dcg += results_gain / log2((double) (i + 2));
/* Calculate change in ideal dcg */
num_at_level++;
while (cur_level >= 0
&& num_at_level > gains.rel_gains[cur_level].num_at_level) {
num_at_level = 1;
cur_level--;
ideal_gain =
(cur_level >= 0) ? gains.rel_gains[cur_level].gain : 0.0;
}
if (ideal_gain > 0.0) {
num_rel++;
ideal_dcg += ideal_gain / log2((double) (i + 2));
}
/* Average will include this point if rel */
if (results_gain > 0) {
sum += results_dcg / ideal_dcg;
num_rel_ret++;
}
if (epi->debug_level > 0)
printf("ndcg_rel: %ld %ld %3.1f %6.4f %3.1f %6.4f %6.4f\n", i,
cur_level, results_gain, results_dcg, ideal_gain, ideal_dcg,
sum);
}
if (i < res_rels.num_ret) {
while (i < res_rels.num_ret) {
/* Calculate change in results dcg */
results_gain = get_gain(res_rels.results_rel_list[i], &gains);
if (results_gain != 0)
results_dcg += results_gain / log2((double) (i + 2));
/* Average will include this point if rel */
if (results_gain > 0) {
sum += results_dcg / ideal_dcg;
num_rel_ret++;
}
if (epi->debug_level > 0)
printf("ndcg_rel: %ld %ld %3.1f %6.4f %3.1f %6.4f %6.4f\n", i,
cur_level, results_gain, results_dcg, 0.0, ideal_dcg,
sum);
i++;
}
}
while (ideal_gain > 0.0) {
/* Calculate change in ideal dcg */
num_at_level++;
while (cur_level >= 0
&& num_at_level > gains.rel_gains[cur_level].num_at_level) {
num_at_level = 1;
cur_level--;
ideal_gain =
(cur_level >= 0) ? gains.rel_gains[cur_level].gain : 0.0;
}
if (ideal_gain > 0.0) {
num_rel++;
ideal_dcg += ideal_gain / log2((double) (i + 2));
}
if (epi->debug_level > 0)
printf("ndcg_rel: %ld %ld %3.1f %6.4f %3.1f %6.4f\n", i, cur_level,
0.0, results_dcg, ideal_gain, ideal_dcg);
i++;
}
sum += ((double) (num_rel - num_rel_ret)) * results_dcg / ideal_dcg;
if (epi->debug_level > 0)
printf("ndcg_rel: %ld %ld %6.4f %6.4f %6.4f\n", i, cur_level,
results_dcg, ideal_dcg, sum);
if (sum > 0.0)
eval->values[tm->eval_index].value = sum / num_rel;
Free(gains.rel_gains);
return (1);
}
static int
te_calc_gm_bpref (const EPI *epi, const REL_INFO *rel_info,
const RESULTS *results, const TREC_MEAS *tm, TREC_EVAL *eval)
{
RES_RELS res_rels;
long j;
long nonrel_so_far, rel_so_far, pool_unjudged_so_far;
long num_nonrel = 0;
double bpref = 0.0;
if (UNDEF == te_form_res_rels (epi, rel_info, results, &res_rels))
return (UNDEF);
for (j = 0; j < epi->relevance_level; j++)
num_nonrel += res_rels.rel_levels[j];
/* Calculate judgement based measures (dependent on only
judged docs; no assumption of non-relevance if not judged) */
/* Binary Preference measures; here expressed as all docs with a higher
value of rel are to be preferred. Optimize by keeping track of nonrel
seen so far */
nonrel_so_far = 0;
rel_so_far = 0;
pool_unjudged_so_far = 0;
for (j = 0; j < res_rels.num_ret; j++) {
if (res_rels.results_rel_list[j] == RELVALUE_NONPOOL)
/* document not in pool. Skip */
continue;
if (res_rels.results_rel_list[j] == RELVALUE_UNJUDGED) {
/* document in pool but unjudged. */
pool_unjudged_so_far++;
continue;
}
if (res_rels.results_rel_list[j] >= 0 &&
res_rels.results_rel_list[j] < epi->relevance_level)
nonrel_so_far++;
else {
/* Judged Rel doc */
rel_so_far++;
/* Add fraction of correct preferences. */
/* Special case nonrel_so_far == 0 to avoid division by 0 */
if (nonrel_so_far > 0) {
bpref += 1.0 -
(((double) MIN (nonrel_so_far, res_rels.num_rel)) /
(double) MIN (num_nonrel, res_rels.num_rel));
}
else
bpref += 1.0;
}
}
if (res_rels.num_rel)
bpref /= res_rels.num_rel;
/* Original measure value is constrained to be greater than
MIN_GEO_MEAN (for time being .00001, since trec_eval prints to
four significant digits) */
eval->values[tm->eval_index].value =
(double) log ((double)(MAX (bpref, MIN_GEO_MEAN)));
if (epi->debug_level > 1)
printf ("gm_bpref: bpref %6.4f, gm_bpref %6.4f",
bpref, eval->values[tm->eval_index].value);
return (1);
}
