size_t sentence_scores(sentence_type *s, const Float w[], Float score[],
Float *max_correct_score, Float *max_score) {
int i, best_i = 0, best_correct_i = -1;
Float sc;
assert(s->nparses > 0);
*max_score = score[0] = sc = parse_score(&s->parse[0], w);
if (s->parse[0].Pyx > 0) {
best_correct_i = 0;
*max_correct_score = sc;
}
for (i = 1; i < s->nparses; ++i) {
score[i] = sc = parse_score(&s->parse[i], w);
if (sc >= *max_score) {
best_i = i;
*max_score = sc;
}
if (s->parse[i].Pyx > 0 && (best_correct_i < 0 || sc > *max_correct_score)) {
best_correct_i = i;
*max_correct_score = sc;
}
}
assert(finite(*max_score));
assert(s->Px == 0 || best_correct_i >= 0);
/* assert(best_correct_i >= 0); assert(finite(*max_correct_score)); */
return best_i;
} /* sentence_scores() */
void ap_sentence_scores(sentence_type *s, const Float w[],
Float *best_correct_score, int *best_correct_i,
Float *best_score, int *best_i) {
int i;
Float sc;
assert(s->nparses > 0);
*best_i = 0;
*best_correct_i = -1;
*best_score = sc = parse_score(&s->parse[0], w);
if (s->parse[0].Pyx > 0) {
*best_correct_i = 0;
*best_correct_score = sc;
}
for (i = 1; i < s->nparses; ++i) {
sc = parse_score(&s->parse[i], w);
if (sc >= *best_score) {
*best_i = i;
*best_score = sc;
}
if (s->parse[i].Pyx > 0 && (*best_correct_i < 0 || sc >= *best_correct_score)) {
*best_correct_i = i;
*best_correct_score = sc;
}
}
assert(finite(*best_score));
assert(s->Px == 0 || (*best_correct_i >= 0 && *best_correct_i < s->nparses));
/* assert(best_correct_i >= 0); assert(finite(*best_correct_score)); */
} /* ap_sentence_scores() */
int Ardb::ZCount(const DBID& db, const Slice& key, const std::string& min,
const std::string& max)
{
bool containmin = true;
bool containmax = true;
double min_score, max_score;
if (parse_score(min, min_score, containmin) < 0
|| parse_score(max, max_score, containmax) < 0)
{
return ERR_INVALID_ARGS;
}
Slice empty;
ZSetKeyObject start(key, empty, min_score, db);
struct ZCountWalk: public WalkHandler
{
double z_min_score;
bool z_containmin;
double z_max_score;
bool z_containmax;
int count;
int OnKeyValue(KeyObject* k, ValueObject* v, uint32 cursor)
{
ZSetKeyObject* zko = (ZSetKeyObject*) k;
std::string str;
if (zko->score > z_min_score && zko->score < z_max_score)
{
count++;
}
if (z_containmin && zko->score == z_min_score)
{
count++;
}
if (z_containmax && zko->score == z_max_score)
{
count++;
}
if (zko->score > z_max_score)
{
return -1;
}
return 0;
}
} walk;
walk.z_min_score = min_score;
walk.z_max_score = max_score;
walk.z_containmin = containmin;
walk.z_containmax = containmax;
walk.count = 0;
Walk(start, false, &walk);
return walk.count;
}
size_type max_score_index(const sentence_type *s, const Float w[]) {
size_type i, max_i = 0;
Float max_score = parse_score(&s->parse[0], w);
assert(finite(max_score));
if (s->nparses <= 1)
return 0;
for (i = 1; i < s->nparses; ++i) {
Float score = parse_score(&s->parse[i], w);
assert(finite(score));
if (score > max_score) {
max_i = i;
max_score = score;
}
}
return max_i;
} /* max_score_index */
static int blame_move_callback(const struct option *option, const char *arg, int unset)
{
int *opt = option->value;
*opt |= PICKAXE_BLAME_MOVE;
if (arg)
blame_move_score = parse_score(arg);
return 0;
}
__inline__
void sentence_scores(sentence_type *s, const Float w[], Float score[],
Float *best_correct_score, int *best_correct_i,
Float *best_score, int *best_i) {
size_type i;
Float sc;
assert(s->nparses > 0);
*best_i = 0;
*best_correct_i = -1;
*best_correct_score = 0;
*best_score = score[0] = sc = parse_score(&s->parse[0], w);
if (s->parse[0].Pyx > 0) {
*best_correct_i = 0;
*best_correct_score = sc;
}
for (i = 1; i < s->nparses; ++i) {
score[i] = sc = parse_score(&s->parse[i], w);
if (sc >= *best_score) {
*best_i = i;
*best_score = sc;
}
if (s->parse[i].Pyx > 0
&& (*best_correct_i < 0 || sc > *best_correct_score)) {
*best_correct_i = i;
*best_correct_score = sc;
}
}
assert(finite(*best_score));
assert(s->Px == 0 || *best_correct_i >= 0);
if (*best_correct_score > *best_score) {
fprintf(stderr, "## best_correct_score = %g, best_score = %g\n## score =",
*best_correct_score, *best_score);
for (i = 0; i < s->nparses; ++i)
fprintf(stderr, " %g", score[i]);
fprintf(stderr, "\n");
}
assert(s->Px == 0 || *best_correct_score <= *best_score);
} /* sentence_scores() */
__inline__
static Float lnn_parse_score(parse_type *p, const lnn_weights_type* wt,
size_type nhidden, size_type nfeatures,
Float score0[]) {
Float score1 = 0;
int j;
for (j = 0; j < nhidden; ++j) {
Float input = parse_score(p, &wt->w0[j*nfeatures]) + wt->b0[j];
score0[j] = tanh(input); /* tanh sigmoid activation function */
score1 += wt->w1[j] * score0[j];
}
return score1;
} /* lnn_parse_score() */
__inline__
int sentence_Pyx(sentence_type *s, const Float w[], Float Py_x[]) {
int i, n = s->nparses, best_i = 0;
Float Z = 0;
const parse_type *parse = s->parse;
Float best_score = Py_x[0] = parse_score(&parse[0], w);
assert(s->nparses > 0);
/* load Py_x[i] with parse_score(), find best_score and best_i */
for (i = 1; i < n; ++i) {
Float sc = Py_x[i] = parse_score(&parse[i], w);
assert(finite(sc));
if (sc >= best_score) {
best_i = i;
best_score = sc;
}
}
assert(finite(best_score));
/* compute Z */
for (i = 0; i < n; ++i)
Z += exp(Py_x[i] - best_score);
assert(finite(Z));
/* compute Py_x[] */
for (i = 0; i < n; ++i) {
Float P = Py_x[i] = exp(Py_x[i] - best_score) / Z;
assert(finite(P));
}
return best_i;
} /* sentence_Pyx() */
Float margins(corpus_type *c, const Float w[], Float m[],
Float *sum_g, Float *sum_p, Float *sum_w)
{
Float min_margin = FLOAT_MAX;
size_type i, j, im = 0;
*sum_g = *sum_p = *sum_w = 0; /* zero precision/recall counters */
for (i = 0; i < c->nsentences; ++i) {
sentence_type *s = &c->sentence[i];
*sum_g += s->g;
if (s->Px > 0) {
Float correct_score = parse_score(&s->parse[s->correct_index], w);
Float best_score = correct_score;
size_type best_index = s->correct_index;
for (j = 0; j < s->nparses; ++j)
if (j != s->correct_index) {
Float score = parse_score(&s->parse[j], w);
Float margin = correct_score - score;
if (score >= best_score) {
best_index = j;
best_score = score;
}
if (margin < min_margin)
min_margin = margin;
assert(im < c->nloserparses);
m[im++] = margin;
}
*sum_p += s->parse[best_index].p;
*sum_w += s->parse[best_index].w;
}
}
assert(im == c->nloserparses);
return min_margin;
} // margins()
static int blame_copy_callback(const struct option *option, const char *arg, int unset)
{
int *opt = option->value;
/*
* -C enables copy from removed files;
* -C -C enables copy from existing files, but only
* when blaming a new file;
* -C -C -C enables copy from existing files for
* everybody
*/
if (*opt & PICKAXE_BLAME_COPY_HARDER)
*opt |= PICKAXE_BLAME_COPY_HARDEST;
if (*opt & PICKAXE_BLAME_COPY)
*opt |= PICKAXE_BLAME_COPY_HARDER;
*opt |= PICKAXE_BLAME_COPY | PICKAXE_BLAME_MOVE;
if (arg)
blame_copy_score = parse_score(arg);
return 0;
}
Float exp_corpus_stats(corpus_type *c, const Float w[], Float dL_dw[],
Float *sum_g, Float *sum_p, Float *sum_w)
{
Float L = 0;
int i, j, k, im = 0;
const Float margin_cutoff = -log(FLOAT_MAX/2)/2;
*sum_g = *sum_p = *sum_w = 0; /* zero precision/recall counters */
for (k = 0; k < c->nfeatures; ++k) /* zero dL_dw[] */
dL_dw[k] = 0;
for (i = 0; i < c->nsentences; ++i) {
sentence_type *s = &c->sentence[i];
*sum_g += s->g;
if (s->Px > 0) {
Float correct_score = parse_score(&s->parse[s->correct_index], w);
Float best_score = correct_score;
size_type best_index = s->correct_index;
Float sum_exp_nmargin = 0;
assert(s->correct_index < s->nparses);
for (j = 0; j < s->nparses; ++j)
if (j != s->correct_index) {
Float score = parse_score(&s->parse[j], w);
Float margin = correct_score - score;
Float exp_nmargin;
if (score >= best_score) { /* save best score */
best_index = j;
best_score = score;
}
++im; /* count number of incorrect parses */
if (margin >= margin_cutoff) {
exp_nmargin = exp(-margin);
assert(finite(exp_nmargin));
L += exp_nmargin;
assert(finite(L));
}
else {
exp_nmargin = exp(-margin_cutoff);
assert(finite(exp_nmargin));
L += (margin_cutoff+1-margin) * exp_nmargin;
assert(finite(L));
}
sum_exp_nmargin += exp_nmargin;
assert(finite(sum_exp_nmargin));
for (k = 0; k < s->parse[j].nf; ++k) /* 1 count features */
dL_dw[s->parse[j].f[k]] += exp_nmargin;
for (k = 0; k < s->parse[j].nfc; ++k) /* arbitrary count features */
dL_dw[s->parse[j].fc[k].f] += exp_nmargin * s->parse[j].fc[k].c;
}
for (k = 0; k < s->parse[s->correct_index].nf; ++k)
dL_dw[s->parse[s->correct_index].f[k]] -= sum_exp_nmargin;
for (k = 0; k < s->parse[s->correct_index].nfc; ++k)
dL_dw[s->parse[s->correct_index].fc[k].f]
-= sum_exp_nmargin * s->parse[s->correct_index].fc[k].c;
*sum_p += s->parse[best_index].p;
*sum_w += s->parse[best_index].w;
}
}
assert(im == c->nloserparses);
return L;
} /* exp_corpus_stats() */
int Ardb::ZRevRangeByScore(const DBID& db, const Slice& key,
const std::string& max, const std::string& min, ValueArray& values,
QueryOptions& options)
{
ZSetMetaValue meta;
if (0 != GetZSetMetaValue(db, key, meta))
{
return ERR_NOT_EXIST;
}
bool containmin = true;
bool containmax = true;
double min_score, max_score;
if (parse_score(min, min_score, containmin) < 0
|| parse_score(max, max_score, containmax) < 0)
{
return ERR_INVALID_ARGS;
}
Slice empty;
ZSetKeyObject tmp(key, empty, max_score, db);
struct ZRangeByScoreWalk: public WalkHandler
{
ValueArray& z_values;
QueryOptions& z_options;
double z_min_score;
bool z_containmin;
bool z_containmax;
double z_max_score;
int z_count;
int OnKeyValue(KeyObject* k, ValueObject* v, uint32 cursor)
{
ZSetKeyObject* zsk = (ZSetKeyObject*) k;
bool inrange = false;
inrange =
z_containmin ?
zsk->score >= z_min_score :
zsk->score > z_min_score;
if (inrange)
{
inrange =
z_containmax ?
zsk->score <= z_max_score :
zsk->score < z_max_score;
}
if (inrange)
{
if (z_options.withlimit)
{
if (z_count >= z_options.limit_offset
&& z_count
<= (z_options.limit_count
+ z_options.limit_offset))
{
inrange = true;
} else
{
inrange = false;
}
}
z_count++;
if (inrange)
{
z_values.push_back(zsk->value);
if (z_options.withscores)
{
z_values.push_back(ValueObject(zsk->score));
}
}
}
if (zsk->score == z_min_score
|| (z_options.withlimit
&& z_count
> (z_options.limit_count
+ z_options.limit_offset)))
{
return -1;
}
return 0;
}
ZRangeByScoreWalk(ValueArray& v, QueryOptions& options) :
z_values(v), z_options(options), z_count(0)
{
}
} walk(values, options);
walk.z_containmax = containmax;
walk.z_containmin = containmin;
walk.z_max_score = max_score;
walk.z_min_score = min_score;
Walk(tmp, true, &walk);
return walk.z_count;
}
int Ardb::ZRemRangeByScore(const DBID& db, const Slice& key,
const std::string& min, const std::string& max)
{
KeyLockerGuard keyguard(m_key_locker, db, key);
ZSetMetaValue meta;
if (0 != GetZSetMetaValue(db, key, meta))
{
return ERR_NOT_EXIST;
}
bool containmin = true;
bool containmax = true;
double min_score, max_score;
if (parse_score(min, min_score, containmin) < 0
|| parse_score(max, max_score, containmax) < 0)
{
return ERR_INVALID_ARGS;
}
Slice empty;
ZSetKeyObject tmp(key, empty, min_score, db);
BatchWriteGuard guard(GetEngine());
struct ZRemRangeByScoreWalk: public WalkHandler
{
Ardb* z_db;
double z_min_score;
bool z_containmin;
bool z_containmax;
double z_max_score;
ZSetMetaValue& z_meta;
int z_count;
int OnKeyValue(KeyObject* k, ValueObject* v, uint32 cursor)
{
ZSetKeyObject* zsk = (ZSetKeyObject*) k;
bool need_delete = false;
need_delete =
z_containmin ?
zsk->score >= z_min_score :
zsk->score > z_min_score;
if (need_delete)
{
need_delete =
z_containmax ?
zsk->score <= z_max_score :
zsk->score < z_max_score;
}
if (need_delete)
{
ZSetScoreKeyObject zk(zsk->key, zsk->value, zsk->db);
z_db->DelValue(zk);
z_db->DelValue(*zsk);
z_meta.size--;
z_count++;
}
if (zsk->score == z_max_score)
{
return -1;
}
return 0;
}
ZRemRangeByScoreWalk(Ardb* db, ZSetMetaValue& meta) :
z_db(db), z_meta(meta), z_count(0)
{
}
} walk(this, meta);
walk.z_containmax = containmax;
walk.z_containmin = containmin;
walk.z_max_score = max_score;
walk.z_min_score = min_score;
Walk(tmp, false, &walk);
SetZSetMetaValue(db, key, meta);
return walk.z_count;
}
static int parse_info(uci_t * uci, const char string[]) {
int event;
parse_t parse[1];
char command[StringSize];
char option[StringSize];
char argument[StringSize];
int n;
int multipvline=0;
sint64 ln;
ASSERT(uci_is_ok(uci));
ASSERT(string!=NULL);
// init
event = EVENT_NONE;
strcpy(command,"info");
parse_open(parse,string);
parse_add_keyword(parse,"cpuload");
parse_add_keyword(parse,"currline");
parse_add_keyword(parse,"currmove");
parse_add_keyword(parse,"currmovenumber");
parse_add_keyword(parse,"depth");
parse_add_keyword(parse,"hashfull");
parse_add_keyword(parse,"multipv");
parse_add_keyword(parse,"nodes");
parse_add_keyword(parse,"nps");
parse_add_keyword(parse,"pv");
parse_add_keyword(parse,"refutation");
parse_add_keyword(parse,"score");
parse_add_keyword(parse,"seldepth");
parse_add_keyword(parse,"string");
parse_add_keyword(parse,"tbhits");
parse_add_keyword(parse,"time");
// loop
while (parse_get_word(parse,option,StringSize)) {
parse_get_string(parse,argument,StringSize);
if (UseDebug) my_log("POLYGLOT COMMAND \"%s\" OPTION \"%s\" ARGUMENT \"%s\"\n",command,option,argument);
if (false) {
} else if (my_string_equal(option,"cpuload")) {
ASSERT(!my_string_empty(argument));
n = atoi(argument);
ASSERT(n>=0);
if (n >= 0) uci->cpu = double(n) / 1000.0;
} else if (my_string_equal(option,"currline")) {
ASSERT(!my_string_empty(argument));
line_from_can(uci->current_line,uci->board,argument,LineSize);
} else if (my_string_equal(option,"currmove")) {
ASSERT(!my_string_empty(argument));
uci->root_move = move_from_can(argument,uci->board);
ASSERT(uci->root_move!=MoveNone);
} else if (my_string_equal(option,"currmovenumber")) {
ASSERT(!my_string_empty(argument));
n = atoi(argument);
ASSERT(n>=1&&n<=uci->root_move_nb);
if (n >= 1 && n <= uci->root_move_nb) {
uci->root_move_pos = n - 1;
ASSERT(uci->root_move_pos>=0&&uci->root_move_pos<uci->root_move_nb);
}
} else if (my_string_equal(option,"depth")) {
ASSERT(!my_string_empty(argument));
n = atoi(argument);
ASSERT(n>=1);
if (n >= 0) {
if (n > uci->depth) event |= EVENT_DEPTH;
uci->depth = n;
}
} else if (my_string_equal(option,"hashfull")) {
ASSERT(!my_string_empty(argument));
n = atoi(argument);
ASSERT(n>=0);
if (n >= 0) uci->hash = double(n) / 1000.0;
} else if (my_string_equal(option,"multipv")) {
ASSERT(!my_string_empty(argument));
n = atoi(argument);
if(Uci->multipv_mode) multipvline=n;
ASSERT(n>=1);
} else if (my_string_equal(option,"nodes")) {
ASSERT(!my_string_empty(argument));
ln = my_atoll(argument);
ASSERT(ln>=0);
if (ln >= 0) uci->node_nb = ln;
} else if (my_string_equal(option,"nps")) {
ASSERT(!my_string_empty(argument));
n = atoi(argument);
ASSERT(n>=0);
if (n >= 0) uci->speed = double(n);
} else if (my_string_equal(option,"pv")) {
ASSERT(!my_string_empty(argument));
line_from_can(uci->pv,uci->board,argument,LineSize);
event |= EVENT_PV;
} else if (my_string_equal(option,"refutation")) {
ASSERT(!my_string_empty(argument));
line_from_can(uci->pv,uci->board,argument,LineSize);
} else if (my_string_equal(option,"score")) {
ASSERT(!my_string_empty(argument));
parse_score(uci,argument);
} else if (my_string_equal(option,"seldepth")) {
ASSERT(!my_string_empty(argument));
n = atoi(argument);
ASSERT(n>=0);
if (n >= 0) uci->sel_depth = n;
} else if (my_string_equal(option,"string")) {
if(!strncmp(argument,"DrawOffer",9))
event |= EVENT_DRAW;
if(!strncmp(argument,"Resign",6))
event |= EVENT_RESIGN;
// TODO: argument to EOS
ASSERT(!my_string_empty(argument));
} else if (my_string_equal(option,"tbhits")) {
ASSERT(!my_string_empty(argument));
ln = my_atoll(argument);
ASSERT(ln>=0);
} else if (my_string_equal(option,"time")) {
ASSERT(!my_string_empty(argument));
n = atoi(argument);
ASSERT(n>=0);
if (n >= 0) uci->time = double(n) / 1000.0;
} else {
my_log("POLYGLOT unknown option \"%s\" for command \"%s\"\n",option,command);
}
}
parse_close(parse);
// update display
//lousy uci,filter out lower depth multipv lines that have been repeated from the engine
if(multipvline>1 && uci->depth<uci->best_depth) event &= ~EVENT_PV;
if ((event & EVENT_PV) != 0) {
uci->best_score = uci->score;
uci->best_depth = uci->depth;
if(multipvline==1)uci->depth=-1; //HACK ,clears the engine outpout window,see send_pv in adapter.cpp
uci->best_sel_depth = uci->sel_depth;
line_copy(uci->best_pv,uci->pv);
}
return event;
}