static void
pgf_read_abstract(PgfReader* rdr, PgfAbstr* abstract)
{
abstract->name = pgf_read_cid(rdr, rdr->opool);
gu_return_on_exn(rdr->err, );
abstract->aflags = pgf_read_flags(rdr);
gu_return_on_exn(rdr->err, );
abstract->eval_gates = pgf_jit_gates(rdr);
abstract->funs = pgf_read_absfuns(rdr, abstract);
gu_return_on_exn(rdr->err, );
abstract->cats = pgf_read_abscats(rdr, abstract);
gu_return_on_exn(rdr->err, );
abstract->abs_lin_fun = gu_new(PgfAbsFun, rdr->opool);
abstract->abs_lin_fun->name = "_";
abstract->abs_lin_fun->type = gu_new(PgfType, rdr->opool);
abstract->abs_lin_fun->type->hypos = NULL;
abstract->abs_lin_fun->type->cid = "_";
abstract->abs_lin_fun->type->n_exprs = 0;
abstract->abs_lin_fun->arity = 0;
abstract->abs_lin_fun->defns = NULL;
abstract->abs_lin_fun->ep.prob = INFINITY;
abstract->abs_lin_fun->ep.expr = gu_null_variant;
}
PgfClosure*
pgf_evaluate_lambda_application(PgfReasoner* rs, PgfExprThunk* lambda,
PgfClosure* arg)
{
PgfEnv* new_env = gu_new(PgfEnv, rs->pool);
new_env->next = lambda->env;
new_env->closure = arg;
PgfExprThunk* thunk = gu_new(PgfExprThunk, rs->pool);
thunk->header.code = rs->eval_gates->evaluate_expr_thunk;
thunk->env = new_env;
thunk->expr = lambda->expr;
return pgf_evaluate_expr_thunk(rs, thunk);
}
static GuEnum*
jpgf_literal_callback_predict(PgfLiteralCallback* self, PgfConcr* concr,
size_t lin_idx,
GuString prefix,
GuPool *out_pool)
{
JPgfLiteralCallback* callback = gu_container(self, JPgfLiteralCallback, callback);
JNIEnv *env;
(*cachedJVM)->AttachCurrentThread(cachedJVM, &env, NULL);
jstring jprefix = gu2j_string(env, prefix);
jobject jiterator = (*env)->CallObjectMethod(env, callback->jcallback, callback->predict_methodId, lin_idx, jprefix);
if (jiterator == NULL)
return NULL;
JPgfTokenProbEnum* en = gu_new(JPgfTokenProbEnum, out_pool);
en->en.next = NULL;
en->jiterator = (*env)->NewGlobalRef(env, jiterator);
en->fin.fn = jpgf_token_prob_enum_fin;
gu_pool_finally(out_pool, &en->fin);
return &en->en;
}
static GuInStream*
jpgf_new_java_stream(JNIEnv* env, jobject java_stream, GuPool* pool)
{
jclass java_stream_class = (*env)->GetObjectClass(env, java_stream);
JInStream* jstream = gu_new(JInStream, pool);
jstream->stream.begin_buffer = jpgf_jstream_begin_buffer;
jstream->stream.end_buffer = jpgf_jstream_end_buffer;
jstream->stream.input = NULL;
jstream->java_stream = java_stream;
jstream->env = env;
jstream->read_method = (*env)->GetMethodID(env, java_stream_class, "read", "([B)I");
if (!jstream->read_method) {
return NULL;
}
jstream->buf_array = (*env)->NewByteArray(env, 1024);
if (!jstream->buf_array) {
return NULL;
}
jstream->buf = NULL;
return &jstream->stream;
}
GuChoice*
gu_new_choice(GuPool* pool)
{
GuChoice* ch = gu_new(GuChoice, pool);
ch->path = gu_new_buf(size_t, pool);
ch->path_idx = 0;
return ch;
}
GuOut*
gu_new_out(GuOutStream* stream, GuPool* pool)
{
GuOut* out = gu_new(GuOut, pool);
*out = gu_init_out(stream);
out->fini.fn = gu_out_fini;
gu_pool_finally(pool, &out->fini);
return out;
}
GuEnum*
gu_map_enum(GuMap* ht, GuPool* pool)
{
GuMapEnum* en = gu_new(GuMapEnum, pool);
en->en.next = gu_map_enum_next;
en->ht = ht;
en->i = 0;
return &en->en;
}
void
gu_raise_errno(GuExn* err)
{
GuExnData* err_data = gu_raise(err, GuErrno);
if (err_data) {
GuErrno* gu_errno = gu_new(GuErrno, err_data->pool);
*gu_errno = errno;
err_data->data = gu_errno;
}
}
PgfLexer*
pgf_new_lexer(GuReader *rdr, GuPool *pool)
{
PgfLexer* lexer = gu_new(PgfLexer, pool);
lexer->rdr = rdr;
lexer->pool = pool;
lexer->ucs = ' ';
lexer->tok = gu_empty_string;
return lexer;
}
GuExn*
gu_new_exn(GuPool* pool)
{
GuExn* exn = gu_new(GuExn, pool);
exn->state = GU_EXN_OK;
exn->caught = NULL;
exn->data.pool = pool;
exn->data.data = NULL;
return exn;
}
GuOut*
gu_new_out(GuOutStream* stream, GuPool* pool)
{
gu_require(stream != NULL);
GuOut* out = gu_new(GuOut, pool);
out->buf_end = NULL,
out->buf_curr = 0,
out->stream = stream,
out->fini.fn = gu_out_fini;
gu_pool_finally(pool, &out->fini);
return out;
}
static void
pgf_metrics_lzn_end_phrase1(PgfLinFuncs** funcs, PgfCId cat, int fid, int lin_idx, PgfCId fun)
{
PgfMetricsLznState* state = gu_container(funcs, PgfMetricsLznState, funcs);
int start = gu_buf_pop(state->marks, int);
int end = state->pos;
if (start != end) {
PgfPhrase* phrase = gu_new(PgfPhrase, state->pool);
phrase->start = start;
phrase->end = end;
phrase->cat = cat;
phrase->lin_idx = lin_idx;
gu_buf_push(state->phrases, PgfPhrase*, phrase);
}
GuMap*
gu_make_map(size_t key_size, GuHasher* hasher,
size_t value_size, const void* default_value,
GuPool* pool)
{
GuMapKind kind =
((!hasher || hasher == gu_addr_hasher)
? GU_MAP_ADDR
: (hasher == gu_string_hasher)
? GU_MAP_STRING
: (key_size == sizeof(GuWord) && hasher == gu_word_hasher)
? GU_MAP_WORD
: GU_MAP_GENERIC);
if (kind == GU_MAP_ADDR || kind == GU_MAP_STRING) {
key_size = sizeof(GuWord);
}
GuMapData data = {
.n_occupied = 0,
.n_entries = 0,
.keys = NULL,
.values = value_size ? NULL : (uint8_t*) gu_map_no_values,
.zero_idx = SIZE_MAX
};
GuMap* map = gu_new(GuMap, pool);
map->default_value = default_value;
map->hasher = hasher;
map->data = data;
map->key_size = key_size;
map->value_size = value_size;
map->fin.fn = gu_map_finalize;
map->kind = kind;
gu_pool_finally(pool, &map->fin);
gu_map_resize(map);
return map;
}
GuMap*
gu_map_type_make(GuMapType* mtype, GuPool* pool)
{
size_t key_size = 0;
if (mtype->hasher && mtype->hasher != gu_addr_hasher) {
key_size = gu_type_size(mtype->key_type);
}
size_t value_size = gu_type_size(mtype->value_type);
return gu_make_map(key_size, mtype->hasher,
value_size, mtype->default_value, pool);
}
size_t
pgf_read_len(PgfReader* rdr)
{
int32_t len = pgf_read_int(rdr);
// It's crucial that we return 0 on failure, so the
// caller can proceed without checking for error
// immediately.
gu_return_on_exn(rdr->err, 0);
if (GU_UNLIKELY(len < 0)) {
GuExnData* err_data = gu_raise(rdr->err, PgfReadTagExn);
if (err_data) {
PgfReadTagExn* rtag = gu_new(PgfReadTagExn, err_data->pool);
rtag->tag = len;
err_data->data = rtag;
}
return 0;
}
return len;
}
static PgfCCat*
pgf_read_fid(PgfReader* rdr, PgfConcr* concr)
{
int fid = pgf_read_int(rdr);
gu_return_on_exn(rdr->err, NULL);
PgfCCat* ccat = gu_map_get(concr->ccats, &fid, PgfCCat*);
if (!ccat) {
ccat = gu_new(PgfCCat, rdr->opool);
ccat->cnccat = NULL;
ccat->lindefs = NULL;
ccat->linrefs = NULL;
ccat->n_synprods = 0;
ccat->prods = NULL;
ccat->viterbi_prob = 0;
ccat->fid = fid;
ccat->conts = NULL;
ccat->answers = NULL;
gu_map_put(concr->ccats, &fid, PgfCCat*, ccat);
}
JNIEXPORT void JNICALL Java_org_grammaticalframework_pgf_Parser_addLiteralCallback
(JNIEnv* env, jclass clazz, jobject jconcr, jlong callbacksRef, jstring jcat, jobject jcallback, jobject jpool)
{
PgfConcr* concr = get_ref(env, jconcr);
GuPool* pool = get_ref(env, jpool);
JPgfLiteralCallback* callback = gu_new(JPgfLiteralCallback, pool);
callback->callback.match = jpgf_literal_callback_match;
callback->callback.predict = jpgf_literal_callback_predict;
callback->jcallback = (*env)->NewGlobalRef(env, jcallback);
callback->fin.fn = jpgf_literal_callback_fin;
jclass callback_class = (*env)->GetObjectClass(env, jcallback);
callback->match_methodId = (*env)->GetMethodID(env, callback_class, "match", "(ILjava/lang/String;I)Lorg/grammaticalframework/pgf/LiteralCallback$CallbackResult;");
callback->predict_methodId = (*env)->GetMethodID(env, callback_class, "predict", "(ILjava/lang/String;)Ljava/util/Iterator;");
gu_pool_finally(pool, &callback->fin);
pgf_callbacks_map_add_literal(concr, l2p(callbacksRef),
j2gu_string(env, jcat, pool), &callback->callback);
}
static PgfExprProb*
jpgf_literal_callback_match(PgfLiteralCallback* self, PgfConcr* concr,
size_t lin_idx,
GuString sentence, size_t* poffset,
GuPool *out_pool)
{
JPgfLiteralCallback* callback = gu_container(self, JPgfLiteralCallback, callback);
JNIEnv *env;
(*cachedJVM)->AttachCurrentThread(cachedJVM, &env, NULL);
jstring jsentence = gu2j_string(env, sentence);
size_t joffset = gu2j_string_offset(sentence, *poffset);
jobject result = (*env)->CallObjectMethod(env, callback->jcallback, callback->match_methodId, lin_idx, jsentence, joffset);
if (result == NULL)
return NULL;
jclass result_class = (*env)->GetObjectClass(env, result);
jfieldID epId = (*env)->GetFieldID(env, result_class, "ep", "Lorg/grammaticalframework/pgf/ExprProb;");
jobject jep = (*env)->GetObjectField(env, result, epId);
jclass ep_class = (*env)->GetObjectClass(env, jep);
jfieldID exprId = (*env)->GetFieldID(env, ep_class, "expr", "Lorg/grammaticalframework/pgf/Expr;");
jobject jexpr = (*env)->GetObjectField(env, jep, exprId);
jfieldID probId = (*env)->GetFieldID(env, ep_class, "prob", "D");
double prob = (*env)->GetDoubleField(env, jep, probId);
jfieldID offsetId = (*env)->GetFieldID(env, result_class, "offset", "I");
*poffset = j2gu_string_offset(sentence, (*env)->GetIntField(env, result, offsetId));
PgfExprProb* ep = gu_new(PgfExprProb, out_pool);
ep->expr = gu_variant_from_ptr(get_ref(env, jexpr));
ep->prob = prob;
{
// This is an uggly hack. We first show the expression ep->expr
// and then we read it back but in out_pool. The whole purpose
// of this is to copy the expression from the temporary pool
// that was created in the Java binding to the parser pool.
// There should be a real copying function or even better
// there must be a way to avoid copying at all.
GuPool* tmp_pool = gu_local_pool();
GuExn* err = gu_exn(tmp_pool);
GuStringBuf* sbuf = gu_string_buf(tmp_pool);
GuOut* out = gu_string_buf_out(sbuf);
pgf_print_expr(ep->expr, NULL, 0, out, err);
GuString str = gu_string_buf_freeze(sbuf, tmp_pool);
GuIn* in = gu_data_in((uint8_t*) str, strlen(str), tmp_pool);
ep->expr = pgf_read_expr(in, out_pool, err);
if (!gu_ok(err) || gu_variant_is_null(ep->expr)) {
throw_string_exception(env, "org/grammaticalframework/pgf/PGFError", "The expression cannot be parsed");
gu_pool_free(tmp_pool);
return NULL;
}
gu_pool_free(tmp_pool);
}
return ep;
}
static PgfExpr
pgf_value2expr(PgfReasoner* rs, int level, PgfClosure* clos)
{
clos = rs->eval_gates->enter(rs, clos);
if (clos == NULL)
return gu_null_variant;
PgfExpr expr = gu_null_variant;
size_t n_args = 0;
PgfClosure** args;
if (clos->code == rs->eval_gates->evaluate_value) {
PgfValue* val = (PgfValue*) clos;
PgfAbsFun* absfun = gu_container(val->con, PgfAbsFun, closure);
expr = absfun->ep.expr;
n_args = absfun->arity;
args = val->args;
} else if (clos->code == rs->eval_gates->evaluate_value_lit) {
PgfValueLit* val = (PgfValueLit*) clos;
PgfExprLit *elit =
gu_new_variant(PGF_EXPR_LIT,
PgfExprLit,
&expr, rs->out_pool);
GuVariantInfo i = gu_variant_open(val->lit);
switch (i.tag) {
case PGF_LITERAL_STR: {
PgfLiteralStr* lstr = i.data;
PgfLiteralStr* new_lstr =
gu_new_flex_variant(PGF_LITERAL_STR,
PgfLiteralStr,
val, strlen(lstr->val)+1,
&elit->lit, rs->out_pool);
strcpy(new_lstr->val, lstr->val);
break;
}
case PGF_LITERAL_INT: {
PgfLiteralInt* lint = i.data;
PgfLiteralInt* new_lint =
gu_new_variant(PGF_LITERAL_INT,
PgfLiteralInt,
&elit->lit, rs->out_pool);
new_lint->val = lint->val;
break;
}
case PGF_LITERAL_FLT: {
PgfLiteralFlt* lflt = i.data;
PgfLiteralFlt* new_lflt =
gu_new_variant(PGF_LITERAL_FLT,
PgfLiteralFlt,
&elit->lit, rs->out_pool);
new_lflt->val = lflt->val;
break;
}
default:
gu_impossible();
}
} else if (clos->code == rs->eval_gates->evaluate_value_pap) {
PgfValuePAP *pap = (PgfValuePAP*) clos;
PgfValueGen* gen =
gu_new(PgfValueGen, rs->pool);
gen->header.code = rs->eval_gates->evaluate_gen;
gen->level = level;
size_t n_args = pap->n_args/sizeof(PgfClosure*);
PgfValuePAP* new_pap = gu_new_flex(rs->pool, PgfValuePAP, args, n_args+1);
new_pap->header.code = rs->eval_gates->evaluate_value_pap;
new_pap->fun = pap->fun;
new_pap->n_args = pap->n_args+sizeof(PgfClosure*);
new_pap->args[0] = &gen->header;
for (size_t i = 0; i < n_args; i++) {
new_pap->args[i+1] = pap->args[i];
}
PgfExprAbs *eabs =
gu_new_variant(PGF_EXPR_ABS,
PgfExprAbs,
&expr, rs->out_pool);
eabs->bind_type = PGF_BIND_TYPE_EXPLICIT;
eabs->id = gu_format_string(rs->out_pool, "v%d", level);
eabs->body = pgf_value2expr(rs, level+1, &new_pap->header);
} else if (clos->code == rs->eval_gates->evaluate_value_const) {
PgfValuePAP* val = (PgfValuePAP*) clos;
if (val->fun->code == rs->eval_gates->evaluate_meta) {
PgfValueMeta* fun = (PgfValueMeta*) val->fun;
PgfExprMeta *emeta =
gu_new_variant(PGF_EXPR_META,
PgfExprMeta,
&expr, rs->out_pool);
emeta->id = fun->id;
} else if (val->fun->code == rs->eval_gates->evaluate_gen) {
PgfValueGen* fun = (PgfValueGen*) val->fun;
PgfExprVar *evar =
gu_new_variant(PGF_EXPR_VAR,
PgfExprVar,
&expr, rs->out_pool);
evar->var = level - fun->level - 1;
} else if (val->fun->code == rs->eval_gates->evaluate_sum) {
PgfValueSum* sum = (PgfValueSum*) val->fun;
PgfExpr e1,e2;
PgfExprFun *efun =
gu_new_flex_variant(PGF_EXPR_FUN,
PgfExprFun,
fun, 2,
&e1, rs->out_pool);
strcpy(efun->fun, "+");
PgfExprLit *elit =
gu_new_variant(PGF_EXPR_LIT,
PgfExprLit,
&e2, rs->out_pool);
elit->lit = sum->lit;
PgfExprApp* eapp =
gu_new_variant(PGF_EXPR_APP,
PgfExprApp,
&expr, rs->out_pool);
eapp->fun = e1;
eapp->arg = e2;
size_t n_consts = gu_buf_length(sum->consts);
for (size_t i = 0; i < n_consts; i++) {
PgfClosure* con =
gu_buf_get(sum->consts, PgfClosure*, i);
PgfExpr fun = expr;
PgfExpr arg =
pgf_value2expr(rs, level, con);
if (gu_variant_is_null(arg))
return gu_null_variant;
PgfExprApp* e =
gu_new_variant(PGF_EXPR_APP,
PgfExprApp,
&expr, rs->out_pool);
e->fun = fun;
e->arg = arg;
}
} else {
PgfClosure*
pgf_evaluate_expr_thunk(PgfReasoner* rs, PgfExprThunk* thunk)
{
PgfEnv* env = thunk->env;
PgfExpr expr = thunk->expr;
size_t n_args = 0;
PgfClosure** args = NULL;
PgfClosure* res = NULL;
repeat:;
GuVariantInfo ei = gu_variant_open(expr);
switch (ei.tag) {
case PGF_EXPR_ABS: {
PgfExprAbs* eabs = ei.data;
if (n_args > 0) {
PgfEnv* new_env = gu_new(PgfEnv, rs->pool);
new_env->next = env;
new_env->closure = args[--n_args];
env = new_env;
expr = eabs->body;
goto repeat;
} else {
thunk->header.code = rs->eval_gates->evaluate_value_lambda;
thunk->expr = eabs->body;
res = &thunk->header;
}
break;
}
case PGF_EXPR_APP: {
PgfExprApp* eapp = ei.data;
PgfExprThunk* thunk =
gu_new(PgfExprThunk, rs->pool);
thunk->header.code = rs->eval_gates->evaluate_expr_thunk;
thunk->env = env;
thunk->expr = eapp->arg;
if (n_args % PGF_ARGS_DELTA == 0) {
args = realloc(args, n_args + PGF_ARGS_DELTA);
}
args[n_args++] = &thunk->header;
expr = eapp->fun;
goto repeat;
}
case PGF_EXPR_LIT: {
PgfExprLit* elit = ei.data;
PgfValueLit* val = (PgfValueLit*) thunk;
val->header.code = rs->eval_gates->evaluate_value_lit;
val->lit = elit->lit;
res = &val->header;
break;
}
case PGF_EXPR_META: {
PgfExprMeta* emeta = ei.data;
PgfValueMeta* val =
gu_new(PgfValueMeta, rs->pool);
val->header.code = rs->eval_gates->evaluate_meta;
val->env = env;
val->id = emeta->id;
res = pgf_mk_pap(rs, &val->header, n_args, args);
break;
}
case PGF_EXPR_FUN: {
PgfExprFun* efun = ei.data;
PgfAbsFun* absfun =
gu_seq_binsearch(rs->abstract->funs, pgf_absfun_order, PgfAbsFun, efun->fun);
gu_assert(absfun != NULL);
if (absfun->closure.code != NULL) {
res = pgf_mk_pap(rs, (PgfClosure*) &absfun->closure, n_args, args);
} else {
size_t arity = absfun->arity;
if (n_args == arity) {
PgfValue* val = gu_new_flex(rs->pool, PgfValue, args, arity);
val->header.code = rs->eval_gates->evaluate_value;
val->con = (PgfClosure*) &absfun->closure;
for (size_t i = 0; i < arity; i++) {
val->args[i] = args[--n_args];
}
res = &val->header;
} else {
gu_assert(n_args < arity);
PgfExprThunk* lambda = gu_new(PgfExprThunk, rs->pool);
lambda->header.code = rs->eval_gates->evaluate_value_lambda;
lambda->env = NULL;
res = pgf_mk_pap(rs, &lambda->header, n_args, args);
for (size_t i = 0; i < arity; i++) {
PgfExpr new_expr, arg;
PgfExprVar *evar =
gu_new_variant(PGF_EXPR_VAR,
PgfExprVar,
&arg, rs->pool);
evar->var = arity-i-1;
PgfExprApp *eapp =
gu_new_variant(PGF_EXPR_APP,
PgfExprApp,
&new_expr, rs->pool);
eapp->fun = expr;
eapp->arg = arg;
expr = new_expr;
}
for (size_t i = 0; i < arity-1; i++) {
PgfExpr new_expr;
PgfExprAbs *eabs =
gu_new_variant(PGF_EXPR_ABS,
PgfExprAbs,
&new_expr, rs->pool);
eabs->bind_type = PGF_BIND_TYPE_EXPLICIT;
eabs->id = "_";
eabs->body = expr;
expr = new_expr;
}
lambda->expr = expr;
}
}
break;
}
case PGF_EXPR_VAR: {
PgfExprVar* evar = ei.data;
PgfEnv* tmp_env = env;
size_t i = evar->var;
while (i > 0) {
tmp_env = tmp_env->next;
if (tmp_env == NULL) {
GuExnData* err_data = gu_raise(rs->err, PgfExn);
if (err_data) {
err_data->data = "invalid de Bruijn index";
}
return NULL;
}
i--;
}
res = pgf_mk_pap(rs, tmp_env->closure, n_args, args);
break;
}
case PGF_EXPR_TYPED: {
PgfExprTyped* etyped = ei.data;
expr = etyped->expr;
goto repeat;
}
case PGF_EXPR_IMPL_ARG: {
PgfExprImplArg* eimpl = ei.data;
expr = eimpl->expr;
goto repeat;
}
default:
gu_impossible();
}
free(args);
return res;
}
