void sentence_delete(Sentence sent)
{
if (!sent) return;
sat_sentence_delete(sent);
free_sentence_words(sent);
wordgraph_delete(sent);
word_queue_delete(sent);
string_set_delete(sent->string_set);
free_parse_info(sent->parse_info);
free_linkages(sent);
post_process_free(sent->postprocessor);
post_process_free(sent->constituent_pp);
global_rand_state = sent->rand_state;
xfree((char *) sent, sizeof(struct Sentence_s));
}
void sentence_delete(Sentence sent)
{
if (!sent) return;
sat_sentence_delete(sent);
free_sentence_words(sent);
wordgraph_delete(sent);
string_set_delete(sent->string_set);
free_linkages(sent);
post_process_free(sent->postprocessor);
post_process_free(sent->constituent_pp);
global_rand_state = sent->rand_state;
pool_delete(sent->fm_Match_node);
pool_delete(sent->Table_connector_pool);
free(sent);
}
static bool setup_linkages(Sentence sent, extractor_t* pex,
fast_matcher_t* mchxt,
count_context_t* ctxt,
Parse_Options opts)
{
bool overflowed = build_parse_set(pex, sent, mchxt, ctxt, sent->null_count, opts);
print_time(opts, "Built parse set");
if (overflowed && (1 < opts->verbosity))
{
err_ctxt ec = { sent };
err_msgc(&ec, lg_Warn, "Count overflow.\n"
"Considering a random subset of %zu of an unknown and large number of linkages\n",
opts->linkage_limit);
}
if (sent->num_linkages_found == 0)
{
sent->num_linkages_alloced = 0;
sent->num_linkages_post_processed = 0;
sent->num_valid_linkages = 0;
sent->lnkages = NULL;
return overflowed;
}
sent->num_linkages_alloced =
MIN(sent->num_linkages_found, (int) opts->linkage_limit);
/* Now actually malloc the array in which we will process linkages. */
/* We may have been called before, e.g. this might be a panic parse,
* and the linkages array may still be there from last time.
* XXX free_linkages() zeros sent->num_linkages_found. */
if (sent->lnkages) free_linkages(sent);
sent->lnkages = linkage_array_new(sent->num_linkages_alloced);
return overflowed;
}
static void select_linkages(Sentence sent, fast_matcher_t* mchxt,
count_context_t* ctxt,
Parse_Options opts)
{
size_t in;
size_t N_linkages_found, N_linkages_alloced;
bool overflowed = build_parse_set(sent, mchxt, ctxt, sent->null_count, opts);
print_time(opts, "Built parse set");
if (overflowed && (1 < opts->verbosity))
{
err_ctxt ec;
ec.sent = sent;
err_msg(&ec, Warn, "Warning: Count overflow.\n"
"Considering a random subset of %zu of an unknown and large number of linkages\n",
opts->linkage_limit);
}
N_linkages_found = sent->num_linkages_found;
if (sent->num_linkages_found == 0)
{
sent->num_linkages_alloced = 0;
sent->num_linkages_post_processed = 0;
sent->num_valid_linkages = 0;
sent->lnkages = NULL;
return;
}
if (N_linkages_found > opts->linkage_limit)
{
N_linkages_alloced = opts->linkage_limit;
if (opts->verbosity > 1)
{
err_ctxt ec;
ec.sent = sent;
err_msg(&ec, Warn,
"Warning: Considering a random subset of %zu of %zu linkages\n",
N_linkages_alloced, N_linkages_found);
}
}
else
{
N_linkages_alloced = N_linkages_found;
}
/* Now actually malloc the array in which we will process linkages. */
/* We may have been called before, e.g this might be a panic parse,
* and the linkages array may still be there from last time.
* XXX free_linkages() zeros sent->num_linkages_found. */
if (sent->lnkages) free_linkages(sent);
sent->num_linkages_found = N_linkages_found;
sent->lnkages = linkage_array_new(N_linkages_alloced);
/* Generate an array of linkage indices to examine */
if (overflowed)
{
/* The negative index means that a random subset of links
* will be picked later on, in extract_links(). */
for (in=0; in < N_linkages_alloced; in++)
{
sent->lnkages[in].lifo.index = -(in+1);
}
}
else if (N_linkages_found == N_linkages_alloced)
{
for (in=0; in<N_linkages_alloced; in++)
sent->lnkages[in].lifo.index = in;
}
else
{
/* There are more linkages found than we can handle */
/* Pick a (quasi-)uniformly distributed random subset. */
if (opts->repeatable_rand)
sent->rand_state = N_linkages_found + sent->length;
for (in=0; in<N_linkages_alloced; in++)
{
size_t block_bottom, block_top;
double frac = (double) N_linkages_found;
frac /= (double) N_linkages_alloced;
block_bottom = (int) (((double) in) * frac);
block_top = (int) (((double) (in+1)) * frac);
sent->lnkages[in].lifo.index = block_bottom +
(rand_r(&sent->rand_state) % (block_top-block_bottom));
}
}
sent->num_linkages_alloced = N_linkages_alloced;
/* Later, we subtract the number of invalid linkages */
sent->num_valid_linkages = N_linkages_alloced;
}
/** Misnamed, this has nothing to do with chart parsing */
static void chart_parse(Sentence sent, Parse_Options opts)
{
int nl;
fast_matcher_t * mchxt;
count_context_t * ctxt;
/* Build lists of disjuncts */
prepare_to_parse(sent, opts);
if (resources_exhausted(opts->resources)) return;
mchxt = alloc_fast_matcher(sent);
ctxt = alloc_count_context(sent->length);
print_time(opts, "Initialized fast matcher");
if (resources_exhausted(opts->resources))
{
free_count_context(ctxt);
free_fast_matcher(mchxt);
return;
}
/* A parse set may have been already been built for this sentence,
* if it was previously parsed. If so we free it up before
* building another. Huh ?? How could that happen? */
free_parse_info(sent->parse_info);
sent->parse_info = parse_info_new(sent->length);
nl = opts->min_null_count;
while (true)
{
Count_bin hist;
s64 total;
if (resources_exhausted(opts->resources)) break;
sent->null_count = nl;
hist = do_parse(sent, mchxt, ctxt, sent->null_count, opts);
total = hist_total(&hist);
if (opts->verbosity > 1)
{
prt_error("Info: Total count with %zu null links: %lld\n",
sent->null_count, total);
}
/* total is 64-bit, num_linkages_found is 32-bit. Clamp */
total = (total > INT_MAX) ? INT_MAX : total;
total = (total < 0) ? INT_MAX : total;
sent->num_linkages_found = (int) total;
print_time(opts, "Counted parses");
select_linkages(sent, mchxt, ctxt, opts);
compute_chosen_disjuncts(sent);
sane_morphism(sent, opts);
post_process_linkages(sent, opts);
if (sent->num_valid_linkages > 0) break;
/* If we are here, then no valid linkages were found.
* If there was a parse overflow, give up now. */
if (PARSE_NUM_OVERFLOW < total) break;
/* loop termination */
if (nl == opts->max_null_count) break;
/* If we are here, we are going round again. Free stuff. */
free_linkages(sent);
nl++;
}
sort_linkages(sent, opts);
free_count_context(ctxt);
free_fast_matcher(mchxt);
}
/**
* classic_parse() -- parse the given sentence.
* Perform parsing, using the original link-grammar parsing algorithm
* given in the original link-grammar papers.
*
* Do the parse with the minimum number of null-links within the range
* specified by opts->min_null_count and opts->max_null_count.
*
* To that end, call do_parse() with an increasing null_count, from
* opts->min_null_count up to (including) opts->max_null_count, until a
* parse is found.
*
* A note about the disjuncts save/restore that is done here:
* To increase the parsing speed, before invoking do_parse(),
* pp_and_power_prune() is invoked to remove connectors which have no
* possibility to connect. It includes a significant optimization when
* null_count==0 that makes a more aggressive removal, but this
* optimization is not appropriate when null_count>0.
*
* So in case this optimization has been done and a complete parse (i.e.
* a parse when null_count==0) is not found, we are left with sentence
* disjuncts which are not appropriate to continue do_parse() tries with
* null_count>0. To solve that, we need to restore the original
* disjuncts of the sentence and call pp_and_power_prune() once again.
*/
void classic_parse(Sentence sent, Parse_Options opts)
{
fast_matcher_t * mchxt = NULL;
count_context_t * ctxt = NULL;
bool pp_and_power_prune_done = false;
Disjunct **disjuncts_copy = NULL;
bool is_null_count_0 = (0 == opts->min_null_count);
int max_null_count = MIN((int)sent->length, opts->max_null_count);
/* Build lists of disjuncts */
prepare_to_parse(sent, opts);
if (resources_exhausted(opts->resources)) return;
if (is_null_count_0 && (0 < max_null_count))
{
/* Save the disjuncts in case we need to parse with null_count>0. */
disjuncts_copy = alloca(sent->length * sizeof(Disjunct *));
for (size_t i = 0; i < sent->length; i++)
disjuncts_copy[i] = disjuncts_dup(sent->word[i].d);
}
for (int nl = opts->min_null_count; nl <= max_null_count; nl++)
{
Count_bin hist;
s64 total;
if (!pp_and_power_prune_done)
{
if (0 != nl)
{
pp_and_power_prune_done = true;
if (is_null_count_0)
opts->min_null_count = 1; /* Don't optimize for null_count==0. */
/* We are parsing now with null_count>0, when previously we
* parsed with null_count==0. Restore the save disjuncts. */
if (NULL != disjuncts_copy)
{
free_sentence_disjuncts(sent);
for (size_t i = 0; i < sent->length; i++)
sent->word[i].d = disjuncts_copy[i];
disjuncts_copy = NULL;
}
}
pp_and_power_prune(sent, opts);
if (is_null_count_0) opts->min_null_count = 0;
if (resources_exhausted(opts->resources)) break;
free_count_context(ctxt, sent);
free_fast_matcher(sent, mchxt);
pack_sentence(sent);
ctxt = alloc_count_context(sent);
mchxt = alloc_fast_matcher(sent);
print_time(opts, "Initialized fast matcher");
}
if (resources_exhausted(opts->resources)) break;
free_linkages(sent);
sent->null_count = nl;
hist = do_parse(sent, mchxt, ctxt, sent->null_count, opts);
total = hist_total(&hist);
lgdebug(D_PARSE, "Info: Total count with %zu null links: %lld\n",
sent->null_count, total);
/* total is 64-bit, num_linkages_found is 32-bit. Clamp */
total = (total > INT_MAX) ? INT_MAX : total;
total = (total < 0) ? INT_MAX : total;
sent->num_linkages_found = (int) total;
print_time(opts, "Counted parses");
extractor_t * pex = extractor_new(sent->length, sent->rand_state);
bool ovfl = setup_linkages(sent, pex, mchxt, ctxt, opts);
process_linkages(sent, pex, ovfl, opts);
free_extractor(pex);
post_process_lkgs(sent, opts);
if (sent->num_valid_linkages > 0) break;
if ((0 == nl) && (0 < max_null_count) && verbosity > 0)
prt_error("No complete linkages found.\n");
/* If we are here, then no valid linkages were found.
* If there was a parse overflow, give up now. */
if (PARSE_NUM_OVERFLOW < total) break;
//if (sent->num_linkages_found > 0 && nl>0) printf("NUM_LINKAGES_FOUND %d\n", sent->num_linkages_found);
}
sort_linkages(sent, opts);
if (NULL != disjuncts_copy)
{
for (size_t i = 0; i < sent->length; i++)
free_disjuncts(disjuncts_copy[i]);
}
free_count_context(ctxt, sent);
free_fast_matcher(sent, mchxt);
}
