/**
* permute_choices
*
* Call append_choices() for each BLOB_CHOICE in BLOB_CHOICE_LIST
* with the given char_choice_index in char_choices.
*/
void Dict::permute_choices(
const char *debug,
const BLOB_CHOICE_LIST_VECTOR &char_choices,
int char_choice_index,
const CHAR_FRAGMENT_INFO *prev_char_frag_info,
WERD_CHOICE *word,
float certainties[],
float *limit,
WERD_CHOICE *best_choice,
int *attempts_left,
void *more_args) {
if (debug) {
tprintf("%s permute_choices: char_choice_index=%d"
" limit=%g rating=%g, certainty=%g word=%s\n",
debug, char_choice_index, *limit, word->rating(),
word->certainty(), word->debug_string().string());
}
if (char_choice_index < char_choices.length()) {
BLOB_CHOICE_IT blob_choice_it;
blob_choice_it.set_to_list(char_choices.get(char_choice_index));
for (blob_choice_it.mark_cycle_pt(); !blob_choice_it.cycled_list();
blob_choice_it.forward()) {
(*attempts_left)--;
append_choices(debug, char_choices, *(blob_choice_it.data()),
char_choice_index, prev_char_frag_info, word,
certainties, limit, best_choice, attempts_left, more_args);
if (*attempts_left <= 0) {
if (debug) tprintf("permute_choices(): attempts_left is 0\n");
break;
}
}
}
}
/**
* print_char_choices_list
*/
void print_char_choices_list(const char *msg,
const BLOB_CHOICE_LIST_VECTOR &char_choices,
const UNICHARSET ¤t_unicharset,
BOOL8 detailed) {
if (*msg != '\0') tprintf("%s\n", msg);
for (int x = 0; x < char_choices.length(); ++x) {
BLOB_CHOICE_IT c_it;
c_it.set_to_list(char_choices.get(x));
tprintf("char[%d]: %s\n", x,
current_unicharset.debug_str( c_it.data()->unichar_id()).string());
if (detailed)
print_ratings_list(" ", char_choices.get(x), current_unicharset);
}
}
void Wordrec::CopyCharChoices(const BLOB_CHOICE_LIST_VECTOR &from,
BLOB_CHOICE_LIST_VECTOR *to) {
to->delete_data_pointers();
to->clear();
for (int i = 0; i < from.size(); ++i) {
BLOB_CHOICE_LIST *cc_list = new BLOB_CHOICE_LIST();
cc_list->deep_copy(from[i], &BLOB_CHOICE::deep_copy);
to->push_back(cc_list);
}
}
/**
* append_choices
*
* Checks to see whether or not the next choice is worth appending to
* the word being generated. If so then keeps going deeper into the word.
*
* This function assumes that Dict::go_deeper_fxn_ is set.
*/
void Dict::append_choices(
const char *debug,
const BLOB_CHOICE_LIST_VECTOR &char_choices,
const BLOB_CHOICE &blob_choice,
int char_choice_index,
const CHAR_FRAGMENT_INFO *prev_char_frag_info,
WERD_CHOICE *word,
float certainties[],
float *limit,
WERD_CHOICE *best_choice,
int *attempts_left,
void *more_args) {
int word_ending =
(char_choice_index == char_choices.length() - 1) ? true : false;
// Deal with fragments.
CHAR_FRAGMENT_INFO char_frag_info;
if (!fragment_state_okay(blob_choice.unichar_id(), blob_choice.rating(),
blob_choice.certainty(), prev_char_frag_info, debug,
word_ending, &char_frag_info)) {
return; // blob_choice must be an invalid fragment
}
// Search the next letter if this character is a fragment.
if (char_frag_info.unichar_id == INVALID_UNICHAR_ID) {
permute_choices(debug, char_choices, char_choice_index + 1,
&char_frag_info, word, certainties, limit,
best_choice, attempts_left, more_args);
return;
}
// Add the next unichar.
float old_rating = word->rating();
float old_certainty = word->certainty();
uint8_t old_permuter = word->permuter();
certainties[word->length()] = char_frag_info.certainty;
word->append_unichar_id_space_allocated(
char_frag_info.unichar_id, char_frag_info.num_fragments,
char_frag_info.rating, char_frag_info.certainty);
// Explore the next unichar.
(this->*go_deeper_fxn_)(debug, char_choices, char_choice_index,
&char_frag_info, word_ending, word, certainties,
limit, best_choice, attempts_left, more_args);
// Remove the unichar we added to explore other choices in it's place.
word->remove_last_unichar_id();
word->set_rating(old_rating);
word->set_certainty(old_certainty);
word->set_permuter(old_permuter);
}
/**
* @name go_deeper_dawg_fxn
*
* If the choice being composed so far could be a dictionary word
* keep exploring choices.
*/
void Dict::go_deeper_dawg_fxn(
const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices,
int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info,
bool word_ending, WERD_CHOICE *word, float certainties[], float *limit,
WERD_CHOICE *best_choice, int *attempts_left, void *void_more_args) {
DawgArgs *more_args = static_cast<DawgArgs *>(void_more_args);
word_ending = (char_choice_index == char_choices.size()-1);
int word_index = word->length() - 1;
if (best_choice->rating() < *limit) return;
// Look up char in DAWG
// If the current unichar is an ngram first try calling
// letter_is_okay() for each unigram it contains separately.
UNICHAR_ID orig_uch_id = word->unichar_id(word_index);
bool checked_unigrams = false;
if (getUnicharset().get_isngram(orig_uch_id)) {
if (dawg_debug_level) {
tprintf("checking unigrams in an ngram %s\n",
getUnicharset().debug_str(orig_uch_id).string());
}
int num_unigrams = 0;
word->remove_last_unichar_id();
GenericVector<UNICHAR_ID> encoding;
const char *ngram_str = getUnicharset().id_to_unichar(orig_uch_id);
// Since the string came out of the unicharset, failure is impossible.
ASSERT_HOST(getUnicharset().encode_string(ngram_str, true, &encoding, nullptr,
nullptr));
bool unigrams_ok = true;
// Construct DawgArgs that reflect the current state.
DawgPositionVector unigram_active_dawgs = *(more_args->active_dawgs);
DawgPositionVector unigram_updated_dawgs;
DawgArgs unigram_dawg_args(&unigram_active_dawgs,
&unigram_updated_dawgs,
more_args->permuter);
// Check unigrams in the ngram with letter_is_okay().
for (int i = 0; unigrams_ok && i < encoding.size(); ++i) {
UNICHAR_ID uch_id = encoding[i];
ASSERT_HOST(uch_id != INVALID_UNICHAR_ID);
++num_unigrams;
word->append_unichar_id(uch_id, 1, 0.0, 0.0);
unigrams_ok = (this->*letter_is_okay_)(
&unigram_dawg_args,
word->unichar_id(word_index+num_unigrams-1),
word_ending && i == encoding.size() - 1);
(*unigram_dawg_args.active_dawgs) = *(unigram_dawg_args.updated_dawgs);
if (dawg_debug_level) {
tprintf("unigram %s is %s\n",
getUnicharset().debug_str(uch_id).string(),
unigrams_ok ? "OK" : "not OK");
}
}
// Restore the word and copy the updated dawg state if needed.
while (num_unigrams-- > 0) word->remove_last_unichar_id();
word->append_unichar_id_space_allocated(orig_uch_id, 1, 0.0, 0.0);
if (unigrams_ok) {
checked_unigrams = true;
more_args->permuter = unigram_dawg_args.permuter;
*(more_args->updated_dawgs) = *(unigram_dawg_args.updated_dawgs);
}
}
// Check which dawgs from the dawgs_ vector contain the word
// up to and including the current unichar.
if (checked_unigrams || (this->*letter_is_okay_)(
more_args, word->unichar_id(word_index), word_ending)) {
// Add a new word choice
if (word_ending) {
if (dawg_debug_level) {
tprintf("found word = %s\n", word->debug_string().string());
}
if (strcmp(output_ambig_words_file.string(), "") != 0) {
if (output_ambig_words_file_ == nullptr) {
output_ambig_words_file_ =
fopen(output_ambig_words_file.string(), "wb+");
if (output_ambig_words_file_ == nullptr) {
tprintf("Failed to open output_ambig_words_file %s\n",
output_ambig_words_file.string());
exit(1);
}
STRING word_str;
word->string_and_lengths(&word_str, nullptr);
word_str += " ";
fprintf(output_ambig_words_file_, "%s", word_str.string());
}
STRING word_str;
word->string_and_lengths(&word_str, nullptr);
word_str += " ";
fprintf(output_ambig_words_file_, "%s", word_str.string());
}
WERD_CHOICE *adjusted_word = word;
adjusted_word->set_permuter(more_args->permuter);
update_best_choice(*adjusted_word, best_choice);
} else { // search the next letter
// Make updated_* point to the next entries in the DawgPositionVector
// arrays (that were originally created in dawg_permute_and_select)
++(more_args->updated_dawgs);
// Make active_dawgs and constraints point to the updated ones.
++(more_args->active_dawgs);
permute_choices(debug, char_choices, char_choice_index + 1,
prev_char_frag_info, word, certainties, limit,
best_choice, attempts_left, more_args);
// Restore previous state to explore another letter in this position.
--(more_args->updated_dawgs);
--(more_args->active_dawgs);
}
} else {
if (dawg_debug_level) {
tprintf("last unichar not OK at index %d in %s\n",
word_index, word->debug_string().string());
}
}
}
/**********************************************************************
* select_blob_to_split
*
* These are the results of the last classification. Find a likely
* place to apply splits.
**********************************************************************/
inT16 Wordrec::select_blob_to_split(const BLOB_CHOICE_LIST_VECTOR &char_choices,
float rating_ceiling,
bool split_next_to_fragment) {
BLOB_CHOICE_IT blob_choice_it;
BLOB_CHOICE *blob_choice;
BLOB_CHOICE_IT temp_it;
int x;
float worst = -MAX_FLOAT32;
int worst_index = -1;
float worst_near_fragment = -MAX_FLOAT32;
int worst_index_near_fragment = -1;
const CHAR_FRAGMENT **fragments = NULL;
if (chop_debug) {
if (rating_ceiling < MAX_FLOAT32)
cprintf("rating_ceiling = %8.4f\n", rating_ceiling);
else
cprintf("rating_ceiling = No Limit\n");
}
if (split_next_to_fragment && char_choices.length() > 0) {
fragments = new const CHAR_FRAGMENT *[char_choices.length()];
if (char_choices.get(0) != NULL) {
temp_it.set_to_list(char_choices.get(0));
fragments[0] = getDict().getUnicharset().get_fragment(
temp_it.data()->unichar_id());
} else {
fragments[0] = NULL;
}
}
for (x = 0; x < char_choices.length(); ++x) {
if (char_choices.get(x) == NULL) {
if (fragments != NULL) {
delete[] fragments;
}
return x;
} else {
blob_choice_it.set_to_list(char_choices.get(x));
blob_choice = blob_choice_it.data();
// Populate fragments for the following position.
if (split_next_to_fragment && x+1 < char_choices.length()) {
if (char_choices.get(x+1) != NULL) {
temp_it.set_to_list(char_choices.get(x+1));
fragments[x+1] = getDict().getUnicharset().get_fragment(
temp_it.data()->unichar_id());
} else {
fragments[x+1] = NULL;
}
}
if (blob_choice->rating() < rating_ceiling &&
blob_choice->certainty() < tessedit_certainty_threshold) {
// Update worst and worst_index.
if (blob_choice->rating() > worst) {
worst_index = x;
worst = blob_choice->rating();
}
if (split_next_to_fragment) {
// Update worst_near_fragment and worst_index_near_fragment.
bool expand_following_fragment =
(x + 1 < char_choices.length() &&
fragments[x+1] != NULL && !fragments[x+1]->is_beginning());
bool expand_preceding_fragment =
(x > 0 && fragments[x-1] != NULL && !fragments[x-1]->is_ending());
if ((expand_following_fragment || expand_preceding_fragment) &&
blob_choice->rating() > worst_near_fragment) {
worst_index_near_fragment = x;
worst_near_fragment = blob_choice->rating();
if (chop_debug) {
cprintf("worst_index_near_fragment=%d"
" expand_following_fragment=%d"
" expand_preceding_fragment=%d\n",
worst_index_near_fragment,
expand_following_fragment,
expand_preceding_fragment);
}
}
}
}
}
}
if (fragments != NULL) {
delete[] fragments;
}
// TODO(daria): maybe a threshold of badness for
// worst_near_fragment would be useful.
return worst_index_near_fragment != -1 ?
worst_index_near_fragment : worst_index;
}
