/** * @name improve_by_chopping * * Start with the current word of blobs and its classification. Find * the worst blobs and try to divide them up to improve the ratings. * As long as ratings are produced by the new blob splitting. When * all the splitting has been accomplished all the ratings memory is * reclaimed. */ void Wordrec::improve_by_chopping(WERD_RES *word, BLOB_CHOICE_LIST_VECTOR *char_choices, STATE *best_state, BLOB_CHOICE_LIST_VECTOR *best_char_choices, DANGERR *fixpt, bool *best_choice_acceptable) { inT32 blob_number; float old_best; int fixpt_valid = 1; bool updated_best_choice = false; while (1) { // improvement loop if (!fixpt_valid) fixpt->clear(); old_best = word->best_choice->rating(); if (improve_one_blob(word->chopped_word, char_choices, &blob_number, &word->seam_array, fixpt, (fragments_guide_chopper && word->best_choice->fragment_mark()))) { getDict().LogNewSplit(blob_number); updated_best_choice = getDict().permute_characters(*char_choices, word->best_choice, word->raw_choice); if (old_best > word->best_choice->rating()) { set_n_ones(best_state, char_choices->length() - 1); fixpt_valid = 1; } else { insert_new_chunk(best_state, blob_number, char_choices->length() - 2); fixpt_valid = 0; } if (chop_debug) print_state("best state = ", best_state, count_blobs(word->chopped_word->blobs) - 1); } else { break; } // Check if we should break from the loop. bool done = false; bool replaced = false; if ((updated_best_choice && (*best_choice_acceptable = getDict().AcceptableChoice(char_choices, word->best_choice, fixpt, CHOPPER_CALLER, &replaced))) || char_choices->length() >= MAX_NUM_CHUNKS) { done = true; } if (replaced) update_blob_classifications(word->chopped_word, *char_choices); if (updated_best_choice) CopyCharChoices(*char_choices, best_char_choices); if (done) break; } if (!fixpt_valid) fixpt->clear(); }
/** * @name chop_one_blob * * Start with the current one-blob word and its classification. Find * the worst blobs and try to divide it up to improve the ratings. * Used for testing chopper. */ SEAM* Wordrec::chop_one_blob(const GenericVector<TBOX>& boxes, const GenericVector<BLOB_CHOICE*>& blob_choices, WERD_RES* word_res, int* blob_number) { if (prioritize_division) { return chop_overlapping_blob(boxes, true, word_res, blob_number); } else { return improve_one_blob(blob_choices, NULL, false, true, word_res, blob_number); } }
/** * @name improve_by_chopping * * Repeatedly chops the worst blob, classifying the new blobs fixing up all * the data, and incrementally runs the segmentation search until a good word * is found, or no more chops can be found. */ void Wordrec::improve_by_chopping(float rating_cert_scale, WERD_RES* word, BestChoiceBundle* best_choice_bundle, BlamerBundle* blamer_bundle, LMPainPoints* pain_points, GenericVector<SegSearchPending>* pending) { int blob_number; do { // improvement loop. // Make a simple vector of BLOB_CHOICEs to make it easy to pick which // one to chop. GenericVector<BLOB_CHOICE*> blob_choices; int num_blobs = word->ratings->dimension(); for (int i = 0; i < num_blobs; ++i) { BLOB_CHOICE_LIST* choices = word->ratings->get(i, i); if (choices == NULL || choices->empty()) { blob_choices.push_back(NULL); } else { BLOB_CHOICE_IT bc_it(choices); blob_choices.push_back(bc_it.data()); } } SEAM* seam = improve_one_blob(blob_choices, &best_choice_bundle->fixpt, false, false, word, &blob_number); if (seam == NULL) break; // A chop has been made. We have to correct all the data structures to // take into account the extra bottom-level blob. // Put the seam into the seam_array and correct everything else on the // word: ratings matrix (including matrix location in the BLOB_CHOICES), // states in WERD_CHOICEs, and blob widths. word->InsertSeam(blob_number, seam); // Insert a new entry in the beam array. best_choice_bundle->beam.insert(new LanguageModelState, blob_number); // Fixpts are outdated, but will get recalculated. best_choice_bundle->fixpt.clear(); // Remap existing pain points. pain_points->RemapForSplit(blob_number); // Insert a new pending at the chop point. pending->insert(SegSearchPending(), blob_number); // Classify the two newly created blobs using ProcessSegSearchPainPoint, // as that updates the pending correctly and adds new pain points. MATRIX_COORD pain_point(blob_number, blob_number); ProcessSegSearchPainPoint(0.0f, pain_point, "Chop1", pending, word, pain_points, blamer_bundle); pain_point.col = blob_number + 1; pain_point.row = blob_number + 1; ProcessSegSearchPainPoint(0.0f, pain_point, "Chop2", pending, word, pain_points, blamer_bundle); if (language_model_->language_model_ngram_on) { // N-gram evaluation depends on the number of blobs in a chunk, so we // have to re-evaluate everything in the word. ResetNGramSearch(word, best_choice_bundle, pending); blob_number = 0; } // Run language model incrementally. (Except with the n-gram model on.) UpdateSegSearchNodes(rating_cert_scale, blob_number, pending, word, pain_points, best_choice_bundle, blamer_bundle); } while (!language_model_->AcceptableChoiceFound() && word->ratings->dimension() < kMaxNumChunks); // If after running only the chopper best_choice is incorrect and no blame // has been yet set, blame the classifier if best_choice is classifier's // top choice and is a dictionary word (i.e. language model could not have // helped). Otherwise blame the tradeoff between the classifier and // the old language model (permuters). if (word->blamer_bundle != NULL && word->blamer_bundle->incorrect_result_reason() == IRR_CORRECT && !word->blamer_bundle->ChoiceIsCorrect(word->best_choice)) { bool valid_permuter = word->best_choice != NULL && Dict::valid_word_permuter(word->best_choice->permuter(), false); word->blamer_bundle->BlameClassifierOrLangModel(word, getDict().getUnicharset(), valid_permuter, wordrec_debug_blamer); } }
/********************************************************************** * improve_by_chopping * * Start with the current word of blobs and its classification. Find * the worst blobs and try to divide them up to improve the ratings. * As long as ratings are produced by the new blob splitting. When * all the splitting has been accomplished all the ratings memory is * reclaimed. **********************************************************************/ void improve_by_chopping(register TWERD *word, CHOICES_LIST *char_choices, int fx, STATE *best_state, A_CHOICE *best_choice, A_CHOICE *raw_choice, SEAMS *seam_list, DANGERR *fixpt, STATE *chop_states, INT32 *state_count, STATE *correct_state, INT32 pass) { INT32 blob_number; INT32 index; //to states CHOICES_LIST choices = *char_choices; float old_best; int fixpt_valid = 1; static INT32 old_count; //from pass1 do { /* Improvement loop */ if (!fixpt_valid) fixpt->index = -1; old_best = class_probability (best_choice); choices = improve_one_blob (word, *char_choices, fx, &blob_number, seam_list, fixpt, chop_states + *state_count, correct_state, pass); if (choices != NULL) { LogNewSplit(blob_number); permute_characters (choices, class_probability (best_choice), best_choice, raw_choice); *char_choices = choices; if (old_best > class_probability (best_choice)) { set_n_ones (best_state, array_count (*char_choices) - 1); fixpt_valid = 1; } else { insert_new_chunk (best_state, blob_number, array_count (*char_choices) - 2); fixpt_valid = 0; } if (*state_count > 0) { if (pass == 0) { for (index = 0; index < *state_count; index++) insert_new_chunk (&chop_states[index], blob_number, array_count (*char_choices) - 2); set_n_ones (&chop_states[index], array_count (*char_choices) - 1); } (*state_count)++; } if (chop_debug) print_state ("best state = ", best_state, count_blobs (word->blobs) - 1); if (first_pass) chops_performed1++; else chops_performed2++; } } while (choices && !AcceptableChoice (*char_choices, best_choice, raw_choice, fixpt) && !blob_skip && array_count (*char_choices) < MAX_NUM_CHUNKS); if (pass == 0) old_count = *state_count; else { if (old_count != *state_count) fprintf (matcher_fp, "Mis-matched state counts, " INT32FORMAT " pass1, " INT32FORMAT " pass2\n", old_count, *state_count); } if (!fixpt_valid) fixpt->index = -1; }