예제 #1
0
// Moves this to the target unicity table.
void FontInfoTable::MoveTo(UnicityTable<FontInfo>* target) {
  target->clear();
  target->set_compare_callback(NewPermanentTessCallback(CompareFontInfo));
  target->set_clear_callback(NewPermanentTessCallback(FontInfoDeleteCallback));
  for (int i = 0; i < size(); ++i) {
    // Bit copy the FontInfo and steal all the pointers.
    target->push_back(get(i));
    get(i).name = NULL;
    get(i).spacing_vec = NULL;
  }
}
예제 #2
0
bool UNICHARSET::load_from_file(tesseract::TFile *file, bool skip_fragments) {
  TessResultCallback2<char *, char *, int> *fgets_cb =
      NewPermanentTessCallback(file, &tesseract::TFile::FGets);
  bool success = load_via_fgets(fgets_cb, skip_fragments);
  delete fgets_cb;
  return success;
}
예제 #3
0
void Dawg::iterate_words(const UNICHARSET &unicharset,
                         TessCallback1<const char *> *cb) const {
  std::unique_ptr<TessCallback1<const WERD_CHOICE *>> shim(
      NewPermanentTessCallback(CallWithUTF8, cb));
  WERD_CHOICE word(&unicharset);
  iterate_words_rec(word, 0, shim.get());
}
예제 #4
0
bool UNICHARSET::load_from_file(FILE *file, bool skip_fragments) {
  LocalFilePointer lfp(file);
  TessResultCallback2<char *, char *, int> *fgets_cb =
      NewPermanentTessCallback(&lfp, &LocalFilePointer::fgets);
  bool success = load_via_fgets(fgets_cb, skip_fragments);
  delete fgets_cb;
  return success;
}
예제 #5
0
bool UNICHARSET::load_from_inmemory_file(
    const char *memory, int mem_size, bool skip_fragments) {
  InMemoryFilePointer mem_fp(memory, mem_size);
  TessResultCallback2<char *, char *, int> *fgets_cb =
      NewPermanentTessCallback(&mem_fp, &InMemoryFilePointer::fgets);
  bool success = load_via_fgets(fgets_cb, skip_fragments);
  delete fgets_cb;
  return success;
}
예제 #6
0
// Moves any non-empty FontSpacingInfo entries from other to this.
void FontInfoTable::MoveSpacingInfoFrom(FontInfoTable* other) {
  set_compare_callback(NewPermanentTessCallback(CompareFontInfo));
  set_clear_callback(NewPermanentTessCallback(FontInfoDeleteCallback));
  for (int i = 0; i < other->size(); ++i) {
    GenericVector<FontSpacingInfo*>* spacing_vec = other->get(i).spacing_vec;
    if (spacing_vec != NULL) {
      int target_index = get_index(other->get(i));
      if (target_index < 0) {
        // Bit copy the FontInfo and steal all the pointers.
        push_back(other->get(i));
        other->get(i).name = NULL;
      } else {
        delete [] get(target_index).spacing_vec;
        get(target_index).spacing_vec = other->get(i).spacing_vec;
      }
      other->get(i).spacing_vec = NULL;
    }
  }
}
예제 #7
0
void Wordrec::InitBlamerForSegSearch(WERD_RES *word_res,
                                     LMPainPoints *pain_points,
                                     BlamerBundle *blamer_bundle,
                                     STRING *blamer_debug) {
  pain_points->Clear();  // Clear pain points heap.
  TessResultCallback2<bool, int, int>* pp_cb = NewPermanentTessCallback(
      pain_points, &LMPainPoints::GenerateForBlamer,
      static_cast<double>(segsearch_max_char_wh_ratio), word_res);
  blamer_bundle->InitForSegSearch(word_res->best_choice, word_res->ratings,
                                  getDict().WildcardID(), wordrec_debug_blamer,
                                  blamer_debug, pp_cb);
  delete pp_cb;
}
예제 #8
0
// returns 0 if successful.
int WriteDawgAsWordlist(const UNICHARSET &unicharset,
                        const tesseract::Dawg *dawg,
                        const char *outfile_name) {
  FILE *out = fopen(outfile_name, "wb");
  if (out == NULL) {
    tprintf("Could not open %s for writing.\n", outfile_name);
    return 1;
  }
  WordOutputter outputter(out);
  TessCallback1<const char *> *print_word_cb =
      NewPermanentTessCallback(&outputter, &WordOutputter::output_word);
  dawg->iterate_words(unicharset, print_word_cb);
  delete print_word_cb;
  return fclose(out);
}
예제 #9
0
// Set the universal_id member of each font to be unique among all
// instances of the same font loaded.
void Tesseract::SetupUniversalFontIds() {
  // Note that we can get away with bitwise copying FontInfo in
  // all_fonts, as it is a temporary structure and we avoid setting the
  // delete callback.
  UnicityTable<FontInfo> all_fonts;
  all_fonts.set_compare_callback(NewPermanentTessCallback(CompareFontInfo));

  // Create the universal ID table.
  CollectFonts(get_fontinfo_table(), &all_fonts);
  for (int i = 0; i < sub_langs_.size(); ++i) {
    CollectFonts(sub_langs_[i]->get_fontinfo_table(), &all_fonts);
  }
  // Assign ids from the table to each font table.
  AssignIds(all_fonts, &get_fontinfo_table());
  for (int i = 0; i < sub_langs_.size(); ++i) {
    AssignIds(all_fonts, &sub_langs_[i]->get_fontinfo_table());
  }
  font_table_size_ = all_fonts.size();
}
예제 #10
0
FontInfoTable::FontInfoTable() {
  set_compare_callback(NewPermanentTessCallback(CompareFontInfo));
  set_clear_callback(NewPermanentTessCallback(FontInfoDeleteCallback));
}
예제 #11
0
Classify::Classify()
  : BOOL_MEMBER(prioritize_division, FALSE,
                "Prioritize blob division over chopping", this->params()),
    INT_MEMBER(tessedit_single_match, FALSE,
               "Top choice only from CP", this->params()),
    BOOL_MEMBER(classify_enable_learning, true,
                "Enable adaptive classifier", this->params()),
    INT_MEMBER(classify_debug_level, 0, "Classify debug level",
               this->params()),
    INT_MEMBER(classify_norm_method, character, "Normalization Method   ...",
               this->params()),
    double_MEMBER(classify_char_norm_range, 0.2,
                  "Character Normalization Range ...", this->params()),
    double_MEMBER(classify_min_norm_scale_x, 0.0, "Min char x-norm scale ...",
                  this->params()),  /* PREV DEFAULT 0.1 */
    double_MEMBER(classify_max_norm_scale_x, 0.325, "Max char x-norm scale ...",
                  this->params()),  /* PREV DEFAULT 0.3 */
    double_MEMBER(classify_min_norm_scale_y, 0.0, "Min char y-norm scale ...",
                  this->params()),  /* PREV DEFAULT 0.1 */
    double_MEMBER(classify_max_norm_scale_y, 0.325, "Max char y-norm scale ...",
                  this->params()),  /* PREV DEFAULT 0.3 */
    double_MEMBER(classify_max_rating_ratio, 1.5,
                  "Veto ratio between classifier ratings", this->params()),
    double_MEMBER(classify_max_certainty_margin, 5.5,
                  "Veto difference between classifier certainties",
                  this->params()),
    BOOL_MEMBER(tess_cn_matching, 0, "Character Normalized Matching",
                this->params()),
    BOOL_MEMBER(tess_bn_matching, 0, "Baseline Normalized Matching",
                this->params()),
    BOOL_MEMBER(classify_enable_adaptive_matcher, 1,
                "Enable adaptive classifier",
                this->params()),
    BOOL_MEMBER(classify_use_pre_adapted_templates, 0,
                "Use pre-adapted classifier templates", this->params()),
    BOOL_MEMBER(classify_save_adapted_templates, 0,
               "Save adapted templates to a file", this->params()),
    BOOL_MEMBER(classify_enable_adaptive_debugger, 0, "Enable match debugger",
                this->params()),
    BOOL_MEMBER(classify_nonlinear_norm, 0,
                "Non-linear stroke-density normalization", this->params()),
    INT_MEMBER(matcher_debug_level, 0, "Matcher Debug Level", this->params()),
    INT_MEMBER(matcher_debug_flags, 0, "Matcher Debug Flags", this->params()),
    INT_MEMBER(classify_learning_debug_level, 0, "Learning Debug Level: ",
               this->params()),
    double_MEMBER(matcher_good_threshold, 0.125, "Good Match (0-1)",
                  this->params()),
    double_MEMBER(matcher_great_threshold, 0.0, "Great Match (0-1)",
                  this->params()),
    double_MEMBER(matcher_perfect_threshold, 0.02, "Perfect Match (0-1)",
                  this->params()),
    double_MEMBER(matcher_bad_match_pad, 0.15, "Bad Match Pad (0-1)",
                  this->params()),
    double_MEMBER(matcher_rating_margin, 0.1, "New template margin (0-1)",
                  this->params()),
    double_MEMBER(matcher_avg_noise_size, 12.0, "Avg. noise blob length",
                  this->params()),
    INT_MEMBER(matcher_permanent_classes_min, 1, "Min # of permanent classes",
               this->params()),
    INT_MEMBER(matcher_min_examples_for_prototyping, 3,
               "Reliable Config Threshold", this->params()),
    INT_MEMBER(matcher_sufficient_examples_for_prototyping, 5,
               "Enable adaption even if the ambiguities have not been seen",
               this->params()),
    double_MEMBER(matcher_clustering_max_angle_delta, 0.015,
                  "Maximum angle delta for prototype clustering",
                  this->params()),
    double_MEMBER(classify_misfit_junk_penalty, 0.0,
                  "Penalty to apply when a non-alnum is vertically out of "
                  "its expected textline position",
                  this->params()),
    double_MEMBER(rating_scale, 1.5, "Rating scaling factor", this->params()),
    double_MEMBER(certainty_scale, 20.0, "Certainty scaling factor",
                  this->params()),
    double_MEMBER(tessedit_class_miss_scale, 0.00390625,
                  "Scale factor for features not used", this->params()),
    double_MEMBER(classify_adapted_pruning_factor, 2.5,
                  "Prune poor adapted results this much worse than best result",
                  this->params()),
    double_MEMBER(classify_adapted_pruning_threshold, -1.0,
                  "Threshold at which classify_adapted_pruning_factor starts",
                  this->params()),
    INT_MEMBER(classify_adapt_proto_threshold, 230,
               "Threshold for good protos during adaptive 0-255",
               this->params()),
    INT_MEMBER(classify_adapt_feature_threshold, 230,
               "Threshold for good features during adaptive 0-255",
               this->params()),
    BOOL_MEMBER(disable_character_fragments, TRUE,
                "Do not include character fragments in the"
                " results of the classifier", this->params()),
    double_MEMBER(classify_character_fragments_garbage_certainty_threshold,
                  -3.0, "Exclude fragments that do not look like whole"
                  " characters from training and adaption", this->params()),
    BOOL_MEMBER(classify_debug_character_fragments, FALSE,
                "Bring up graphical debugging windows for fragments training",
                this->params()),
    BOOL_MEMBER(matcher_debug_separate_windows, FALSE,
                "Use two different windows for debugging the matching: "
                "One for the protos and one for the features.", this->params()),
    STRING_MEMBER(classify_learn_debug_str, "", "Class str to debug learning",
                  this->params()),
    INT_MEMBER(classify_class_pruner_threshold, 229,
               "Class Pruner Threshold 0-255", this->params()),
    INT_MEMBER(classify_class_pruner_multiplier, 15,
               "Class Pruner Multiplier 0-255:       ", this->params()),
    INT_MEMBER(classify_cp_cutoff_strength, 7,
               "Class Pruner CutoffStrength:         ", this->params()),
    INT_MEMBER(classify_integer_matcher_multiplier, 10,
               "Integer Matcher Multiplier  0-255:   ", this->params()),
    EnableLearning(true),
    INT_MEMBER(il1_adaption_test, 0, "Dont adapt to i/I at beginning of word",
               this->params()),
    BOOL_MEMBER(classify_bln_numeric_mode, 0,
                "Assume the input is numbers [0-9].", this->params()),
    double_MEMBER(speckle_large_max_size, 0.30, "Max large speckle size",
                  this->params()),
    double_MEMBER(speckle_rating_penalty, 10.0,
                  "Penalty to add to worst rating for noise", this->params()),
    shape_table_(NULL),
    dict_(this),
    static_classifier_(NULL) {
  fontinfo_table_.set_compare_callback(
      NewPermanentTessCallback(CompareFontInfo));
  fontinfo_table_.set_clear_callback(
      NewPermanentTessCallback(FontInfoDeleteCallback));
  fontset_table_.set_compare_callback(
      NewPermanentTessCallback(CompareFontSet));
  fontset_table_.set_clear_callback(
      NewPermanentTessCallback(FontSetDeleteCallback));
  AdaptedTemplates = NULL;
  PreTrainedTemplates = NULL;
  AllProtosOn = NULL;
  AllConfigsOn = NULL;
  AllConfigsOff = NULL;
  TempProtoMask = NULL;
  NormProtos = NULL;

  NumAdaptationsFailed = 0;

  learn_debug_win_ = NULL;
  learn_fragmented_word_debug_win_ = NULL;
  learn_fragments_debug_win_ = NULL;

  CharNormCutoffs = new uinT16[MAX_NUM_CLASSES];
  BaselineCutoffs = new uinT16[MAX_NUM_CLASSES];
}
예제 #12
0
// Deletes all samples with zero features marked by KillSample.
    void TrainingSampleSet::DeleteDeadSamples() {
        samples_.compact(
                NewPermanentTessCallback(this, &TrainingSampleSet::DeleteableSample));
        num_raw_samples_ = samples_.size();
        // Samples must be re-organized now we have deleted a few.
    }
예제 #13
0
// Apart from command-line flags, input is a collection of lstmf files, that
// were previously created using tesseract with the lstm.train config file.
// The program iterates over the inputs, feeding the data to the network,
// until the error rate reaches a specified target or max_iterations is reached.
int main(int argc, char **argv) {
  ParseArguments(&argc, &argv);
  // Purify the model name in case it is based on the network string.
  if (FLAGS_model_output.empty()) {
    tprintf("Must provide a --model_output!\n");
    return 1;
  }
  STRING model_output = FLAGS_model_output.c_str();
  for (int i = 0; i < model_output.length(); ++i) {
    if (model_output[i] == '[' || model_output[i] == ']')
      model_output[i] = '-';
    if (model_output[i] == '(' || model_output[i] == ')')
      model_output[i] = '_';
  }
  // Setup the trainer.
  STRING checkpoint_file = FLAGS_model_output.c_str();
  checkpoint_file += "_checkpoint";
  STRING checkpoint_bak = checkpoint_file + ".bak";
  tesseract::LSTMTrainer trainer(
      NULL, NULL, NULL, NULL, FLAGS_model_output.c_str(),
      checkpoint_file.c_str(), FLAGS_debug_interval,
      static_cast<inT64>(FLAGS_max_image_MB) * 1048576);

  // Reading something from an existing model doesn't require many flags,
  // so do it now and exit.
  if (FLAGS_stop_training || FLAGS_debug_network) {
    if (!trainer.TryLoadingCheckpoint(FLAGS_continue_from.c_str())) {
      tprintf("Failed to read continue from: %s\n",
              FLAGS_continue_from.c_str());
      return 1;
    }
    if (FLAGS_debug_network) {
      trainer.DebugNetwork();
    } else {
      if (FLAGS_train_mode & tesseract::TF_INT_MODE)
        trainer.ConvertToInt();
      GenericVector<char> recognizer_data;
      trainer.SaveRecognitionDump(&recognizer_data);
      if (!tesseract::SaveDataToFile(recognizer_data,
                                     FLAGS_model_output.c_str())) {
        tprintf("Failed to write recognition model : %s\n",
                FLAGS_model_output.c_str());
      }
    }
    return 0;
  }

  // Get the list of files to process.
  if (FLAGS_train_listfile.empty()) {
    tprintf("Must supply a list of training filenames! --train_listfile\n");
    return 1;
  }
  GenericVector<STRING> filenames;
  if (!tesseract::LoadFileLinesToStrings(FLAGS_train_listfile.c_str(),
                                         &filenames)) {
    tprintf("Failed to load list of training filenames from %s\n",
            FLAGS_train_listfile.c_str());
    return 1;
  }

  UNICHARSET unicharset;
  // Checkpoints always take priority if they are available.
  if (trainer.TryLoadingCheckpoint(checkpoint_file.string()) ||
      trainer.TryLoadingCheckpoint(checkpoint_bak.string())) {
    tprintf("Successfully restored trainer from %s\n",
            checkpoint_file.string());
  } else {
    if (!FLAGS_continue_from.empty()) {
      // Load a past model file to improve upon.
      if (!trainer.TryLoadingCheckpoint(FLAGS_continue_from.c_str())) {
        tprintf("Failed to continue from: %s\n", FLAGS_continue_from.c_str());
        return 1;
      }
      tprintf("Continuing from %s\n", FLAGS_continue_from.c_str());
      trainer.InitIterations();
    }
    if (FLAGS_continue_from.empty() || FLAGS_append_index >= 0) {
      // We need a unicharset to start from scratch or append.
      string unicharset_str;
      // Character coding to be used by the classifier.
      if (!unicharset.load_from_file(FLAGS_U.c_str())) {
        tprintf("Error: must provide a -U unicharset!\n");
        return 1;
      }
      tesseract::SetupBasicProperties(true, &unicharset);
      if (FLAGS_append_index >= 0) {
        tprintf("Appending a new network to an old one!!");
        if (FLAGS_continue_from.empty()) {
          tprintf("Must set --continue_from for appending!\n");
          return 1;
        }
      }
      // We are initializing from scratch.
      trainer.InitCharSet(unicharset, FLAGS_script_dir.c_str(),
                          FLAGS_train_mode);
      if (!trainer.InitNetwork(FLAGS_net_spec.c_str(), FLAGS_append_index,
                               FLAGS_net_mode, FLAGS_weight_range,
                               FLAGS_learning_rate, FLAGS_momentum)) {
        tprintf("Failed to create network from spec: %s\n",
                FLAGS_net_spec.c_str());
        return 1;
      }
      trainer.set_perfect_delay(FLAGS_perfect_sample_delay);
    }
  }
  if (!trainer.LoadAllTrainingData(filenames)) {
    tprintf("Load of images failed!!\n");
    return 1;
  }

  bool best_dumped = true;
  char* best_model_dump = NULL;
  size_t best_model_size = 0;
  STRING best_model_name;
  tesseract::LSTMTester tester(static_cast<inT64>(FLAGS_max_image_MB) *
                               1048576);
  tesseract::TestCallback tester_callback = nullptr;
  if (!FLAGS_eval_listfile.empty()) {
    if (!tester.LoadAllEvalData(FLAGS_eval_listfile.c_str())) {
      tprintf("Failed to load eval data from: %s\n",
              FLAGS_eval_listfile.c_str());
      return 1;
    }
    tester_callback =
        NewPermanentTessCallback(&tester, &tesseract::LSTMTester::RunEvalAsync);
  }
  do {
    // Train a few.
    int iteration = trainer.training_iteration();
    for (int target_iteration = iteration + kNumPagesPerBatch;
         iteration < target_iteration;
         iteration = trainer.training_iteration()) {
      trainer.TrainOnLine(&trainer, false);
    }
    STRING log_str;
    trainer.MaintainCheckpoints(tester_callback, &log_str);
    tprintf("%s\n", log_str.string());
  } while (trainer.best_error_rate() > FLAGS_target_error_rate &&
           (trainer.training_iteration() < FLAGS_max_iterations ||
            FLAGS_max_iterations == 0));
  delete tester_callback;
  tprintf("Finished! Error rate = %g\n", trainer.best_error_rate());
  return 0;
} /* main */