void indri::collection::CompressedCollection::close() { _lookup.close(); if( _output ) { _output->flush(); delete _output; _output = 0; } _storage.close(); indri::utility::HashTable<const char*, lemur::file::Keyfile*>::iterator iter; for( iter = _forwardLookups.begin(); iter != _forwardLookups.end(); iter++ ) { (*iter->second)->close(); delete (*iter->second); } for( iter = _reverseLookups.begin(); iter != _reverseLookups.end(); iter++ ) { (*iter->second)->close(); delete (*iter->second); } _forwardLookups.clear(); _reverseLookups.clear(); string_set_delete( _strings ); _strings = string_set_create(); }
void dictionary_delete(Dictionary dict) { if (!dict) return; if (verbosity > 0) { prt_error("Info: Freeing dictionary %s", dict->name); } #ifdef USE_CORPUS lg_corpus_delete(dict->corpus); #endif if (dict->affix_table != NULL) { affix_list_delete(dict->affix_table); dictionary_delete(dict->affix_table); } spellcheck_destroy(dict->spell_checker); connector_set_delete(dict->unlimited_connector_set); if (dict->close) dict->close(dict); pp_knowledge_close(dict->base_knowledge); pp_knowledge_close(dict->hpsg_knowledge); string_set_delete(dict->string_set); free_regexs(dict->regex_root); #ifdef USE_ANYSPLIT free_anysplit(dict); #endif free_dictionary(dict); xfree(dict, sizeof(struct Dictionary_s)); object_open(NULL, NULL, NULL); /* Free the directory path cache */ }
indri::collection::CompressedCollection::~CompressedCollection() { close(); delete _output; deflateEnd( _stream ); delete _stream; string_set_delete( _strings ); }
void pp_knowledge_close(pp_knowledge *k) { if (!k) return; /* clear the memory taken up by k */ free(k->starting_link_lookup_table); free_link_sets(k); free_rules(k); pp_linkset_close(k->set_of_links_starting_bounded_domain); string_set_delete(k->string_set); if (NULL != k->lt) pp_lexer_close(k->lt); free(k); }
void pp_knowledge_close(pp_knowledge *k) { /* clear the memory taken up by k */ xfree((void*)k->starting_link_lookup_table, ((1+k->nStartingLinks)*sizeof(StartingLinkAndDomain))); free_link_sets(k); free_rules(k); pp_linkset_close(k->set_of_links_starting_bounded_domain); string_set_delete(k->string_set); pp_lexer_close(k->lt); xfree((void*)k, sizeof(pp_knowledge)); }
Sentence sentence_create(char *input_string, Dictionary dict) { Sentence sent; int i; free_lookup_list(); sent = (Sentence) xalloc(sizeof(struct Sentence_s)); sent->dict = dict; sent->length = 0; sent->num_linkages_found = 0; sent->num_linkages_alloced = 0; sent->num_linkages_post_processed = 0; sent->num_valid_linkages = 0; sent->link_info = NULL; sent->deletable = NULL; sent->effective_dist = NULL; sent->num_valid_linkages = 0; sent->null_count = 0; sent->parse_info = NULL; sent->string_set = string_set_create(); if (!separate_sentence(input_string, sent)) { string_set_delete(sent->string_set); xfree(sent, sizeof(struct Sentence_s)); return NULL; } sent->q_pruned_rules = FALSE; /* for post processing */ sent->is_conjunction = (char *) xalloc(sizeof(char)*sent->length); set_is_conjunction(sent); initialize_conjunction_tables(sent); for (i=0; i<sent->length; i++) { /* in case we free these before they set to anything else */ sent->word[i].x = NULL; sent->word[i].d = NULL; } if (!(dict->unknown_word_defined && dict->use_unknown_word)) { if (!sentence_in_dictionary(sent)) { sentence_delete(sent); return NULL; } } if (!build_sentence_expressions(sent)) { sentence_delete(sent); return NULL; } return sent; }
void post_process_close_sentence(Postprocessor *pp) { if (pp==NULL) return; pp_linkset_clear(pp->set_of_links_of_sentence); pp_linkset_clear(pp->set_of_links_in_an_active_rule); string_set_delete(pp->sentence_link_name_set); pp->sentence_link_name_set = string_set_create(); pp->n_local_rules_firing = 0; pp->n_global_rules_firing = 0; pp->relevant_contains_one_rules[0] = -1; pp->relevant_contains_none_rules[0] = -1; free_pp_node(pp); }
void sentence_delete(Sentence sent) { /*free_andlists(sent); */ free_sentence_disjuncts(sent); free_sentence_expressions(sent); string_set_delete(sent->string_set); free_parse_set(sent); free_post_processing(sent); post_process_close_sentence(sent->dict->postprocessor); free_lookup_list(); free_deletable(sent); free_effective_dist(sent); xfree(sent->is_conjunction, sizeof(char)*sent->length); xfree((char *) sent, sizeof(struct Sentence_s)); }
void sentence_delete(Sentence sent) { if (!sent) return; sat_sentence_delete(sent); free_sentence_words(sent); wordgraph_delete(sent); word_queue_delete(sent); string_set_delete(sent->string_set); free_parse_info(sent->parse_info); free_linkages(sent); post_process_free(sent->postprocessor); post_process_free(sent->constituent_pp); global_rand_state = sent->rand_state; xfree((char *) sent, sizeof(struct Sentence_s)); }
void sentence_delete(Sentence sent) { if (!sent) return; sat_sentence_delete(sent); free_sentence_words(sent); wordgraph_delete(sent); string_set_delete(sent->string_set); free_linkages(sent); post_process_free(sent->postprocessor); post_process_free(sent->constituent_pp); global_rand_state = sent->rand_state; pool_delete(sent->fm_Match_node); pool_delete(sent->Table_connector_pool); free(sent); }
void post_process_close(Postprocessor *pp) { /* frees up memory associated with pp, previously allocated by open */ if (pp==NULL) return; string_set_delete(pp->sentence_link_name_set); pp_linkset_close(pp->set_of_links_of_sentence); pp_linkset_close(pp->set_of_links_in_an_active_rule); xfree(pp->relevant_contains_one_rules, (1+pp->knowledge->n_contains_one_rules) *(sizeof pp->relevant_contains_one_rules[0])); xfree(pp->relevant_contains_none_rules, (1+pp->knowledge->n_contains_none_rules) *(sizeof pp->relevant_contains_none_rules[0])); pp_knowledge_close(pp->knowledge); free_pp_node(pp); xfree(pp, sizeof(Postprocessor)); }
void post_process_free(Postprocessor *pp) { /* frees up memory associated with pp, previously allocated by open */ if (pp == NULL) return; string_set_delete(pp->string_set); pp_linkset_close(pp->set_of_links_of_sentence); pp_linkset_close(pp->set_of_links_in_an_active_rule); free(pp->relevant_contains_one_rules); free(pp->relevant_contains_none_rules); pp->knowledge = NULL; free_pp_node(pp); free(pp->visited); post_process_free_data(&pp->pp_data); free(pp->pp_data.domain_array); free(pp->pp_data.word_links); free(pp); }
int dictionary_delete(Dictionary dict) { if (verbosity > 0) { fprintf(stderr, "Freeing dictionary %s\n", dict->name); } if (dict->affix_table != NULL) { dictionary_delete(dict->affix_table); } connector_set_delete(dict->andable_connector_set); connector_set_delete(dict->unlimited_connector_set); post_process_close(dict->postprocessor); post_process_close(dict->constituent_pp); string_set_delete(dict->string_set); free_dictionary(dict); xfree(dict, sizeof(struct Dictionary_s)); return 0; }
/* Inserts the CNT strings from 0 to CNT - 1 (inclusive) into a set in the order specified by INSERTIONS, then deletes them in the order specified by DELETIONS, checking the set's contents for correctness after each operation. */ static void test_insert_delete (const int insertions[], const int deletions[], size_t cnt) { struct string_set set; size_t i; string_set_init (&set); check_string_set (&set, NULL, 0); for (i = 0; i < cnt; i++) { check (string_set_insert (&set, make_string (insertions[i]))); check_string_set (&set, insertions, i + 1); } for (i = 0; i < cnt; i++) { check (string_set_delete (&set, make_string (deletions[i]))); check_string_set (&set, deletions + i + 1, cnt - i - 1); } string_set_destroy (&set); }
/* Inserts strings into a set in ascending order, then delete in ascending order. */ static void test_insert_ordered (void) { const int max_elems = 64; int *values; struct string_set set; int i; string_set_init (&set); values = xnmalloc (max_elems, sizeof *values); for (i = 0; i < max_elems; i++) { values[i] = i; string_set_insert_nocopy (&set, xstrdup (make_string (i))); check_string_set (&set, values, i + 1); } for (i = 0; i < max_elems; i++) { string_set_delete (&set, make_string (i)); check_string_set (&set, values + i + 1, max_elems - i - 1); } string_set_destroy (&set); free (values); }
static char * print_flat_constituents(con_context_t *ctxt, Linkage linkage) { int num_words; Sentence sent; Postprocessor * pp; int s, numcon_total, numcon_subl, num_subl; char * q; sent = linkage_get_sentence(linkage); ctxt->phrase_ss = string_set_create(); pp = linkage->sent->dict->constituent_pp; numcon_total = 0; count_words_used(ctxt, linkage); num_subl = linkage->num_sublinkages; if(num_subl > MAXSUBL) { num_subl=MAXSUBL; if(verbosity>=2) printf("Number of sublinkages exceeds maximum: only considering first %d sublinkages\n", MAXSUBL); } if(linkage->unionized==1 && num_subl>1) num_subl--; for (s=0; s<num_subl; s++) { linkage_set_current_sublinkage(linkage, s); linkage_post_process(linkage, pp); num_words = linkage_get_num_words(linkage); generate_misc_word_info(ctxt, linkage); numcon_subl = read_constituents_from_domains(ctxt, linkage, numcon_total, s); numcon_total = numcon_total + numcon_subl; } numcon_total = merge_constituents(ctxt, linkage, numcon_total); numcon_total = last_minute_fixes(ctxt, linkage, numcon_total); q = exprint_constituent_structure(ctxt, linkage, numcon_total); string_set_delete(ctxt->phrase_ss); ctxt->phrase_ss = NULL; return q; }
static char * do_print_flat_constituents(con_context_t *ctxt, Linkage linkage) { int numcon_total= 0, numcon_subl; char * q; Sentence sent = linkage->sent; assert(NULL != sent->lnkages, "No linkages"); /* Sentence already free()'d */ ctxt->phrase_ss = string_set_create(); generate_misc_word_info(ctxt, linkage); if (NULL == sent->constituent_pp) /* First time for this sentence */ sent->constituent_pp = post_process_new(sent->dict->hpsg_knowledge); do_post_process(sent->constituent_pp, linkage, linkage->is_sent_long); /** No-op. If we wanted to debug domain names, we could do this... * linkage_free_pp_info(linkage); * linkage_set_domain_names(sent->constituent_pp, linkage); */ linkage->hpsg_pp_data = sent->constituent_pp->pp_data; pp_new_domain_array(&linkage->hpsg_pp_data); numcon_subl = read_constituents_from_domains(ctxt, linkage, numcon_total); numcon_total += numcon_subl; assert (numcon_total < ctxt->conlen, "Too many constituents (c)"); numcon_total = merge_constituents(ctxt, linkage, numcon_total); assert (numcon_total < ctxt->conlen, "Too many constituents (d)"); numcon_total = new_style_conjunctions(ctxt, linkage, numcon_total); assert (numcon_total < ctxt->conlen, "Too many constituents (e)"); numcon_total = last_minute_fixes(ctxt, linkage, numcon_total); assert (numcon_total < ctxt->conlen, "Too many constituents (f)"); q = exprint_constituent_structure(ctxt, linkage, numcon_total); string_set_delete(ctxt->phrase_ss); ctxt->phrase_ss = NULL; return q; }
/** * Read dictionary entries from a wide-character string "input". * All other parts are read from files. */ static Dictionary dictionary_six_str(const char * lang, const char * input, const char * dict_name, const char * pp_name, const char * cons_name, const char * affix_name, const char * regex_name) { const char * t; Dictionary dict; Dict_node *dict_node; dict = (Dictionary) xalloc(sizeof(struct Dictionary_s)); memset(dict, 0, sizeof(struct Dictionary_s)); dict->num_entries = 0; dict->is_special = false; dict->already_got_it = '\0'; dict->line_number = 0; dict->root = NULL; dict->regex_root = NULL; dict->word_file_header = NULL; dict->exp_list = NULL; dict->affix_table = NULL; dict->recursive_error = false; dict->version = NULL; #ifdef HAVE_SQLITE dict->db_handle = NULL; #endif #ifdef USE_ANYSPLIT dict->anysplit = NULL; #endif /* Language and file-name stuff */ dict->string_set = string_set_create(); dict->lang = lang; t = strrchr (lang, '/'); if (t) dict->lang = string_set_add(t+1, dict->string_set); dict->name = string_set_add(dict_name, dict->string_set); /* * A special setup per dictionary type. The check here assumes the affix * dictionary name contains "affix". FIXME: For not using this * assumption, the dictionary creating stuff needs a rearrangement. */ if (0 == strstr(dict->name, "affix")) { /* To disable spell-checking, just set the checker to NULL */ dict->spell_checker = spellcheck_create(dict->lang); dict->insert_entry = insert_list; dict->lookup_list = lookup_list; dict->free_lookup = free_llist; dict->lookup = boolean_lookup; } else { /* * Affix dictionary. */ size_t i; dict->insert_entry = load_affix; dict->lookup = return_true; /* initialize the class table */ dict->afdict_class = malloc(sizeof(*dict->afdict_class) * NUMELEMS(afdict_classname)); for (i = 0; i < NUMELEMS(afdict_classname); i++) { dict->afdict_class[i].mem_elems = 0; dict->afdict_class[i].length = 0; dict->afdict_class[i].string = NULL; } } dict->affix_table = NULL; /* Read dictionary from the input string. */ dict->input = input; dict->pin = dict->input; if (!read_dictionary(dict)) { dict->pin = NULL; dict->input = NULL; goto failure; } dict->pin = NULL; dict->input = NULL; if (NULL == affix_name) { /* * The affix table is handled alone in this invocation. * Skip the rest of processing! * FIXME: The dictionary creating stuff needs a rearrangement. */ return dict; } dict->affix_table = dictionary_six(lang, affix_name, NULL, NULL, NULL, NULL); if (dict->affix_table == NULL) { prt_error("Error: Could not open affix file %s", affix_name); goto failure; } if (! afdict_init(dict)) goto failure; if (read_regex_file(dict, regex_name)) goto failure; if (compile_regexs(dict->regex_root, dict)) goto failure; #ifdef USE_CORPUS dict->corpus = lg_corpus_new(); #endif dict->left_wall_defined = boolean_dictionary_lookup(dict, LEFT_WALL_WORD); dict->right_wall_defined = boolean_dictionary_lookup(dict, RIGHT_WALL_WORD); dict->empty_word_defined = boolean_dictionary_lookup(dict, EMPTY_WORD_MARK); dict->base_knowledge = pp_knowledge_open(pp_name); dict->hpsg_knowledge = pp_knowledge_open(cons_name); dict->unknown_word_defined = boolean_dictionary_lookup(dict, UNKNOWN_WORD); dict->use_unknown_word = true; dict_node = dictionary_lookup_list(dict, UNLIMITED_CONNECTORS_WORD); if (dict_node != NULL) { dict->unlimited_connector_set = connector_set_create(dict_node->exp); } else { dict->unlimited_connector_set = NULL; } free_lookup(dict_node); return dict; failure: string_set_delete(dict->string_set); if (dict->affix_table) xfree(dict->affix_table, sizeof(struct Dictionary_s)); xfree(dict, sizeof(struct Dictionary_s)); return NULL; }
indri::parse::NormalizationTransformation::~NormalizationTransformation() { if(_acronyms) string_set_delete(_acronyms); }
static Dictionary dictionary_six_str(const char * lang, const char * input, const char * dict_name, const char * pp_name, const char * cons_name, const char * affix_name, const char * regex_name) { const char * t; Dictionary dict; Dict_node *dict_node; dict = (Dictionary) xalloc(sizeof(struct Dictionary_s)); memset(dict, 0, sizeof(struct Dictionary_s)); /* Language and file-name stuff */ dict->string_set = string_set_create(); t = strrchr (lang, '/'); t = (NULL == t) ? lang : t+1; dict->lang = string_set_add(t, dict->string_set); lgdebug(D_USER_FILES, "Debug: Language: %s\n", dict->lang); dict->name = string_set_add(dict_name, dict->string_set); /* * A special setup per dictionary type. The check here assumes the affix * dictionary name contains "affix". FIXME: For not using this * assumption, the dictionary creating stuff needs a rearrangement. */ if (0 == strstr(dict->name, "affix")) { /* To disable spell-checking, just set the checker to NULL */ dict->spell_checker = spellcheck_create(dict->lang); #if defined HAVE_HUNSPELL || defined HAVE_ASPELL /* TODO: * 1. Set the spell option to 0, to signify no spell checking is done. * 2. On verbosity >= 1, add a detailed message on the reason. */ if (NULL == dict->spell_checker) prt_error("Info: Spell checker disabled."); #endif dict->insert_entry = insert_list; dict->lookup_list = lookup_list; dict->free_lookup = free_llist; dict->lookup = boolean_lookup; } else { /* * Affix dictionary. */ size_t i; dict->insert_entry = load_affix; dict->lookup = return_true; /* initialize the class table */ dict->afdict_class = malloc(sizeof(*dict->afdict_class) * ARRAY_SIZE(afdict_classname)); for (i = 0; i < ARRAY_SIZE(afdict_classname); i++) { dict->afdict_class[i].mem_elems = 0; dict->afdict_class[i].length = 0; dict->afdict_class[i].string = NULL; } } dict->affix_table = NULL; /* Read dictionary from the input string. */ dict->input = input; dict->pin = dict->input; if (!read_dictionary(dict)) { dict->pin = NULL; dict->input = NULL; goto failure; } dict->pin = NULL; dict->input = NULL; if (NULL == affix_name) { /* * The affix table is handled alone in this invocation. * Skip the rest of processing! * FIXME: The dictionary creating stuff needs a rearrangement. */ return dict; } /* If we don't have a locale per dictionary, the following * will also set the program's locale. */ dict->locale = linkgrammar_get_dict_locale(dict); set_utf8_program_locale(); #ifdef HAVE_LOCALE_T /* We have a locale per dictionary. */ if (NULL != dict->locale) dict->locale_t = newlocale_LC_CTYPE(dict->locale); /* If we didn't succeed to set the dictionary locale, the program will * SEGFAULT when it tries to use it with the isw*() functions. * So set it to the current program's locale as a last resort. */ if (NULL == dict->locale) { dict->locale = setlocale(LC_CTYPE, NULL); dict->locale_t = newlocale_LC_CTYPE(setlocale(LC_CTYPE, NULL)); prt_error("Warning: Couldn't set dictionary locale! " "Using current program locale %s", dict->locale); } /* If dict->locale is still not set, there is a bug. */ assert((locale_t)0 != dict->locale_t, "Dictionary locale is not set."); #else /* We don't have a locale per dictionary - but anyway make sure * dict->locale is consistent with the current program's locale, * and especially that it is not NULL. It still indicates the intended * locale of this dictionary and the locale of the compiled regexs. */ dict->locale = setlocale(LC_CTYPE, NULL); #endif /* HAVE_LOCALE_T */ dict->affix_table = dictionary_six(lang, affix_name, NULL, NULL, NULL, NULL); if (dict->affix_table == NULL) { prt_error("Error: Could not open affix file %s", affix_name); goto failure; } if (! afdict_init(dict)) goto failure; /* * Process the regex file. * We have to compile regexs using the dictionary locale, * so make a temporary locale swap. */ if (read_regex_file(dict, regex_name)) goto failure; const char *locale = setlocale(LC_CTYPE, NULL); locale = strdupa(locale); /* setlocale() uses static memory. */ setlocale(LC_CTYPE, dict->locale); lgdebug(+D_DICT, "Regexs locale %s\n", setlocale(LC_CTYPE, NULL)); if (compile_regexs(dict->regex_root, dict)) { locale = setlocale(LC_CTYPE, locale); goto failure; } locale = setlocale(LC_CTYPE, locale); assert(NULL != locale, "Cannot restore program locale\n"); #ifdef USE_CORPUS dict->corpus = lg_corpus_new(); #endif dict->left_wall_defined = boolean_dictionary_lookup(dict, LEFT_WALL_WORD); dict->right_wall_defined = boolean_dictionary_lookup(dict, RIGHT_WALL_WORD); dict->empty_word_defined = boolean_dictionary_lookup(dict, EMPTY_WORD_MARK); dict->base_knowledge = pp_knowledge_open(pp_name); dict->hpsg_knowledge = pp_knowledge_open(cons_name); dict->unknown_word_defined = boolean_dictionary_lookup(dict, UNKNOWN_WORD); dict->use_unknown_word = true; dict_node = dictionary_lookup_list(dict, UNLIMITED_CONNECTORS_WORD); if (dict_node != NULL) dict->unlimited_connector_set = connector_set_create(dict_node->exp); free_lookup(dict_node); return dict; failure: string_set_delete(dict->string_set); if (dict->affix_table) xfree(dict->affix_table, sizeof(struct Dictionary_s)); xfree(dict, sizeof(struct Dictionary_s)); return NULL; }
/* The following function is dictionary_create with an extra paramater called "path". If this is non-null, then the path used to find the file is taken from that path. Otherwise the path is taken from the dict_name. This is only needed because an affix_file is opened by a recursive call to this function. */ static Dictionary internal_dictionary_create(char * dict_name, char * pp_name, char * cons_name, char * affix_name, char * path) { Dictionary dict; static int rand_table_inited=FALSE; Dict_node *dict_node; char * dictionary_path_name; dict = (Dictionary) xalloc(sizeof(struct Dictionary_s)); if (!rand_table_inited) { init_randtable(); rand_table_inited=TRUE; } dict->string_set = string_set_create(); dict->name = string_set_add(dict_name, dict->string_set); dict->num_entries = 0; dict->is_special = FALSE; dict->already_got_it = '\0'; dict->line_number = 1; dict->root = NULL; dict->word_file_header = NULL; dict->exp_list = NULL; dict->affix_table = NULL; /* *DS* remove this if (pp_name != NULL) { dict->post_process_filename = string_set_add(pp_name, dict->string_set); } else { dict->post_process_filename = NULL; } */ if (path != NULL) dictionary_path_name = path; else dictionary_path_name = dict_name; if (!open_dictionary(dictionary_path_name, dict)) { lperror(NODICT, dict_name); string_set_delete(dict->string_set); xfree(dict, sizeof(struct Dictionary_s)); return NULL; } if (!read_dictionary(dict)) { string_set_delete(dict->string_set); xfree(dict, sizeof(struct Dictionary_s)); return NULL; } dict->left_wall_defined = boolean_dictionary_lookup(dict, LEFT_WALL_WORD); dict->right_wall_defined = boolean_dictionary_lookup(dict, RIGHT_WALL_WORD); dict->postprocessor = post_process_open(dict->name, pp_name); dict->constituent_pp = post_process_open(dict->name, cons_name); dict->affix_table = NULL; if (affix_name != NULL) { dict->affix_table = internal_dictionary_create(affix_name, NULL, NULL, NULL, dict_name); if (dict->affix_table == NULL) { fprintf(stderr, "%s\n", lperrmsg); exit(-1); } } dict->unknown_word_defined = boolean_dictionary_lookup(dict, UNKNOWN_WORD); dict->use_unknown_word = TRUE; dict->capitalized_word_defined = boolean_dictionary_lookup(dict, PROPER_WORD); dict->pl_capitalized_word_defined = boolean_dictionary_lookup(dict, PL_PROPER_WORD); dict->hyphenated_word_defined = boolean_dictionary_lookup(dict, HYPHENATED_WORD); dict->number_word_defined = boolean_dictionary_lookup(dict, NUMBER_WORD); dict->ing_word_defined = boolean_dictionary_lookup(dict, ING_WORD); dict->s_word_defined = boolean_dictionary_lookup(dict, S_WORD); dict->ed_word_defined = boolean_dictionary_lookup(dict, ED_WORD); dict->ly_word_defined = boolean_dictionary_lookup(dict, LY_WORD); dict->max_cost = 1000; if ((dict_node = dictionary_lookup(dict, ANDABLE_CONNECTORS_WORD)) != NULL) { dict->andable_connector_set = connector_set_create(dict_node->exp); } else { dict->andable_connector_set = NULL; } if ((dict_node = dictionary_lookup(dict, UNLIMITED_CONNECTORS_WORD)) != NULL) { dict->unlimited_connector_set = connector_set_create(dict_node->exp); } else { dict->unlimited_connector_set = NULL; } free_lookup_list(); return dict; }