SpellCheck::~SpellCheck(){ if (spell_checker) aspell_speller_save_all_word_lists(spell_checker); if (config) WSSET(WS_APP_ASPELL_LANG, aspell_config_retrieve(config, "lang")); delete_aspell_config(config); delete_aspell_speller(spell_checker); }
/** * Free memory structures used wiith spell-checker 'chk' */ void spellcheck_destroy(void * chk) { struct linkgrammar_aspell *aspell = (struct linkgrammar_aspell *)chk; if (aspell) { delete_aspell_speller(aspell->speller); delete_aspell_config(aspell->config); free(aspell); aspell = NULL; } }
ASpellChecker::~ASpellChecker() { if(config_) { delete_aspell_config(config_); config_ = NULL; } if(speller_) { delete_aspell_speller(speller_); speller_ = NULL; } }
bool AspellAdapterImpl::isDictionaryAvailable(const std::string & langCode) { AspellConfig * tempAspellConfig = new_aspell_config(); aspell_config_replace(tempAspellConfig, "lang", langCode.c_str()); AspellCanHaveError * possibleError = new_aspell_speller(tempAspellConfig); bool result = true; if (aspell_error_number(possibleError) != 0) { result = false; } else { AspellSpeller * tempSpeller = to_aspell_speller(possibleError); delete_aspell_speller(tempSpeller); } delete_aspell_config(tempAspellConfig); return result; }
void weechat_aspell_speller_free_value_cb (struct t_hashtable *hashtable, const void *key, void *value) { AspellSpeller *ptr_speller; /* make C compiler happy */ (void) hashtable; if (weechat_aspell_plugin->debug) { weechat_printf (NULL, "%s: removing speller for lang \"%s\"", ASPELL_PLUGIN_NAME, (const char *)key); } /* free aspell data */ ptr_speller = (AspellSpeller *)value; aspell_speller_save_all_word_lists (ptr_speller); delete_aspell_speller (ptr_speller); }
/** * Dispose of a userdata * @param u the object to dispose */ void userdata_dispose( userdata *u ) { if ( u != NULL ) { if ( u->rules != NULL ) u->rules = recipe_dispose( u->rules ); if ( u->ignoring != NULL ) stack_delete( u->ignoring ); if ( u->range_stack != NULL ) stack_delete( u->range_stack ); if ( u->spell_config != NULL ) delete_aspell_config(u->spell_config); if ( u->spell_checker != NULL ) delete_aspell_speller(u->spell_checker); if ( u->last_word != NULL ) free( u->last_word ); if ( u->dest_map != NULL ) hashmap_dispose( u->dest_map ); // we don't own the hh_exceptions free( u ); } }
static gboolean gtkspell_set_language_internal(GtkSpell *spell, const gchar *lang, GError **error) { AspellConfig *config; AspellCanHaveError *err; if (lang == NULL) { lang = g_getenv("LANG"); if (lang) { if (g_strncasecmp(lang, "C", 1) == 0) lang = NULL; else if (lang[0] == 0) lang = NULL; } } config = new_aspell_config(); if (lang) aspell_config_replace(config, "language-tag", lang); aspell_config_replace(config, "encoding", "utf-8"); err = new_aspell_speller(config); delete_aspell_config(config); if (aspell_error_number(err) != 0) { #ifdef USING_ASPELL g_set_error(error, GTKSPELL_ERROR, GTKSPELL_ERROR_BACKEND, "aspell: %s", aspell_error_message(err)); #elif defined USING_PSPELL g_set_error(error, GTKSPELL_ERROR, GTKSPELL_ERROR_BACKEND, "pspell: %s", aspell_error_message(err)); #endif return FALSE; } if (spell->speller) delete_aspell_speller(spell->speller); spell->speller = to_aspell_speller(err); return TRUE; }
bool SpellChecker::buildCheckers() { foreach(AspellSpeller *speller, checkers.values()) delete_aspell_speller(speller); checkers.clear(); // load languages to check from configuration QString checkedStr = config_file.readEntry("ASpell", "Checked", "pl"); QStringList checkedList = checkedStr.split(',', QString::SkipEmptyParts); if (config_file.readBoolEntry("ASpell", "Accents", false)) aspell_config_replace(spellConfig, "ignore-accents", "true"); else aspell_config_replace(spellConfig, "ignore-accents", "false"); if (config_file.readBoolEntry( "ASpell", "Case", false)) aspell_config_replace(spellConfig, "ignore-case", "true"); else aspell_config_replace(spellConfig, "ignore-case", "false"); // create aspell checkers for each language for (unsigned int i = 0; i < checkedList.count(); i++) { addCheckedLang(checkedList[i]); /* if ( !addCheckedLang( checkedList[i] ) ) { delete_aspell_config( spellConfig ); delete config; return false; } */ } return true; }
Speller::~Speller() { if (speller) delete_aspell_speller(speller); }
static int get_aspell_hits(const char *word, int len) { if (len < 2) { log_message(DEBUG, " [-]Skip aspell checking (word is very short)"); return NO_LANGUAGE; } AspellConfig *spell_config = new_aspell_config(); for (int lang = 0; lang < xconfig->total_languages; lang++) { char *lang_word = (char *) malloc(1 * sizeof(char)); if (lang_word == NULL) continue; lang_word[0] = NULLSYM; for (int i = 0; i < len; i++) { KeyCode kc; int modifier; main_window->xkeymap->char_to_keycode(main_window->xkeymap, word[i], &kc, &modifier); char *symbol = keycode_to_symbol(kc, lang, modifier); if ((symbol == NULL) || (lang_word == NULL)) continue; lang_word = (char *) realloc(lang_word, (strlen(lang_word) + strlen(symbol) + 1) * sizeof(char)); if (lang_word != NULL) strcat(lang_word, symbol); free(symbol); } if (lang_word == NULL) continue; aspell_config_replace(spell_config, "lang", xconfig->languages[lang].dir); AspellCanHaveError *possible_err = new_aspell_speller(spell_config); AspellSpeller *spell_checker = 0; if (aspell_error_number(possible_err) == 0) { spell_checker = to_aspell_speller(possible_err); int correct = aspell_speller_check(spell_checker, lang_word, strlen(lang_word)); if (correct) { log_message(DEBUG, " [+]Found this word in %s aspell dictionary", xconfig->get_lang_name(xconfig, lang)); delete_aspell_speller(spell_checker); free(lang_word); return lang; } } else { log_message(DEBUG, " [!]Error aspell checking for %s aspell dictionary", xconfig->get_lang_name(xconfig, lang)); } delete_aspell_speller(spell_checker); free(lang_word); } log_message(DEBUG, " [-]This word has no hits for all aspell dictionaries"); return NO_LANGUAGE; }
void SpellChecker::setLanguage(const QString &lang) { if (lang.isEmpty()) return; delete_aspell_speller(spell_checker1); delete_aspell_speller(spell_checker2); bad_language.clear(); m_lang2 = "en"; m_lang1 = m_map->value(lang, QString("en")); if (lang.contains("+")) { QStringList sl = lang.split("+"); m_lang1 = m_map->value(sl[0], QString("en")); m_lang2 = m_map->value(sl[1], QString("en")); if ((m_lang1 == "deu")||(lang == "ger")) { m_lang1 = "de_DE"; } if ((m_lang2 == "deu")||(lang == "ger")) { m_lang2 = "de_DE"; } } if (lang == "rus_fra") { m_lang1 = "ru"; m_lang2 = "fr"; } else if (lang == "rus_ger") { m_lang1 = "ru"; m_lang2 = "de"; } else if (lang == "rus_spa") { m_lang1 = "ru"; m_lang2 = "es"; } if ((lang == "deu")||(lang == "ger")) { m_lang1 = "de_DE"; m_lang2 = "de_AT"; } if (lang == "ruseng") { m_lang1 = "ru"; m_lang2 = "en"; } aspell_config_replace(spell_config1, "lang", m_lang1.toAscii()); aspell_config_replace(spell_config2, "lang", m_lang2.toAscii()); AspellCanHaveError *possible_err = new_aspell_speller(spell_config1); spell_checker1 = 0; if (aspell_error_number(possible_err) == 0) spell_checker1 = to_aspell_speller(possible_err); else delete_aspell_can_have_error(possible_err); possible_err = new_aspell_speller(spell_config2); spell_checker2 = 0; if (aspell_error_number(possible_err) == 0) spell_checker2 = to_aspell_speller(possible_err); else delete_aspell_can_have_error(possible_err); // Check absent dictionary if (spell_checker1 == 0) bad_language = m_lang1; if (spell_checker2 == 0) bad_language = m_lang2; }
void closeAspell() { delete_aspell_config(spell_config); delete_aspell_speller(spell_checker); }
// runs in O(t^2) time where t is the number of tokens in the input corpus // We consider maxK to be fairly constant void rawr::compile(int maxK) { _maxK = maxK; std::vector<std::vector<token_id>> tokens; std::set<std::string> thashtags; std::set<std::string> fv_emoticons; std::ifstream fvefile("emoticons.txt"); if (fvefile) { std::string line; while (getline(fvefile, line)) { fv_emoticons.insert(line); emoticons.forms.add(line); } } fvefile.close(); std::map<std::string, std::string> canonical_form; AspellConfig* spell_config = new_aspell_config(); AspellCanHaveError* possible_err = new_aspell_speller(spell_config); if (aspell_error_number(possible_err) != 0) { std::cout << "aspell error: " << aspell_error_message(possible_err) << std::endl; exit(1); } AspellSpeller* spell_checker = to_aspell_speller(possible_err); std::cout << "Reading emojis..." << std::endl; prefix_search emojis; std::ifstream emoji_file("emojis.txt"); if (emoji_file) { while (!emoji_file.eof()) { std::string rawmojis; getline(emoji_file, rawmojis); if (rawmojis.back() == '\r') { rawmojis.pop_back(); } emojis.add(rawmojis); } emoji_file.close(); } std::cout << "Tokenizing corpus... 0%" << std::flush; int len = 0; for (auto c : _corpora) { len += c.length(); } int startper = 0; int per = 0; int perprime = 0; std::cout.fill(' '); for (int i = 0; i < _corpora.size(); i++) { size_t start = 0; int end = 0; std::vector<token_id> tkcor; while (end != std::string::npos) { perprime = (startper + end) * 100 / len; if (perprime != per) { per = perprime; std::cout << "\b\b\b\b" << std::right; std::cout.width(3); std::cout << per << "%" << std::flush; } end = _corpora[i].find_first_of(" \n", start); bool emoji = false; std::string te = _corpora[i].substr(start, (end == std::string::npos) ? std::string::npos : end - start + 1); std::string t = ""; if (te.compare("") && te.compare(".") && te.compare(" ")) { if (te.back() == ' ') { te.pop_back(); } // Extract strings of emojis into their own tokens even if they're not space delimited int m = emojis.match(te); emoji = m > 0; if (m == 0) m = 1; t = te.substr(0,m); te = te.substr(m); while (!te.empty()) { m = emojis.match(te); if (emoji == (m > 0)) { if (m == 0) m = 1; t += te.substr(0,m); te = te.substr(m); } else { end = start + t.length() - 1; break; } } std::string tc(t); std::transform(tc.begin(), tc.end(), tc.begin(), ::tolower); int pst = tc.find_first_not_of("\"([*"); int dst = tc.find_last_not_of("\")]*.,?!\n;:"); std::string canonical(""); if ((pst != std::string::npos) && (dst != std::string::npos)) { canonical = std::string(tc, pst, dst - pst + 1); } word& w = ([&] () -> word& { // Hashtag freevar if (canonical[0] == '#') { thashtags.insert(canonical); return hashtags; } // Emoticon freevar if (emoji) { emoticons.forms.add(canonical); return emoticons; } if ((pst != std::string::npos) && (dst != std::string::npos)) { std::string emoticon_canon(t, pst, t.find_last_not_of("\"]*\n.,?!") - pst + 1); if (fv_emoticons.count(emoticon_canon) == 1) { emoticons.forms.add(emoticon_canon); return emoticons; } } // Basically any other word if (canonical_form.count(canonical) == 0) { if ( // Legacy freevars should be distinct from tokens containing similar words (canonical.find("$name$") != std::string::npos) // Words with no letters will be mangled by the spell checker || (canonical.find_first_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz") == std::string::npos) ) { canonical_form[canonical] = canonical; words.emplace(canonical, canonical); } else { int correct = aspell_speller_check(spell_checker, canonical.c_str(), canonical.size()); if (correct) { words.emplace(canonical, canonical); canonical_form[canonical] = canonical; } else { const AspellWordList* suggestions = aspell_speller_suggest(spell_checker, canonical.c_str(), canonical.size()); AspellStringEnumeration* elements = aspell_word_list_elements(suggestions); const char* replacement = aspell_string_enumeration_next(elements); if (replacement != NULL) { std::string sugrep(replacement); canonical_form[canonical] = sugrep; if (words.count(sugrep) == 0) { words.emplace(sugrep, sugrep); } } else { words.emplace(canonical, canonical); canonical_form[canonical] = canonical; } delete_aspell_string_enumeration(elements); } } } word& tw = words.at(canonical_form.at(canonical)); tw.forms.add(canonical); return tw; })(); token tk(w); tk.raw = t; for (char c : t) { if (c == '*') { tk.delimiters[{parentype::asterisk, doublestatus::opening}]++; } else if (c == '[') { tk.delimiters[{parentype::square_bracket, doublestatus::opening}]++; } else if (c == '(') { tk.delimiters[{parentype::paren, doublestatus::opening}]++; } else if (c == '"') { tk.delimiters[{parentype::quote, doublestatus::opening}]++; } else { break; } } int backtrack = t.find_last_not_of(".,?!])\"*\n;:") + 1; if (backtrack != t.length()) { std::string ending = t.substr(backtrack); std::string suffix; bool newline = false; bool terminating = false; for (char c : ending) { if ((c == '.') || (c == ',') || (c == '?') || (c == '!') || (c == ';') || (c == ':')) { suffix += c; terminating = true; continue; } else if (c == '\n') { newline = true; terminating = true; continue; } parentype pt = ([&] { switch (c) { case ']': return parentype::square_bracket; case ')': return parentype::paren; case '*': return parentype::asterisk; case '"': return parentype::quote; } })(); if (tk.delimiters[{pt, doublestatus::opening}] > 0) { tk.delimiters[{pt, doublestatus::opening}]--; tk.delimiters[{pt, doublestatus::both}]++; } else { tk.delimiters[{pt, doublestatus::closing}]++; } } if (terminating) { if ((suffix == ",") && (!newline)) { tk.suffix = suffixtype::comma; } else { tk.suffix = suffixtype::terminating; if (!newline) { w.terms.add({suffix, false}); } else { w.terms.add({".", false}); } } } } tkcor.push_back(_tokenstore.add(tk)); } start = ((end > (std::string::npos - 1) ) ? std::string::npos : end + 1); } tokens.push_back(tkcor); startper += _corpora[i].length(); } std::cout << "\b\b\b\b100%" << std::endl; delete_aspell_speller(spell_checker); delete_aspell_config(spell_config); std::cout << canonical_form.size() << " distinct forms" << std::endl; std::cout << words.size() << " distinct words" << std::endl; // Time to condense the distribution stuff for the words std::cout << "Compiling token histograms..." << std::endl; for (auto& it : words) { it.second.forms.compile(); it.second.terms.compile(); } // Hashtag freevar is not frequency distributed for (auto& it : thashtags) { hashtags.forms.add(it); } hashtags.forms.compile(); hashtags.terms.compile(); // Compile other freevars emoticons.forms.compile(); emoticons.terms.compile(); // Compile the interned tokens. _tokenstore.compile(); // kgram distribution std::cout << "Creating markov chain... 0%" << std::flush; std::map<kgram, std::map<token_id, token_data> > tstats; len = 0; for (auto c : tokens) { len += (maxK-1) * c.size(); } startper = 0; per = 0; perprime = 0; int corpid = 0; for (auto corpus : tokens) { for (int k=0; k<maxK && k<corpus.size(); k++) { // The zero'th token should be a terminator. token_id fid = corpus[k]; const token& f = _tokenstore.get(fid); kgram term_prefix(corpus.begin(), corpus.begin()+k); term_prefix.push_front(wildcardQuery); if (tstats[term_prefix].count(fid) == 0) { tstats[term_prefix].emplace(fid, fid); } token_data& td2 = tstats[term_prefix].at(fid); td2.all++; td2.corpora.insert(corpid); if (std::find_if(f.raw.begin(), f.raw.end(), ::islower) == f.raw.end()) { td2.uppercase++; } else if (isupper(f.raw[0])) { td2.titlecase++; } } for (int k=1; k<maxK && k<corpus.size(); k++) { for (int i=0; i<(corpus.size() - k); i++) { perprime = (startper+i) * 100 / len; if (perprime != per) { per = perprime; std::cout << "\b\b\b\b" << std::right; std::cout.width(3); std::cout << per << "%" << std::flush; } kgram prefix(corpus.begin()+i, corpus.begin()+i+k); token_id fid = corpus[i+k]; const token& f = _tokenstore.get(fid); if (tstats[prefix].count(fid) == 0) { tstats[prefix].emplace(fid, fid); } token_data& td = tstats[prefix].at(fid); td.all++; td.corpora.insert(corpid); if (std::find_if(f.raw.begin(), f.raw.end(), ::islower) == f.raw.end()) { td.uppercase++; } else if (isupper(f.raw[0])) { td.titlecase++; } const token& startTok = _tokenstore.get(std::begin(prefix)->tok); if (startTok.suffix == suffixtype::terminating) { kgram term_prefix(prefix); term_prefix.pop_front(); term_prefix.push_front(wildcardQuery); if (tstats[term_prefix].count(fid) == 0) { tstats[term_prefix].emplace(fid, fid); } token_data& td2 = tstats[term_prefix].at(fid); td2.all++; td2.corpora.insert(corpid); if (std::find_if(f.raw.begin(), f.raw.end(), ::islower) == f.raw.end()) { td2.uppercase++; } else if (isupper(f.raw[0])) { td2.titlecase++; } } } startper += corpus.size(); } corpid++; } std::cout << "\b\b\b\b100%" << std::endl; // Condense the kgram distribution std::cout << "Compiling kgram distributions... 0%"; len = tstats.size(); per = 0; perprime = 0; int indicator = 0; for (auto& it : tstats) { indicator++; perprime = indicator * 100 / len; if (per != perprime) { per = perprime; std::cout << "\b\b\b\b" << std::right; std::cout.width(3); std::cout << per << "%" << std::flush; } kgram klist = it.first; auto& probtable = it.second; auto& distribution = _stats[klist]; int max = 0; for (auto& kt : probtable) { max += kt.second.all; distribution.emplace(max, kt.second); } } std::cout << "\b\b\b\b100%" << std::endl; _compiled = true; }
KAspellChecker::~KAspellChecker() { if (m_speller) delete_aspell_speller(m_speller); }
/* Delete speller *************************************************************/ static void speller_dealloc(PyObject* self) { delete_aspell_speller( Speller(self) ); PyObject_Del(self); }
/** * This method is called from the garbage collector during finalization. * @param p pointer to spellchecker-object. */ static void aspell_free(void *p) { delete_aspell_speller(p); }
int main(int argc, const char *argv[]) { AspellCanHaveError * ret; AspellSpeller * speller; int have; char word[81]; char * p; char * word_end; AspellConfig * config; if (argc < 2) { printf("Usage: %s <language> [<size>|- [[<jargon>|- [<encoding>]]]\n", argv[0]); return 1; } config = new_aspell_config(); aspell_config_replace(config, "lang", argv[1]); if (argc >= 3 && argv[2][0] != '-' && argv[2][1] != '\0') aspell_config_replace(config, "size", argv[2]); if (argc >= 4 && argv[3][0] != '-') aspell_config_replace(config, "jargon", argv[3]); if (argc >= 5 && argv[4][0] != '-') aspell_config_replace(config, "encoding", argv[4]); ret = new_aspell_speller(config); delete_aspell_config(config); if (aspell_error(ret) != 0) { printf("Error: %s\n",aspell_error_message(ret)); delete_aspell_can_have_error(ret); return 2; } speller = to_aspell_speller(ret); config = aspell_speller_config(speller); fputs("Using: ", stdout); fputs(aspell_config_retrieve(config, "lang"), stdout); fputs("-", stdout); fputs(aspell_config_retrieve(config, "jargon"), stdout); fputs("-", stdout); fputs(aspell_config_retrieve(config, "size"), stdout); fputs("-", stdout); fputs(aspell_config_retrieve(config, "module"), stdout); fputs("\n\n", stdout); puts("Type \"h\" for help.\n"); while (fgets(word, 80, stdin) != 0) { /* remove trailing spaces */ word_end = strchr(word, '\0') - 1; while (word_end != word && (*word_end == '\n' || *word_end == ' ')) --word_end; ++word_end; *word_end = '\0'; putchar('\n'); switch (word[0]) { case '\0': break; case 'h': puts( "Usage: \n" " h(elp) help\n" " c <word> check if a word is the correct spelling\n" " s <word> print out a list of suggestions for a word\n" " a <word> add a word to the personal word list\n" " i <word> ignore a word for the rest of the session\n" " d <file> spell checks a document\n" " p dumps the personal word list\n" " P dumps the session word list\n" " m dumps the main word list\n" " o <option> <value> sets a config option\n" " r <option> retrieves a config option\n" " l <option> retrieves a config option as a list\n" " S saves all word lists\n" " C clear the curent sesstion word list\n" " x quite\n" ); break; case 'p': print_word_list(speller, aspell_speller_personal_word_list(speller), '\n'); break; case 'P': print_word_list(speller, aspell_speller_session_word_list(speller), '\n'); break; case 'm': print_word_list(speller, aspell_speller_main_word_list(speller), '\n'); break; case 'S': aspell_speller_save_all_word_lists(speller); check_for_error(speller); break; case 'C': aspell_speller_clear_session(speller); check_for_error(speller); break; case 'x': goto END; case 'c': if (strlen(word) < 3) { printf("Usage: %c <word>\n", word[0]); } else { have = aspell_speller_check(speller, word + 2, -1); if (have == 1) puts("correct"); else if (have == 0) puts("incorrect"); else printf("Error: %s\n", aspell_speller_error_message(speller)); } break; case 's': if (strlen(word) < 3) { printf("Usage: %c <word>\n", word[0]); } else { print_word_list(speller, aspell_speller_suggest(speller, word + 2, -1), '\n'); } break; case 'a': if (strlen(word) < 3) { printf("Usage: %c <word>\n", word[0]); } else { aspell_speller_add_to_personal(speller, word + 2, -1); check_for_error(speller); } break; case 'i': if (strlen(word) < 3) { printf("Usage: %c <word>\n", word[0]); } else { aspell_speller_add_to_session(speller, word + 2, -1); check_for_error(speller); } break; case 'o': word[80] = '\0'; /* to make sure strchr doesn't run off end of string */ p = strchr(word + 3, ' '); if (strlen(word) < 3 || p == 0) { printf("Usage: %c <option> <value>\n", word[0]); } else { *p = '\0'; ++p; aspell_config_replace(config, word + 2, p); check_for_config_error(config); } break; case 'r': if (strlen(word) < 3) { printf("Usage: %c <option>\n", word[0]); } else { const char * val = aspell_config_retrieve(config, word + 2); check_for_config_error(config); if (val) printf("%s = \"%s\"\n", word + 2, val); } break; case 'l': if (strlen(word) < 3) { printf("Usage: %c <option>\n", word[0]); } else { AspellStringList * lst = new_aspell_string_list(); AspellMutableContainer * lst0 = aspell_string_list_to_mutable_container(lst); AspellStringEnumeration * els; const char * val; aspell_config_retrieve_list(config, word + 2, lst0); check_for_config_error(config); els = aspell_string_list_elements(lst); printf("%s:\n", word + 2); while ( (val = aspell_string_enumeration_next(els)) != 0) printf(" %s\n", val); delete_aspell_string_enumeration(els); delete_aspell_string_list(lst); } break; case 'd': if (strlen(word) < 3) { printf("Usage: %c <file>\n", word[0]); } else { check_document(speller, word + 2); printf("\n"); } break; default: printf("Unknown Command: %s\n", word); } putchar('\n'); } END: delete_aspell_speller(speller); return 0; }