void append(const std::string& word)
{
std::string lower;
cybozu::ToLower(lower, word);
std::pair<Str2Int::iterator, bool> ret = word2id_.insert(Str2Int::value_type(lower, (int)id2word_.size()));
//printf("word=%s, id=%d, ret=%d\n", ret.first->first.c_str(), ret.first->second, ret.second);
if (ret.second) {
id2word_.push_back(lower);
df_.resize(id2word_.size());
}
if (set_.insert(word).second) {
df_[ret.first->second]++;
}
}
bool loadKeywordFile(const std::string& keyFile)
{
std::ifstream ifs(keyFile.c_str(), std::ios::binary);
if (!ifs) return false;
std::string word;
while (std::getline(ifs, word)) {
size_t pos = word.find('\t');
if (pos == std::string::npos) break;
word.resize(pos);
std::pair<Str2Int::iterator, bool> ret = word2id_.insert(Str2Int::value_type(word, (int)id2word_.size()));
if (ret.second) {
id2word_.push_back(word);
} else {
fprintf(stderr, "ERR already set %s\n", word.c_str());
}
}
df_.resize(id2word_.size());
fprintf(stderr, "#word = %d\n", (int)df_.size());
return true;
}
