void DumpSearchTokens(string const & fPath, size_t maxTokensToShow)
{
using TValue = FeatureIndexValue;
FilesContainerR container(make_unique<FileReader>(fPath));
feature::DataHeader header(container);
serial::CodingParams codingParams(trie::GetCodingParams(header.GetDefCodingParams()));
auto const trieRoot = trie::ReadTrie<ModelReaderPtr, ValueList<TValue>>(
container.GetReader(SEARCH_INDEX_FILE_TAG), SingleValueSerializer<TValue>(codingParams));
SearchTokensCollector<TValue> f;
trie::ForEachRef(*trieRoot, f, strings::UniString());
f.Finish();
for (size_t i = 0; i < min(maxTokensToShow, f.m_tokens.size()); ++i)
{
auto const & s = f.m_tokens[i].second;
cout << f.m_tokens[i].first << " " << strings::ToUtf8(s) << endl;
}
}
void DumpSearchTokens(string const & fPath)
{
FilesContainerR container(new FileReader(fPath));
feature::DataHeader header(container);
serial::CodingParams cp(trie::GetCodingParams(header.GetDefCodingParams()));
unique_ptr<trie::DefaultIterator> const pTrieRoot(
trie::ReadTrie(container.GetReader(SEARCH_INDEX_FILE_TAG), trie::ValueReader(cp),
trie::TEdgeValueReader()));
SearchTokensCollector f;
trie::ForEachRef(*pTrieRoot, f, strings::UniString());
f.Finish();
while (!f.tokens.empty())
{
strings::UniString const & s = f.tokens.top().second;
cout << f.tokens.top().first << " '" << strings::ToUtf8(s) << "'" << endl;
f.tokens.pop();
}
}
