LexicalReorderingTableCreator::LexicalReorderingTableCreator(
std::string inPath, std::string outPath, std::string tempfilePath,
size_t orderBits, size_t fingerPrintBits, bool multipleScoreTrees,
size_t quantize
#ifdef WITH_THREADS
, size_t threads
#endif
)
: m_inPath(inPath), m_outPath(outPath), m_tempfilePath(tempfilePath),
m_orderBits(orderBits), m_fingerPrintBits(fingerPrintBits),
m_numScoreComponent(0), m_multipleScoreTrees(multipleScoreTrees),
m_quantize(quantize), m_separator(" ||| "),
m_hash(m_orderBits, m_fingerPrintBits), m_lastFlushedLine(-1)
#ifdef WITH_THREADS
, m_threads(threads)
#endif
{
PrintInfo();
m_outFile = std::fopen(m_outPath.c_str(), "w");
std::cerr << "Pass 1/2: Creating phrase index + Counting scores" << std::endl;
m_hash.BeginSave(m_outFile);
if(tempfilePath.size()) {
MmapAllocator<unsigned char> allocEncoded(util::FMakeTemp(tempfilePath));
m_encodedScores = new StringVector<unsigned char, unsigned long, MmapAllocator>(allocEncoded);
}
else {
m_encodedScores = new StringVector<unsigned char, unsigned long, MmapAllocator>();
}
EncodeScores();
std::cerr << "Intermezzo: Calculating Huffman code sets" << std::endl;
CalcHuffmanCodes();
std::cerr << "Pass 2/2: Compressing scores" << std::endl;
if(tempfilePath.size()) {
MmapAllocator<unsigned char> allocCompressed(util::FMakeTemp(tempfilePath));
m_compressedScores = new StringVector<unsigned char, unsigned long, MmapAllocator>(allocCompressed);
}
else {
m_compressedScores = new StringVector<unsigned char, unsigned long, MmapAllocator>();
}
CompressScores();
std::cerr << "Saving to " << m_outPath << std::endl;
Save();
std::cerr << "Done" << std::endl;
std::fclose(m_outFile);
}
LexicalReorderingTableCreator::LexicalReorderingTableCreator(
std::string inPath, std::string outPath,
size_t orderBits, size_t fingerPrintBits, bool multipleScoreTrees,
size_t quantize
#ifdef WITH_THREADS
, size_t threads
#endif
)
: m_inPath(inPath), m_outPath(outPath), m_orderBits(orderBits),
m_fingerPrintBits(fingerPrintBits), m_numScoreComponent(0),
m_multipleScoreTrees(multipleScoreTrees), m_quantize(quantize),
m_separator(" ||| "), m_hash(m_orderBits, m_fingerPrintBits),
m_lastFlushedLine(-1)
#ifdef WITH_THREADS
, m_threads(threads)
#endif
{
PrintInfo();
m_outFile = std::fopen(m_outPath.c_str(), "w");
std::cerr << "Pass 1/2: Creating phrase index + Counting scores" << std::endl;
m_hash.BeginSave(m_outFile);
EncodeScores();
std::cerr << "Intermezzo: Calculating Huffman code sets" << std::endl;
CalcHuffmanCodes();
std::cerr << "Pass 2/2: Compressing scores" << std::endl;
CompressScores();
std::cerr << "Saving to " << m_outPath << std::endl;
Save();
std::cerr << "Done" << std::endl;
std::fclose(m_outFile);
}