void CThreadInfo::EncodeBlock2(const Byte *block, UInt32 blockSize, UInt32 numPasses)
{
UInt32 numCrcs = m_NumCrcs;
bool needCompare = false;
UInt32 startBytePos = m_OutStreamCurrent->GetBytePos();
UInt32 startPos = m_OutStreamCurrent->GetPos();
Byte startCurByte = m_OutStreamCurrent->GetCurByte();
Byte endCurByte = 0;
UInt32 endPos = 0;
if (numPasses > 1 && blockSize >= (1 << 10))
{
UInt32 blockSize0 = blockSize / 2;
for (;(block[blockSize0] == block[blockSize0 - 1] ||
block[blockSize0 - 1] == block[blockSize0 - 2]) &&
blockSize0 < blockSize; blockSize0++);
if (blockSize0 < blockSize)
{
EncodeBlock2(block, blockSize0, numPasses - 1);
EncodeBlock2(block + blockSize0, blockSize - blockSize0, numPasses - 1);
endPos = m_OutStreamCurrent->GetPos();
endCurByte = m_OutStreamCurrent->GetCurByte();
if ((endPos & 7) > 0)
WriteBits2(0, 8 - (endPos & 7));
m_OutStreamCurrent->SetCurState((startPos & 7), startCurByte);
needCompare = true;
}
}
UInt32 startBytePos2 = m_OutStreamCurrent->GetBytePos();
UInt32 startPos2 = m_OutStreamCurrent->GetPos();
UInt32 crcVal = EncodeBlockWithHeaders(block, blockSize);
UInt32 endPos2 = m_OutStreamCurrent->GetPos();
if (needCompare)
{
UInt32 size2 = endPos2 - startPos2;
if (size2 < endPos - startPos)
{
UInt32 numBytes = m_OutStreamCurrent->GetBytePos() - startBytePos2;
Byte *buffer = m_OutStreamCurrent->GetStream();
for (UInt32 i = 0; i < numBytes; i++)
buffer[startBytePos + i] = buffer[startBytePos2 + i];
m_OutStreamCurrent->SetPos(startPos + endPos2 - startPos2);
m_NumCrcs = numCrcs;
m_CRCs[m_NumCrcs++] = crcVal;
}
else
{
m_OutStreamCurrent->SetPos(endPos);
m_OutStreamCurrent->SetCurState((endPos & 7), endCurByte);
}
}
else
{
m_NumCrcs = numCrcs;
m_CRCs[m_NumCrcs++] = crcVal;
}
}
// blockSize > 0
void CThreadInfo::EncodeBlock(const Byte *block, UInt32 blockSize)
{
WriteBit2(false); // Randomised = false
{
UInt32 origPtr = BlockSort(m_BlockSorterIndex, block, blockSize);
// if (m_BlockSorterIndex[origPtr] != 0) throw 1;
m_BlockSorterIndex[origPtr] = blockSize;
WriteBits2(origPtr, kNumOrigBits);
}
CMtf8Encoder mtf;
int numInUse = 0;
{
bool inUse[256];
bool inUse16[16];
UInt32 i;
for (i = 0; i < 256; i++)
inUse[i] = false;
for (i = 0; i < 16; i++)
inUse16[i] = false;
for (i = 0; i < blockSize; i++)
inUse[block[i]] = true;
for (i = 0; i < 256; i++)
if (inUse[i])
{
inUse16[i >> 4] = true;
mtf.Buffer[numInUse++] = (Byte)i;
}
for (i = 0; i < 16; i++)
WriteBit2(inUse16[i]);
for (i = 0; i < 256; i++)
if (inUse16[i >> 4])
WriteBit2(inUse[i]);
}
int alphaSize = numInUse + 2;
Byte *mtfs = m_MtfArray;
UInt32 mtfArraySize = 0;
UInt32 symbolCounts[kMaxAlphaSize];
{
for (int i = 0; i < kMaxAlphaSize; i++)
symbolCounts[i] = 0;
}
{
UInt32 rleSize = 0;
UInt32 i = 0;
const UInt32 *bsIndex = m_BlockSorterIndex;
block--;
do
{
int pos = mtf.FindAndMove(block[bsIndex[i]]);
if (pos == 0)
rleSize++;
else
{
while (rleSize != 0)
{
rleSize--;
mtfs[mtfArraySize++] = (Byte)(rleSize & 1);
symbolCounts[rleSize & 1]++;
rleSize >>= 1;
}
if (pos >= 0xFE)
{
mtfs[mtfArraySize++] = 0xFF;
mtfs[mtfArraySize++] = (Byte)(pos - 0xFE);
}
else
mtfs[mtfArraySize++] = (Byte)(pos + 1);
symbolCounts[pos + 1]++;
}
}
while (++i < blockSize);
while (rleSize != 0)
{
rleSize--;
mtfs[mtfArraySize++] = (Byte)(rleSize & 1);
symbolCounts[rleSize & 1]++;
rleSize >>= 1;
}
if (alphaSize < 256)
mtfs[mtfArraySize++] = (Byte)(alphaSize - 1);
else
{
mtfs[mtfArraySize++] = 0xFF;
mtfs[mtfArraySize++] = (Byte)(alphaSize - 256);
}
symbolCounts[alphaSize - 1]++;
}
UInt32 numSymbols = 0;
{
for (int i = 0; i < kMaxAlphaSize; i++)
numSymbols += symbolCounts[i];
}
int bestNumTables = kNumTablesMin;
UInt32 bestPrice = 0xFFFFFFFF;
UInt32 startPos = m_OutStreamCurrent->GetPos();
Byte startCurByte = m_OutStreamCurrent->GetCurByte();
for (int nt = kNumTablesMin; nt <= kNumTablesMax + 1; nt++)
{
int numTables;
if(m_OptimizeNumTables)
{
m_OutStreamCurrent->SetPos(startPos);
m_OutStreamCurrent->SetCurState((startPos & 7), startCurByte);
if (nt <= kNumTablesMax)
numTables = nt;
else
numTables = bestNumTables;
}
else
{
if (numSymbols < 200) numTables = 2;
else if (numSymbols < 600) numTables = 3;
else if (numSymbols < 1200) numTables = 4;
else if (numSymbols < 2400) numTables = 5;
else numTables = 6;
}
WriteBits2(numTables, kNumTablesBits);
UInt32 numSelectors = (numSymbols + kGroupSize - 1) / kGroupSize;
WriteBits2(numSelectors, kNumSelectorsBits);
{
UInt32 remFreq = numSymbols;
int gs = 0;
int t = numTables;
do
{
UInt32 tFreq = remFreq / t;
int ge = gs;
UInt32 aFreq = 0;
while (aFreq < tFreq) // && ge < alphaSize)
aFreq += symbolCounts[ge++];
if (ge - 1 > gs && t != numTables && t != 1 && (((numTables - t) & 1) == 1))
aFreq -= symbolCounts[--ge];
Byte *lens = Lens[t - 1];
int i = 0;
do
lens[i] = (i >= gs && i < ge) ? 0 : 1;
while (++i < alphaSize);
gs = ge;
remFreq -= aFreq;
}
while(--t != 0);
}
for (int pass = 0; pass < kNumHuffPasses; pass++)
{
{
int t = 0;
do
memset(Freqs[t], 0, sizeof(Freqs[t]));
while(++t < numTables);
}
{
UInt32 mtfPos = 0;
UInt32 g = 0;
do
{
UInt32 symbols[kGroupSize];
int i = 0;
do
{
UInt32 symbol = mtfs[mtfPos++];
if (symbol >= 0xFF)
symbol += mtfs[mtfPos++];
symbols[i] = symbol;
}
while (++i < kGroupSize && mtfPos < mtfArraySize);
UInt32 bestPrice = 0xFFFFFFFF;
int t = 0;
do
{
const Byte *lens = Lens[t];
UInt32 price = 0;
int j = 0;
do
price += lens[symbols[j]];
while (++j < i);
if (price < bestPrice)
{
m_Selectors[g] = (Byte)t;
bestPrice = price;
}
}
while(++t < numTables);
UInt32 *freqs = Freqs[m_Selectors[g++]];
int j = 0;
do
freqs[symbols[j]]++;
while (++j < i);
}
while (mtfPos < mtfArraySize);
}
int t = 0;
do
{
UInt32 *freqs = Freqs[t];
int i = 0;
do
if (freqs[i] == 0)
freqs[i] = 1;
while(++i < alphaSize);
Huffman_Generate(freqs, Codes[t], Lens[t], kMaxAlphaSize, kMaxHuffmanLenForEncoding);
}
while(++t < numTables);
}
{
Byte mtfSel[kNumTablesMax];
{
int t = 0;
do
mtfSel[t] = (Byte)t;
while(++t < numTables);
}
UInt32 i = 0;
do
{
Byte sel = m_Selectors[i];
int pos;
for (pos = 0; mtfSel[pos] != sel; pos++)
WriteBit2(true);
WriteBit2(false);
for (; pos > 0; pos--)
mtfSel[pos] = mtfSel[pos - 1];
mtfSel[0] = sel;
}
while(++i < numSelectors);
}
{
int t = 0;
do
{
const Byte *lens = Lens[t];
UInt32 len = lens[0];
WriteBits2(len, kNumLevelsBits);
int i = 0;
do
{
UInt32 level = lens[i];
while (len != level)
{
WriteBit2(true);
if (len < level)
{
WriteBit2(false);
len++;
}
else
{
WriteBit2(true);
len--;
}
}
WriteBit2(false);
}
while (++i < alphaSize);
}
while(++t < numTables);
}
{
UInt32 groupSize = 0;
UInt32 groupIndex = 0;
const Byte *lens = 0;
const UInt32 *codes = 0;
UInt32 mtfPos = 0;
do
{
UInt32 symbol = mtfs[mtfPos++];
if (symbol >= 0xFF)
symbol += mtfs[mtfPos++];
if (groupSize == 0)
{
groupSize = kGroupSize;
int t = m_Selectors[groupIndex++];
lens = Lens[t];
codes = Codes[t];
}
groupSize--;
m_OutStreamCurrent->WriteBits(codes[symbol], lens[symbol]);
}
while (mtfPos < mtfArraySize);
}
if (!m_OptimizeNumTables)
break;
UInt32 price = m_OutStreamCurrent->GetPos() - startPos;
if (price <= bestPrice)
{
if (nt == kNumTablesMax)
break;
bestPrice = price;
bestNumTables = nt;
}
}
}
void CThreadInfo::WriteBit2(bool v) { WriteBits2((v ? 1 : 0), 1); }
void CThreadInfo::WriteByte2(Byte b) { WriteBits2(b , 8); }
void CThreadInfo::WriteBit2(Byte v) { WriteBits2(v, 1); }
// blockSize > 0
void CThreadInfo::EncodeBlock(Byte *block, UInt32 blockSize)
{
WriteBit2(false); // Randomised = false
{
UInt32 origPtr = m_BlockSorter.Sort(block, blockSize);
WriteBits2(origPtr, kNumOrigBits);
}
CMtf8Encoder mtf;
int numInUse = 0;
{
bool inUse[256];
bool inUse16[16];
UInt32 i;
for (i = 0; i < 256; i++)
inUse[i] = false;
for (i = 0; i < 16; i++)
inUse16[i] = false;
for (i = 0; i < blockSize; i++)
inUse[block[i]] = true;
for (i = 0; i < 256; i++)
if (inUse[i])
{
inUse16[i >> 4] = true;
mtf.Buffer[numInUse++] = (Byte)i;
}
for (i = 0; i < 16; i++)
WriteBit2(inUse16[i]);
for (i = 0; i < 256; i++)
if (inUse16[i >> 4])
WriteBit2(inUse[i]);
}
int alphaSize = numInUse + 2;
Byte *mtfs = m_MtfArray;
UInt32 mtfArraySize = 0;
UInt32 symbolCounts[kMaxAlphaSize];
{
for (int i = 0; i < kMaxAlphaSize; i++)
symbolCounts[i] = 0;
}
{
UInt32 rleSize = 0;
UInt32 i = 0;
do
{
UInt32 index = m_BlockSorter.Indices[i];
if (index == 0)
index = blockSize - 1;
else
index--;
int pos = mtf.FindAndMove(block[index]);
if (pos == 0)
rleSize++;
else
{
while (rleSize != 0)
{
rleSize--;
mtfs[mtfArraySize++] = (Byte)(rleSize & 1);
symbolCounts[rleSize & 1]++;
rleSize >>= 1;
}
if (pos >= 0xFE)
{
mtfs[mtfArraySize++] = 0xFF;
mtfs[mtfArraySize++] = (Byte)(pos - 0xFE);
}
else
mtfs[mtfArraySize++] = (Byte)(pos + 1);
symbolCounts[pos + 1]++;
}
}
while (++i < blockSize);
while (rleSize != 0)
{
rleSize--;
mtfs[mtfArraySize++] = (Byte)(rleSize & 1);
symbolCounts[rleSize & 1]++;
rleSize >>= 1;
}
if (alphaSize < 256)
mtfs[mtfArraySize++] = (Byte)(alphaSize - 1);
else
{
mtfs[mtfArraySize++] = 0xFF;
mtfs[mtfArraySize++] = (Byte)(alphaSize - 256);
}
symbolCounts[alphaSize - 1]++;
}
UInt32 numSymbols = 0;
{
for (int i = 0; i < kMaxAlphaSize; i++)
numSymbols += symbolCounts[i];
}
int bestNumTables = kNumTablesMin;
UInt32 bestPrice = 0xFFFFFFFF;
UInt32 startPos = m_OutStreamCurrent->GetPos();
Byte startCurByte = m_OutStreamCurrent->GetCurByte();
for (int nt = kNumTablesMin; nt <= kNumTablesMax + 1; nt++)
{
int numTables;
if(m_OptimizeNumTables)
{
m_OutStreamCurrent->SetPos(startPos);
m_OutStreamCurrent->SetCurState((startPos & 7), startCurByte);
if (nt <= kNumTablesMax)
numTables = nt;
else
numTables = bestNumTables;
}
else
{
if (numSymbols < 200) numTables = 2;
else if (numSymbols < 600) numTables = 3;
else if (numSymbols < 1200) numTables = 4;
else if (numSymbols < 2400) numTables = 5;
else numTables = 6;
}
WriteBits2(numTables, kNumTablesBits);
UInt32 numSelectors = (numSymbols + kGroupSize - 1) / kGroupSize;
WriteBits2(numSelectors, kNumSelectorsBits);
{
UInt32 remFreq = numSymbols;
int gs = 0;
int t = numTables;
do
{
UInt32 tFreq = remFreq / t;
int ge = gs;
UInt32 aFreq = 0;
while (aFreq < tFreq) // && ge < alphaSize)
aFreq += symbolCounts[ge++];
if (ge - 1 > gs && t != numTables && t != 1 && (((numTables - t) & 1) == 1))
aFreq -= symbolCounts[--ge];
NCompression::NHuffman::CEncoder &huffEncoder = m_HuffEncoders[t - 1];
int i = 0;
do
huffEncoder.m_Items[i].Len = (i >= gs && i < ge) ? 0 : 1;
while (++i < alphaSize);
gs = ge;
remFreq -= aFreq;
}
while(--t != 0);
}
for (int pass = 0; pass < kNumHuffPasses; pass++)
{
{
int t = 0;
do
m_HuffEncoders[t].StartNewBlock();
while(++t < numTables);
}
{
UInt32 mtfPos = 0;
UInt32 g = 0;
do
{
UInt32 symbols[kGroupSize];
int i = 0;
do
{
UInt32 symbol = mtfs[mtfPos++];
if (symbol >= 0xFF)
symbol += mtfs[mtfPos++];
symbols[i] = symbol;
}
while (++i < kGroupSize && mtfPos < mtfArraySize);
UInt32 bestPrice = 0xFFFFFFFF;
int t = 0;
do
{
NCompression::NHuffman::CItem *items = m_HuffEncoders[t].m_Items;
UInt32 price = 0;
int j = 0;
do
price += items[symbols[j]].Len;
while (++j < i);
if (price < bestPrice)
{
m_Selectors[g] = (Byte)t;
bestPrice = price;
}
}
while(++t < numTables);
NCompression::NHuffman::CEncoder &huffEncoder = m_HuffEncoders[m_Selectors[g++]];
int j = 0;
do
huffEncoder.AddSymbol(symbols[j]);
while (++j < i);
}
while (mtfPos < mtfArraySize);
}
int t = 0;
do
{
NCompression::NHuffman::CEncoder &huffEncoder = m_HuffEncoders[t];
int i = 0;
do
if (huffEncoder.m_Items[i].Freq == 0)
huffEncoder.m_Items[i].Freq = 1;
while(++i < alphaSize);
Byte levels[kMaxAlphaSize];
huffEncoder.BuildTree(levels);
}
while(++t < numTables);
}
{
Byte mtfSel[kNumTablesMax];
{
int t = 0;
do
mtfSel[t] = (Byte)t;
while(++t < numTables);
}
UInt32 i = 0;
do
{
Byte sel = m_Selectors[i];
int pos;
for (pos = 0; mtfSel[pos] != sel; pos++)
WriteBit2(true);
WriteBit2(false);
for (; pos > 0; pos--)
mtfSel[pos] = mtfSel[pos - 1];
mtfSel[0] = sel;
}
while(++i < numSelectors);
}
{
int t = 0;
do
{
NCompression::NHuffman::CItem *items = m_HuffEncoders[t].m_Items;
UInt32 len = items[0].Len;
WriteBits2(len, kNumLevelsBits);
int i = 0;
do
{
UInt32 level = items[i].Len;
while (len != level)
{
WriteBit2(true);
if (len < level)
{
WriteBit2(false);
len++;
}
else
{
WriteBit2(true);
len--;
}
}
WriteBit2(false);
}
while (++i < alphaSize);
}
while(++t < numTables);
}
{
UInt32 groupSize = 0;
UInt32 groupIndex = 0;
NCompression::NHuffman::CEncoder *huffEncoder = 0;
UInt32 mtfPos = 0;
do
{
UInt32 symbol = mtfs[mtfPos++];
if (symbol >= 0xFF)
symbol += mtfs[mtfPos++];
if (groupSize == 0)
{
groupSize = kGroupSize;
huffEncoder = &m_HuffEncoders[m_Selectors[groupIndex++]];
}
groupSize--; \
huffEncoder->CodeOneValue(m_OutStreamCurrent, symbol);
}
while (mtfPos < mtfArraySize);
}
if (!m_OptimizeNumTables)
break;
UInt32 price = m_OutStreamCurrent->GetPos() - startPos;
if (price <= bestPrice)
{
if (nt == kNumTablesMax)
break;
bestPrice = price;
bestNumTables = nt;
}
}
}