/* =============================================================================
Function: BlockSortGenerations // local //
Author: Rammi
Date: 04/22/2002
Return: < 0 A < B
0 A == B
> 0 A > B
Parameter: A, B
Purpose: sort function for qsort, using the generations counter
for sorting, too
============================================================================= */
static int BlockSortGenerations(const begin **A, const begin **B)
{
int ret = BlockSort(A, B);
if (ret) {
return ret;
}
/* sort for generation */
return (*A)->Generation - (*B)->Generation;
}
// blockSize > 0
void CThreadInfo::EncodeBlock(const Byte *block, UInt32 blockSize)
{
WriteBit2(false); // Randomised = false
{
UInt32 origPtr = BlockSort(m_BlockSorterIndex, block, blockSize);
// if (m_BlockSorterIndex[origPtr] != 0) throw 1;
m_BlockSorterIndex[origPtr] = blockSize;
WriteBits2(origPtr, kNumOrigBits);
}
CMtf8Encoder mtf;
int numInUse = 0;
{
bool inUse[256];
bool inUse16[16];
UInt32 i;
for (i = 0; i < 256; i++)
inUse[i] = false;
for (i = 0; i < 16; i++)
inUse16[i] = false;
for (i = 0; i < blockSize; i++)
inUse[block[i]] = true;
for (i = 0; i < 256; i++)
if (inUse[i])
{
inUse16[i >> 4] = true;
mtf.Buffer[numInUse++] = (Byte)i;
}
for (i = 0; i < 16; i++)
WriteBit2(inUse16[i]);
for (i = 0; i < 256; i++)
if (inUse16[i >> 4])
WriteBit2(inUse[i]);
}
int alphaSize = numInUse + 2;
Byte *mtfs = m_MtfArray;
UInt32 mtfArraySize = 0;
UInt32 symbolCounts[kMaxAlphaSize];
{
for (int i = 0; i < kMaxAlphaSize; i++)
symbolCounts[i] = 0;
}
{
UInt32 rleSize = 0;
UInt32 i = 0;
const UInt32 *bsIndex = m_BlockSorterIndex;
block--;
do
{
int pos = mtf.FindAndMove(block[bsIndex[i]]);
if (pos == 0)
rleSize++;
else
{
while (rleSize != 0)
{
rleSize--;
mtfs[mtfArraySize++] = (Byte)(rleSize & 1);
symbolCounts[rleSize & 1]++;
rleSize >>= 1;
}
if (pos >= 0xFE)
{
mtfs[mtfArraySize++] = 0xFF;
mtfs[mtfArraySize++] = (Byte)(pos - 0xFE);
}
else
mtfs[mtfArraySize++] = (Byte)(pos + 1);
symbolCounts[pos + 1]++;
}
}
while (++i < blockSize);
while (rleSize != 0)
{
rleSize--;
mtfs[mtfArraySize++] = (Byte)(rleSize & 1);
symbolCounts[rleSize & 1]++;
rleSize >>= 1;
}
if (alphaSize < 256)
mtfs[mtfArraySize++] = (Byte)(alphaSize - 1);
else
{
mtfs[mtfArraySize++] = 0xFF;
mtfs[mtfArraySize++] = (Byte)(alphaSize - 256);
}
symbolCounts[alphaSize - 1]++;
}
UInt32 numSymbols = 0;
{
for (int i = 0; i < kMaxAlphaSize; i++)
numSymbols += symbolCounts[i];
}
int bestNumTables = kNumTablesMin;
UInt32 bestPrice = 0xFFFFFFFF;
UInt32 startPos = m_OutStreamCurrent->GetPos();
Byte startCurByte = m_OutStreamCurrent->GetCurByte();
for (int nt = kNumTablesMin; nt <= kNumTablesMax + 1; nt++)
{
int numTables;
if(m_OptimizeNumTables)
{
m_OutStreamCurrent->SetPos(startPos);
m_OutStreamCurrent->SetCurState((startPos & 7), startCurByte);
if (nt <= kNumTablesMax)
numTables = nt;
else
numTables = bestNumTables;
}
else
{
if (numSymbols < 200) numTables = 2;
else if (numSymbols < 600) numTables = 3;
else if (numSymbols < 1200) numTables = 4;
else if (numSymbols < 2400) numTables = 5;
else numTables = 6;
}
WriteBits2(numTables, kNumTablesBits);
UInt32 numSelectors = (numSymbols + kGroupSize - 1) / kGroupSize;
WriteBits2(numSelectors, kNumSelectorsBits);
{
UInt32 remFreq = numSymbols;
int gs = 0;
int t = numTables;
do
{
UInt32 tFreq = remFreq / t;
int ge = gs;
UInt32 aFreq = 0;
while (aFreq < tFreq) // && ge < alphaSize)
aFreq += symbolCounts[ge++];
if (ge - 1 > gs && t != numTables && t != 1 && (((numTables - t) & 1) == 1))
aFreq -= symbolCounts[--ge];
Byte *lens = Lens[t - 1];
int i = 0;
do
lens[i] = (i >= gs && i < ge) ? 0 : 1;
while (++i < alphaSize);
gs = ge;
remFreq -= aFreq;
}
while(--t != 0);
}
for (int pass = 0; pass < kNumHuffPasses; pass++)
{
{
int t = 0;
do
memset(Freqs[t], 0, sizeof(Freqs[t]));
while(++t < numTables);
}
{
UInt32 mtfPos = 0;
UInt32 g = 0;
do
{
UInt32 symbols[kGroupSize];
int i = 0;
do
{
UInt32 symbol = mtfs[mtfPos++];
if (symbol >= 0xFF)
symbol += mtfs[mtfPos++];
symbols[i] = symbol;
}
while (++i < kGroupSize && mtfPos < mtfArraySize);
UInt32 bestPrice = 0xFFFFFFFF;
int t = 0;
do
{
const Byte *lens = Lens[t];
UInt32 price = 0;
int j = 0;
do
price += lens[symbols[j]];
while (++j < i);
if (price < bestPrice)
{
m_Selectors[g] = (Byte)t;
bestPrice = price;
}
}
while(++t < numTables);
UInt32 *freqs = Freqs[m_Selectors[g++]];
int j = 0;
do
freqs[symbols[j]]++;
while (++j < i);
}
while (mtfPos < mtfArraySize);
}
int t = 0;
do
{
UInt32 *freqs = Freqs[t];
int i = 0;
do
if (freqs[i] == 0)
freqs[i] = 1;
while(++i < alphaSize);
Huffman_Generate(freqs, Codes[t], Lens[t], kMaxAlphaSize, kMaxHuffmanLenForEncoding);
}
while(++t < numTables);
}
{
Byte mtfSel[kNumTablesMax];
{
int t = 0;
do
mtfSel[t] = (Byte)t;
while(++t < numTables);
}
UInt32 i = 0;
do
{
Byte sel = m_Selectors[i];
int pos;
for (pos = 0; mtfSel[pos] != sel; pos++)
WriteBit2(true);
WriteBit2(false);
for (; pos > 0; pos--)
mtfSel[pos] = mtfSel[pos - 1];
mtfSel[0] = sel;
}
while(++i < numSelectors);
}
{
int t = 0;
do
{
const Byte *lens = Lens[t];
UInt32 len = lens[0];
WriteBits2(len, kNumLevelsBits);
int i = 0;
do
{
UInt32 level = lens[i];
while (len != level)
{
WriteBit2(true);
if (len < level)
{
WriteBit2(false);
len++;
}
else
{
WriteBit2(true);
len--;
}
}
WriteBit2(false);
}
while (++i < alphaSize);
}
while(++t < numTables);
}
{
UInt32 groupSize = 0;
UInt32 groupIndex = 0;
const Byte *lens = 0;
const UInt32 *codes = 0;
UInt32 mtfPos = 0;
do
{
UInt32 symbol = mtfs[mtfPos++];
if (symbol >= 0xFF)
symbol += mtfs[mtfPos++];
if (groupSize == 0)
{
groupSize = kGroupSize;
int t = m_Selectors[groupIndex++];
lens = Lens[t];
codes = Codes[t];
}
groupSize--;
m_OutStreamCurrent->WriteBits(codes[symbol], lens[symbol]);
}
while (mtfPos < mtfArraySize);
}
if (!m_OptimizeNumTables)
break;
UInt32 price = m_OutStreamCurrent->GetPos() - startPos;
if (price <= bestPrice)
{
if (nt == kNumTablesMax)
break;
bestPrice = price;
bestNumTables = nt;
}
}
}
/* =============================================================================
Function: Rmalloc_stat // extern //
Author: Rammi
Date: 04/15/1995
Return: ---
Parameter: file caled from
Purpose: Show statistic
============================================================================= */
void Rmalloc_stat(const char *file)
{
#if RM_TEST_DEPTH > 0
TestAll(file);
#define STAT_HEAD "<MALLOC_STATS>\t"
fprintf(stderr,
STAT_HEAD "============ STATISTICS (%s) =============\n", file);
if (!Global.BlockCount) {
fprintf(stderr, STAT_HEAD "Nothing allocated.\n");
}
else {
const begin **BlockVec;
if ((BlockVec = (const begin **)malloc(Global.BlockCount*sizeof(begin *))) == NULL) {
fprintf(stderr, STAT_HEAD "Couldn't allocate enough memory for statistics. Going on...\n");
}
else {
unsigned i = 0;
unsigned j;
begin *B;
unsigned count;
size_t Mem = 0;
unsigned nrBlocks;
#ifdef WITH_FLAGS
size_t StaticMem = 0;
#endif
#ifdef GENERATIONS
unsigned gen;
#endif
/* add all blocks to vector */
for (j = 0; j < HASHSIZE; j++) {
for (B = Chain[j].Next; B != &Chain[j]; B = B->Next) {
#ifdef WITH_FLAGS
if (B->Flags & RM_STATIC) {
StaticMem += B->Size;
}
else {
BlockVec[i++] = B;
}
#else
BlockVec[i++] = B;
#endif
}
}
#ifdef WITH_FLAGS
assert(i <= Global.BlockCount);
#else
assert(i == Global.BlockCount);
#endif
nrBlocks = i;
/* --- sort --- */
#ifdef GENERATIONS
qsort(BlockVec, nrBlocks,
sizeof(begin *),
(int (*)(const void *, const void *))BlockSortGenerations);
#else
qsort(BlockVec, nrBlocks,
sizeof(begin *),
(int (*)(const void *, const void *))BlockSort);
#endif
for (i = 0; i < nrBlocks; i = j) {
count = 1;
for (j = i+1; j < nrBlocks; j++) {
if (BlockSort(BlockVec+i, BlockVec+j) != 0) {
break;
}
/* are equal */
count++;
}
#ifdef GENERATIONS
fprintf(stderr,
STAT_HEAD "%6d x %8u Bytes in %s, generations:",
count,
(unsigned) BlockVec[i]->Size,
BlockVec[i]->File);
for (gen = 0; gen < count; gen++) {
if (gen == MAX_STAT_GENERATIONS) {
fprintf(stderr, " ...");
break;
}
fprintf(stderr, " %d", BlockVec[gen+i]->Generation);
}
fprintf(stderr, "\n");
#else
fprintf(stderr,
STAT_HEAD "%6d x %8u Bytes in %s\n",
count,
(unsigned) BlockVec[i]->Size,
BlockVec[i]->File);
#endif
Mem += count*BlockVec[i]->Size;
}
/* and give free */
free(BlockVec);
#ifdef WITH_FLAGS
fprintf(stderr, STAT_HEAD "*Variable*\t%12u Bytes\n",
(unsigned) Mem);
fprintf(stderr, STAT_HEAD "*Static* \t%12u Bytes\n",
(unsigned) StaticMem);
fprintf(stderr, STAT_HEAD "*Total* \t%12u Bytes\n",
(unsigned) (Mem+StaticMem));
#else
fprintf(stderr, STAT_HEAD "*Total*\t%u Bytes\n",
(unsigned) Mem);
#endif
}
}
fprintf(stderr, STAT_HEAD "============ END OF STATISTICS =============\n");
#else
fprintf(stderr, HEAD __FILE__ " not compiled with RM_TEST_DEPTH > 0, call in %s senseless.\n", file);
#endif
}