Exemple #1
0
void CThreadInfo::EncodeBlock2(const Byte *block, UInt32 blockSize, UInt32 numPasses)
{
  UInt32 numCrcs = m_NumCrcs;
  bool needCompare = false;

  UInt32 startBytePos = m_OutStreamCurrent->GetBytePos();
  UInt32 startPos = m_OutStreamCurrent->GetPos();
  Byte startCurByte = m_OutStreamCurrent->GetCurByte();
  Byte endCurByte = 0;
  UInt32 endPos = 0;
  if (numPasses > 1 && blockSize >= (1 << 10))
  {
    UInt32 blockSize0 = blockSize / 2;
    for (;(block[blockSize0] == block[blockSize0 - 1] ||
          block[blockSize0 - 1] == block[blockSize0 - 2]) && 
          blockSize0 < blockSize; blockSize0++);
    if (blockSize0 < blockSize)
    {
      EncodeBlock2(block, blockSize0, numPasses - 1);
      EncodeBlock2(block + blockSize0, blockSize - blockSize0, numPasses - 1);
      endPos = m_OutStreamCurrent->GetPos();
      endCurByte = m_OutStreamCurrent->GetCurByte();
      if ((endPos & 7) > 0)
        WriteBits2(0, 8 - (endPos & 7));
      m_OutStreamCurrent->SetCurState((startPos & 7), startCurByte);
      needCompare = true;
    }
  }

  UInt32 startBytePos2 = m_OutStreamCurrent->GetBytePos();
  UInt32 startPos2 = m_OutStreamCurrent->GetPos();
  UInt32 crcVal = EncodeBlockWithHeaders(block, blockSize);
  UInt32 endPos2 = m_OutStreamCurrent->GetPos();

  if (needCompare)
  {
    UInt32 size2 = endPos2 - startPos2;
    if (size2 < endPos - startPos)
    { 
      UInt32 numBytes = m_OutStreamCurrent->GetBytePos() - startBytePos2;
      Byte *buffer = m_OutStreamCurrent->GetStream();
      for (UInt32 i = 0; i < numBytes; i++)
        buffer[startBytePos + i] = buffer[startBytePos2 + i];
      m_OutStreamCurrent->SetPos(startPos + endPos2 - startPos2);
      m_NumCrcs = numCrcs;
      m_CRCs[m_NumCrcs++] = crcVal;
    }
    else
    {
      m_OutStreamCurrent->SetPos(endPos);
      m_OutStreamCurrent->SetCurState((endPos & 7), endCurByte);
    }
  }
  else
  {
    m_NumCrcs = numCrcs;
    m_CRCs[m_NumCrcs++] = crcVal;
  }
}
Exemple #2
0
// blockSize > 0
void CThreadInfo::EncodeBlock(const Byte *block, UInt32 blockSize)
{
  WriteBit2(false); // Randomised = false
  
  {
    UInt32 origPtr = BlockSort(m_BlockSorterIndex, block, blockSize);
    // if (m_BlockSorterIndex[origPtr] != 0) throw 1;
    m_BlockSorterIndex[origPtr] = blockSize;
    WriteBits2(origPtr, kNumOrigBits);
  }

  CMtf8Encoder mtf;
  int numInUse = 0;
  {
    bool inUse[256];
    bool inUse16[16];
    UInt32 i;
    for (i = 0; i < 256; i++) 
      inUse[i] = false;
    for (i = 0; i < 16; i++) 
      inUse16[i] = false;
    for (i = 0; i < blockSize; i++)
      inUse[block[i]] = true;
    for (i = 0; i < 256; i++) 
      if (inUse[i])
      {
        inUse16[i >> 4] = true;
        mtf.Buffer[numInUse++] = (Byte)i;
      }
    for (i = 0; i < 16; i++) 
      WriteBit2(inUse16[i]);
    for (i = 0; i < 256; i++) 
      if (inUse16[i >> 4])
        WriteBit2(inUse[i]);
  }
  int alphaSize = numInUse + 2;

  Byte *mtfs = m_MtfArray;
  UInt32 mtfArraySize = 0;
  UInt32 symbolCounts[kMaxAlphaSize];
  {
    for (int i = 0; i < kMaxAlphaSize; i++)
      symbolCounts[i] = 0;
  }

  {
    UInt32 rleSize = 0;
    UInt32 i = 0;
    const UInt32 *bsIndex = m_BlockSorterIndex;
    block--;
    do 
    {
      int pos = mtf.FindAndMove(block[bsIndex[i]]);
      if (pos == 0)
        rleSize++;
      else
      {
        while (rleSize != 0)
        {
          rleSize--;
          mtfs[mtfArraySize++] = (Byte)(rleSize & 1);
          symbolCounts[rleSize & 1]++;
          rleSize >>= 1;
        }
        if (pos >= 0xFE)
        {
          mtfs[mtfArraySize++] = 0xFF;
          mtfs[mtfArraySize++] = (Byte)(pos - 0xFE);
        }
        else
          mtfs[mtfArraySize++] = (Byte)(pos + 1);
        symbolCounts[pos + 1]++;
      }
    }
    while (++i < blockSize);

    while (rleSize != 0)
    {
      rleSize--;
      mtfs[mtfArraySize++] = (Byte)(rleSize & 1);
      symbolCounts[rleSize & 1]++;
      rleSize >>= 1;
    }

    if (alphaSize < 256)
      mtfs[mtfArraySize++] = (Byte)(alphaSize - 1);
    else
    {
      mtfs[mtfArraySize++] = 0xFF;
      mtfs[mtfArraySize++] = (Byte)(alphaSize - 256);
    }
    symbolCounts[alphaSize - 1]++;
  }

  UInt32 numSymbols = 0;
  {
    for (int i = 0; i < kMaxAlphaSize; i++)
      numSymbols += symbolCounts[i];
  }

  int bestNumTables = kNumTablesMin;
  UInt32 bestPrice = 0xFFFFFFFF;
  UInt32 startPos = m_OutStreamCurrent->GetPos();
  Byte startCurByte = m_OutStreamCurrent->GetCurByte();
  for (int nt = kNumTablesMin; nt <= kNumTablesMax + 1; nt++)
  {
    int numTables;

    if(m_OptimizeNumTables)
    {
      m_OutStreamCurrent->SetPos(startPos);
      m_OutStreamCurrent->SetCurState((startPos & 7), startCurByte);
      if (nt <= kNumTablesMax)
        numTables = nt;
      else
        numTables = bestNumTables;
    }
    else
    {
      if (numSymbols < 200)  numTables = 2; 
      else if (numSymbols < 600) numTables = 3; 
      else if (numSymbols < 1200) numTables = 4; 
      else if (numSymbols < 2400) numTables = 5; 
      else numTables = 6;
    }

    WriteBits2(numTables, kNumTablesBits);
    
    UInt32 numSelectors = (numSymbols + kGroupSize - 1) / kGroupSize;
    WriteBits2(numSelectors, kNumSelectorsBits);
    
    {
      UInt32 remFreq = numSymbols;
      int gs = 0;
      int t = numTables;
      do
      {
        UInt32 tFreq = remFreq / t;
        int ge = gs;
        UInt32 aFreq = 0;
        while (aFreq < tFreq) //  && ge < alphaSize) 
          aFreq += symbolCounts[ge++];
        
        if (ge - 1 > gs && t != numTables && t != 1 && (((numTables - t) & 1) == 1)) 
          aFreq -= symbolCounts[--ge];
        
        Byte *lens = Lens[t - 1];
        int i = 0;
        do
          lens[i] = (i >= gs && i < ge) ? 0 : 1;
        while (++i < alphaSize);
        gs = ge;
        remFreq -= aFreq;
      }
      while(--t != 0);
    }
    
    
    for (int pass = 0; pass < kNumHuffPasses; pass++)
    {
      {
        int t = 0;
        do
          memset(Freqs[t], 0, sizeof(Freqs[t]));
        while(++t < numTables);
      }
      
      {
        UInt32 mtfPos = 0;
        UInt32 g = 0;
        do 
        {
          UInt32 symbols[kGroupSize];
          int i = 0;
          do
          {
            UInt32 symbol = mtfs[mtfPos++];
            if (symbol >= 0xFF)
              symbol += mtfs[mtfPos++];
            symbols[i] = symbol;
          }
          while (++i < kGroupSize && mtfPos < mtfArraySize);
          
          UInt32 bestPrice = 0xFFFFFFFF;
          int t = 0;
          do
          {
            const Byte *lens = Lens[t];
            UInt32 price = 0;
            int j = 0;
            do
              price += lens[symbols[j]];
            while (++j < i);
            if (price < bestPrice)
            {
              m_Selectors[g] = (Byte)t;
              bestPrice = price;
            }
          }
          while(++t < numTables);
          UInt32 *freqs = Freqs[m_Selectors[g++]];
          int j = 0;
          do
            freqs[symbols[j]]++;
          while (++j < i);
        }
        while (mtfPos < mtfArraySize);
      }
      
      int t = 0;
      do
      {
        UInt32 *freqs = Freqs[t];
        int i = 0;
        do
          if (freqs[i] == 0)
            freqs[i] = 1;
        while(++i < alphaSize);
        Huffman_Generate(freqs, Codes[t], Lens[t], kMaxAlphaSize, kMaxHuffmanLenForEncoding);
      }
      while(++t < numTables);
    }
    
    {
      Byte mtfSel[kNumTablesMax];
      {
        int t = 0;
        do
          mtfSel[t] = (Byte)t;
        while(++t < numTables);
      }
      
      UInt32 i = 0;
      do
      {
        Byte sel = m_Selectors[i];
        int pos;
        for (pos = 0; mtfSel[pos] != sel; pos++)
          WriteBit2(true);
        WriteBit2(false);
        for (; pos > 0; pos--)
          mtfSel[pos] = mtfSel[pos - 1];
        mtfSel[0] = sel;
      }
      while(++i < numSelectors);
    }
    
    {
      int t = 0;
      do
      {
        const Byte *lens = Lens[t];
        UInt32 len = lens[0];
        WriteBits2(len, kNumLevelsBits);
        int i = 0;
        do
        {
          UInt32 level = lens[i];
          while (len != level)
          {
            WriteBit2(true);
            if (len < level)
            {
              WriteBit2(false);
              len++;
            }
            else 
            {
              WriteBit2(true);
              len--;
            }
          }
          WriteBit2(false);
        }
        while (++i < alphaSize);
      }
      while(++t < numTables);
    }
    
    {
      UInt32 groupSize = 0;
      UInt32 groupIndex = 0;
      const Byte *lens = 0;
      const UInt32 *codes = 0;
      UInt32 mtfPos = 0;
      do
      {
        UInt32 symbol = mtfs[mtfPos++];
        if (symbol >= 0xFF)
          symbol += mtfs[mtfPos++];
        if (groupSize == 0) 
        {
          groupSize = kGroupSize;
          int t = m_Selectors[groupIndex++];
          lens = Lens[t];
          codes = Codes[t];
        }
        groupSize--;
        m_OutStreamCurrent->WriteBits(codes[symbol], lens[symbol]);
      }
      while (mtfPos < mtfArraySize);
    }

    if (!m_OptimizeNumTables)
      break;
    UInt32 price = m_OutStreamCurrent->GetPos() - startPos;
    if (price <= bestPrice)
    {
      if (nt == kNumTablesMax)
        break;
      bestPrice = price;
      bestNumTables = nt;
    }
  }
}
Exemple #3
0
void CThreadInfo::WriteBit2(bool v) { WriteBits2((v ? 1 : 0), 1); }
Exemple #4
0
void CThreadInfo::WriteByte2(Byte b) { WriteBits2(b , 8); }
Exemple #5
0
void CThreadInfo::WriteBit2(Byte v) { WriteBits2(v, 1); }
// blockSize > 0
void CThreadInfo::EncodeBlock(Byte *block, UInt32 blockSize)
{
  WriteBit2(false); // Randomised = false
  
  {
    UInt32 origPtr = m_BlockSorter.Sort(block, blockSize);
    WriteBits2(origPtr, kNumOrigBits);
  }

  CMtf8Encoder mtf;
  int numInUse = 0;
  {
    bool inUse[256];
    bool inUse16[16];
    UInt32 i;
    for (i = 0; i < 256; i++) 
      inUse[i] = false;
    for (i = 0; i < 16; i++) 
      inUse16[i] = false;
    for (i = 0; i < blockSize; i++)
      inUse[block[i]] = true;
    for (i = 0; i < 256; i++) 
      if (inUse[i])
      {
        inUse16[i >> 4] = true;
        mtf.Buffer[numInUse++] = (Byte)i;
      }
    for (i = 0; i < 16; i++) 
      WriteBit2(inUse16[i]);
    for (i = 0; i < 256; i++) 
      if (inUse16[i >> 4])
        WriteBit2(inUse[i]);
  }
  int alphaSize = numInUse + 2;

  Byte *mtfs = m_MtfArray;
  UInt32 mtfArraySize = 0;
  UInt32 symbolCounts[kMaxAlphaSize];
  {
    for (int i = 0; i < kMaxAlphaSize; i++)
      symbolCounts[i] = 0;
  }

  {
    UInt32 rleSize = 0;
    UInt32 i = 0;
    do 
    {
      UInt32 index = m_BlockSorter.Indices[i];
      if (index == 0)
        index = blockSize - 1;
      else
        index--;
      int pos = mtf.FindAndMove(block[index]);
      if (pos == 0)
        rleSize++;
      else
      {
        while (rleSize != 0)
        {
          rleSize--;
          mtfs[mtfArraySize++] = (Byte)(rleSize & 1);
          symbolCounts[rleSize & 1]++;
          rleSize >>= 1;
        }
        if (pos >= 0xFE)
        {
          mtfs[mtfArraySize++] = 0xFF;
          mtfs[mtfArraySize++] = (Byte)(pos - 0xFE);
        }
        else
          mtfs[mtfArraySize++] = (Byte)(pos + 1);
        symbolCounts[pos + 1]++;
      }
    }
    while (++i < blockSize);

    while (rleSize != 0)
    {
      rleSize--;
      mtfs[mtfArraySize++] = (Byte)(rleSize & 1);
      symbolCounts[rleSize & 1]++;
      rleSize >>= 1;
    }

    if (alphaSize < 256)
      mtfs[mtfArraySize++] = (Byte)(alphaSize - 1);
    else
    {
      mtfs[mtfArraySize++] = 0xFF;
      mtfs[mtfArraySize++] = (Byte)(alphaSize - 256);
    }
    symbolCounts[alphaSize - 1]++;
  }

  UInt32 numSymbols = 0;
  {
    for (int i = 0; i < kMaxAlphaSize; i++)
      numSymbols += symbolCounts[i];
  }

  int bestNumTables = kNumTablesMin;
  UInt32 bestPrice = 0xFFFFFFFF;
  UInt32 startPos = m_OutStreamCurrent->GetPos();
  Byte startCurByte = m_OutStreamCurrent->GetCurByte();
  for (int nt = kNumTablesMin; nt <= kNumTablesMax + 1; nt++)
  {
    int numTables;

    if(m_OptimizeNumTables)
    {
      m_OutStreamCurrent->SetPos(startPos);
      m_OutStreamCurrent->SetCurState((startPos & 7), startCurByte);
      if (nt <= kNumTablesMax)
        numTables = nt;
      else
        numTables = bestNumTables;
    }
    else
    {
      if (numSymbols < 200)  numTables = 2; 
      else if (numSymbols < 600) numTables = 3; 
      else if (numSymbols < 1200) numTables = 4; 
      else if (numSymbols < 2400) numTables = 5; 
      else numTables = 6;
    }

    WriteBits2(numTables, kNumTablesBits);
    
    UInt32 numSelectors = (numSymbols + kGroupSize - 1) / kGroupSize;
    WriteBits2(numSelectors, kNumSelectorsBits);
    
    {
      UInt32 remFreq = numSymbols;
      int gs = 0;
      int t = numTables;
      do
      {
        UInt32 tFreq = remFreq / t;
        int ge = gs;
        UInt32 aFreq = 0;
        while (aFreq < tFreq) //  && ge < alphaSize) 
          aFreq += symbolCounts[ge++];
        
        if (ge - 1 > gs && t != numTables && t != 1 && (((numTables - t) & 1) == 1)) 
          aFreq -= symbolCounts[--ge];
        
        NCompression::NHuffman::CEncoder &huffEncoder = m_HuffEncoders[t - 1];
        int i = 0;
        do
          huffEncoder.m_Items[i].Len = (i >= gs && i < ge) ? 0 : 1;
        while (++i < alphaSize);
        gs = ge;
        remFreq -= aFreq;
      }
      while(--t != 0);
    }
    
    
    for (int pass = 0; pass < kNumHuffPasses; pass++)
    {
      {
        int t = 0;
        do
          m_HuffEncoders[t].StartNewBlock();
        while(++t < numTables);
      }
      
      {
        UInt32 mtfPos = 0;
        UInt32 g = 0;
        do 
        {
          UInt32 symbols[kGroupSize];
          int i = 0;
          do
          {
            UInt32 symbol = mtfs[mtfPos++];
            if (symbol >= 0xFF)
              symbol += mtfs[mtfPos++];
            symbols[i] = symbol;
          }
          while (++i < kGroupSize && mtfPos < mtfArraySize);
          
          UInt32 bestPrice = 0xFFFFFFFF;
          int t = 0;
          do
          {
            NCompression::NHuffman::CItem *items = m_HuffEncoders[t].m_Items;
            UInt32 price = 0;
            int j = 0;
            do
              price += items[symbols[j]].Len;
            while (++j < i);
            if (price < bestPrice)
            {
              m_Selectors[g] = (Byte)t;
              bestPrice = price;
            }
          }
          while(++t < numTables);
          NCompression::NHuffman::CEncoder &huffEncoder = m_HuffEncoders[m_Selectors[g++]];
          int j = 0;
          do
            huffEncoder.AddSymbol(symbols[j]);
          while (++j < i);
        }
        while (mtfPos < mtfArraySize);
      }
      
      int t = 0;
      do
      {
        NCompression::NHuffman::CEncoder &huffEncoder = m_HuffEncoders[t];
        int i = 0;
        do
          if (huffEncoder.m_Items[i].Freq == 0)
            huffEncoder.m_Items[i].Freq = 1;
        while(++i < alphaSize);
        Byte levels[kMaxAlphaSize];
        huffEncoder.BuildTree(levels);
      }
      while(++t < numTables);
    }
    
    {
      Byte mtfSel[kNumTablesMax];
      {
        int t = 0;
        do
          mtfSel[t] = (Byte)t;
        while(++t < numTables);
      }
      
      UInt32 i = 0;
      do
      {
        Byte sel = m_Selectors[i];
        int pos;
        for (pos = 0; mtfSel[pos] != sel; pos++)
          WriteBit2(true);
        WriteBit2(false);
        for (; pos > 0; pos--)
          mtfSel[pos] = mtfSel[pos - 1];
        mtfSel[0] = sel;
      }
      while(++i < numSelectors);
    }
    
    {
      int t = 0;
      do
      {
        NCompression::NHuffman::CItem *items = m_HuffEncoders[t].m_Items;
        UInt32 len = items[0].Len;
        WriteBits2(len, kNumLevelsBits);
        int i = 0;
        do
        {
          UInt32 level = items[i].Len;
          while (len != level)
          {
            WriteBit2(true);
            if (len < level)
            {
              WriteBit2(false);
              len++;
            }
            else 
            {
              WriteBit2(true);
              len--;
            }
          }
          WriteBit2(false);
        }
        while (++i < alphaSize);
      }
      while(++t < numTables);
    }
    
    {
      UInt32 groupSize = 0;
      UInt32 groupIndex = 0;
      NCompression::NHuffman::CEncoder *huffEncoder = 0;
      UInt32 mtfPos = 0;
      do
      {
        UInt32 symbol = mtfs[mtfPos++];
        if (symbol >= 0xFF)
          symbol += mtfs[mtfPos++];
        if (groupSize == 0) 
        {
          groupSize = kGroupSize;
          huffEncoder = &m_HuffEncoders[m_Selectors[groupIndex++]];
        }
        groupSize--;                                    \
        huffEncoder->CodeOneValue(m_OutStreamCurrent, symbol);
      }
      while (mtfPos < mtfArraySize);
    }

    if (!m_OptimizeNumTables)
      break;
    UInt32 price = m_OutStreamCurrent->GetPos() - startPos;
    if (price <= bestPrice)
    {
      if (nt == kNumTablesMax)
        break;
      bestPrice = price;
      bestNumTables = nt;
    }
  }
}