Ejemplo n.º 1
0
int main(int argc, char ** argv) {

  if(argc!=4) {
    cout << "Checks the array class generating <length> elements between 0 and <maxv> using <seed> as seed for the numbers generation" << endl << endl;
    cout << "usage: " << argv[0] << " <seed> <length> <maxv>" << endl;
    return 0;
  }

  srand(transform(string(argv[1])));
  uint len = transform(string(argv[2]));
  uint maxv = transform(string(argv[3]));

  //cout << "maxv = " << maxv << endl;
  //cout << "len  = " << len << endl;

  Array a(len,maxv);
  for(uint i=0;i<len;i++) {
    a.setField(i,rand()%maxv);
    cout << "s[" << i << "]=" << a[i] << endl;
  }
  
  HuffmanCoder * hc = new HuffmanCoder(a);
  cout << "Max length: " << hc->maxLength() << endl;

  for(size_t i=0;i<len;i++) {
      size_t ptr = 0;
      uint stream[10];
      cout << "coding " << a[i] << ": ";
      ptr = hc->encode(a[i],stream,ptr);
      for(size_t k=0;k<ptr;k++)
          cout << bitget(stream,k);
      cout << endl;
  }
  return 0;
}
Ejemplo n.º 2
0
int main() {
	std::ifstream file("text.txt");
	std::string message, line;
	while (std::getline(file, line)) 
		message += line;

	HuffmanCoder hf;
	const auto& a = hf.encodeMessage(message);
	const auto& b = hf.decodeMessage(a);

	std::cout << "Original Message: " << message << "\n\nEncoded message: " << a << "\n\nDecoded Message: " << b;
}
Ejemplo n.º 3
0
int _tmain(int argc, _TCHAR* argv[])
{
	HuffmanCoder H;

	//string lorem = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
				   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
	//string lorem = "Lorem ipsum dolor sit amet, \
				   consectetur adipiscing elit. \
				   Videamus animi partes, \
				   quarum est conspectus illustrior; \
				   Quae cum essent dicta, \
				   finem fecimus et ambulandi et disputandi. \
				   Ad corpus diceres pertinere, \
				   sed ea, quae dixi, ad corpusne refers? \
				   Duo Reges: constructio interrete. \
				   Expectoque quid ad id, \
				   quod quaerebam, respondeas. \
				   Nunc ita separantur, ut disiuncta sint, \
				   quo nihil potest esse perversius. \
				   At miser, si in flagitiosa et \
				   vitiosa vita afflueret voluptatibus.";
	string lorem;
	ifstream myfile("C:\\Users\\Sam\\Desktop\\Compressor\\Genesis.txt");
	for(int i = 0; myfile && i<16000; i++)
	{
		lorem += myfile.get();
	}
	myfile.close();
	cout << "read" << endl;

	string coded = H.encode(lorem);
	cout << lorem.length()*8 << "->" << coded.length() << endl;

	PAUSE
	return 0;
}
	///////////////////////////////////////////////////////////////////////////////
	// Process the records for a terminal node.
	// Return value:
	//	bool		true on success, false on error or abort
	///////////////////////////////////////////////////////////////////////////////
	void GeoLoadStreetIntersectionSoundex::Process()
	{
		numberOfOutputRecords = 0;


		// Get the first record from upstream first, to allow complex processing 
		// to complete before opening the output file.
		bool inputOK = m_readCSV.ReadRecord();

		if (!inputOK)
			throw TsString("No records to read");

		// Frequency-counting tables.
		FreqTable<int> stateFreqTable;
		FreqTable<int> soundex1FreqTable;
		FreqTable<int> streetNameID1FreqTable;
		FreqTable<int> streetSegmentOffset1FreqTable;
		FreqTable<int> soundex2FreqTable;
		FreqTable<int> streetNameID2FreqTable;
		FreqTable<int> streetSegmentOffset2FreqTable;

		// Some variables used in processing
		int tmpInt;
		unsigned int tmpUInt;

		int prevState;
		int prevSoundex1;
		int prevStreetNameID1;
		int prevSoundex2;
		int prevStreetNameID2;

		// Get local variable equivalents of bound fields.
		FieldAccessor stateValue = m_mapFieldAccessors["STATE"];
		FieldAccessor soundex1Value = m_mapFieldAccessors["SOUNDEX1"];
		FieldAccessor streetNameID1Value = m_mapFieldAccessors["STREET_NAME_ID1"];
		FieldAccessor streetSegmentOffset1Value = m_mapFieldAccessors["STREET_SEGMENT_OFFSET1"];
		FieldAccessor soundex2Value = m_mapFieldAccessors["SOUNDEX2"];
		FieldAccessor streetNameID2Value = m_mapFieldAccessors["STREET_NAME_ID2"];
		FieldAccessor streetSegmentOffset2Value = m_mapFieldAccessors["STREET_SEGMENT_OFFSET2"];

		// Read all records and analyze frequency counts for Huffman coding.
		// Buffer the records to a temp file so we can read them back later.
		{
			int chunkCount = 0;



			do {
				unsigned char tmpBuf[10];
				int varIntLength;
			
				// State
				tmpInt = stateValue.GetAsInt();
				if (chunkCount == 0) {
					// no freq count
				} else {
					int diff = tmpInt - prevState;
					varIntLength = IntToVarLengthBuf(diff, tmpBuf);
					AddArrayToFreqTable(stateFreqTable, tmpBuf, varIntLength);
				}

				// Soundex1
				tmpInt = PackSoundex(soundex1Value.GetAsString().c_str());
				if (chunkCount == 0) {
					// no freq count
				} else {
					int diff = tmpInt - prevSoundex1;
					varIntLength = IntToVarLengthBuf(diff, tmpBuf);
					AddArrayToFreqTable(soundex1FreqTable, tmpBuf, varIntLength);
				}

				// StreetNameID1
				tmpInt = streetNameID1Value.GetAsInt();
				if (chunkCount == 0) {
					// no freq count
				} else {
					int diff = tmpInt - prevStreetNameID1;
					varIntLength = IntToVarLengthBuf(diff, tmpBuf);
					AddArrayToFreqTable(streetNameID1FreqTable, tmpBuf, varIntLength);
				}

				// StreetSegmentOffset1
				tmpUInt = streetSegmentOffset1Value.GetAsInt();
				varIntLength = IntToVarLengthBuf(tmpUInt, tmpBuf);
				AddArrayToFreqTable(streetSegmentOffset1FreqTable, tmpBuf, varIntLength);

				// Soundex2
				tmpInt = PackSoundex(soundex2Value.GetAsString().c_str());
				if (chunkCount == 0) {
					// no freq count
				} else {
					int diff = tmpInt - prevSoundex2;
					varIntLength = IntToVarLengthBuf(diff, tmpBuf);
					AddArrayToFreqTable(soundex2FreqTable, tmpBuf, varIntLength);
				}

				// StreetNameID2
				tmpInt = streetNameID2Value.GetAsInt();
				if (chunkCount == 0) {
					// no freq count
				} else {
					int diff = tmpInt - prevStreetNameID2;
					varIntLength = IntToVarLengthBuf(diff, tmpBuf);
					AddArrayToFreqTable(streetNameID2FreqTable, tmpBuf, varIntLength);
				}

				// StreetSegmentOffset2
				tmpUInt = streetSegmentOffset2Value.GetAsInt();
				varIntLength = IntToVarLengthBuf(tmpUInt, tmpBuf);
				AddArrayToFreqTable(streetSegmentOffset2FreqTable, tmpBuf, varIntLength);

				// Save previous values
				prevState = stateValue.GetAsInt();
				prevSoundex1 = PackSoundex(soundex1Value.GetAsString().c_str());
				prevStreetNameID1 = streetNameID1Value.GetAsInt();
				prevSoundex2 = PackSoundex(soundex2Value.GetAsString().c_str());
				prevStreetNameID2 = streetNameID2Value.GetAsInt();

				chunkCount++;
				if (chunkCount == StreetIntersectionSoundexChunkSize) {
					chunkCount = 0;
				}
			} while (m_readCSV.ReadRecord());
		}

		// Open files
		File dataFile;
		TsString filename = outdir + "/" + STREET_INTERSECTION_SOUNDEX_FILE;
		if (!dataFile.Open(File::CreateAndWrite, filename, FileBufferSize)) {
			throw TsString(
				"Cannot open file " + filename + " for output"
			);
		}
		File positionIndexFile;
		filename = outdir + "/" + STREET_INTERSECTION_SOUNDEX_POSITION_INDEX_FILE;
		if (!positionIndexFile.Open(File::CreateAndWrite, filename, FileBufferSize)) {
			throw TsString(
				"Cannot open file " + filename + " for output"
			);
		}

		// Create an output BitStream for the data file
		BitStreamWrite dataBitStream(new FileBitStreamAdaptor(dataFile));

		// Create an output BitStream for the position-index file
		BitStreamWrite positionIndexBitStream(new FileBitStreamAdaptor(positionIndexFile));

		// Set up Huffman coding.
		HuffmanCoder<int, std::less<int> > stateCoder;
		HuffmanCoder<int, std::less<int> > soundex1Coder;
		HuffmanCoder<int, std::less<int> > streetNameID1Coder;
		HuffmanCoder<int, std::less<int> > streetSegmentOffset1Coder;
		HuffmanCoder<int, std::less<int> > soundex2Coder;
		HuffmanCoder<int, std::less<int> > streetNameID2Coder;
		HuffmanCoder<int, std::less<int> > streetSegmentOffset2Coder;

		// Populate the huffman coder tables 
		stateCoder.AddEntries(stateFreqTable);
		soundex1Coder.AddEntries(soundex1FreqTable);
		streetNameID1Coder.AddEntries(streetNameID1FreqTable);
		streetSegmentOffset1Coder.AddEntries(streetSegmentOffset1FreqTable);
		soundex2Coder.AddEntries(soundex2FreqTable);
		streetNameID2Coder.AddEntries(streetNameID2FreqTable);
		streetSegmentOffset2Coder.AddEntries(streetSegmentOffset2FreqTable);

		// Generate Huffman codes
		stateCoder.MakeCodes();
		soundex1Coder.MakeCodes();
		streetNameID1Coder.MakeCodes();
		streetSegmentOffset1Coder.MakeCodes();
		soundex2Coder.MakeCodes();
		streetNameID2Coder.MakeCodes();
		streetSegmentOffset2Coder.MakeCodes();

		int chunkCount = 0;

		m_readCSV.ReOpen();
		m_readCSV.ReadRecord();


		while (m_readCSV.ReadRecord()) 
		{
			// Write position-index records.
			// Must be written before the data file, to get the correct bit offset.
			if (chunkCount == 0) {
				__int64 bitsWritten = dataBitStream.GetNumberOfBitsWritten();

				if (bitsWritten >= ((__int64)1 << StreetIntersectionPositionIndexBitSize)) {
					throw TsString("StreetIntersectionSoundex position index uses " + FormatInteger(StreetIntersectionPositionIndexBitSize) + " bits but must be larger");
				}

				if (StreetIntersectionPositionIndexBitSize > 32) {
					throw TsString("StreetIntersectionSoundex position index uses " + FormatInteger(StreetIntersectionPositionIndexBitSize) + " bits; must recode to use __int64 variables");
				}

				positionIndexBitStream.WriteBitsFromInt(StreetIntersectionPositionIndexBitSize, (int)bitsWritten);
			}

			// Write the fields of the data record.

			// state
			tmpInt = stateValue.GetAsInt();
			if (chunkCount == 0) {
				dataBitStream.WriteBitsFromInt(StreetIntersectionStateBitSize, tmpInt);
			} else {
				CodeVarLengthIntToBitStream(
					tmpInt - prevState,
					stateCoder,
					dataBitStream
				);
			}

			// Soundex1
			tmpInt = PackSoundex(soundex1Value.GetAsString().c_str());
			if (chunkCount == 0) {
				dataBitStream.WriteBitsFromInt(StreetIntersectionSoundexBitSize, tmpInt);
			} else {
				CodeVarLengthIntToBitStream(
					tmpInt - prevSoundex1,
					soundex1Coder,
					dataBitStream
				);
			}

			// StreetNameID1
			tmpInt = streetNameID1Value.GetAsInt();
			if (chunkCount == 0) {
				dataBitStream.WriteBitsFromInt(StreetIntersectionStreetNameIDBitSize, tmpInt);
			} else {
				CodeVarLengthIntToBitStream(
					tmpInt - prevStreetNameID1,
					streetNameID1Coder,
					dataBitStream
				);
			}

			// StreetSegmentOffset1
			tmpUInt = streetSegmentOffset1Value.GetAsInt();
			CodeVarLengthIntToBitStream(
				tmpUInt,
				streetSegmentOffset1Coder,
				dataBitStream
			);

			// Soundex2
			tmpInt = PackSoundex(soundex2Value.GetAsString().c_str());
			if (chunkCount == 0) {
				dataBitStream.WriteBitsFromInt(StreetIntersectionSoundexBitSize, tmpInt);
			} else {
				CodeVarLengthIntToBitStream(
					tmpInt - prevSoundex2,
					soundex2Coder,
					dataBitStream
				);
			}

			// StreetNameID2
			tmpInt = streetNameID2Value.GetAsInt();
			if (chunkCount == 0) {
				dataBitStream.WriteBitsFromInt(StreetIntersectionStreetNameIDBitSize, tmpInt);
			} else {
				CodeVarLengthIntToBitStream(
					tmpInt - prevStreetNameID2,
					streetNameID2Coder,
					dataBitStream
				);
			}

			// StreetSegmentOffset2
			tmpUInt = streetSegmentOffset2Value.GetAsInt();
			CodeVarLengthIntToBitStream(
				tmpUInt,
				streetSegmentOffset2Coder,
				dataBitStream
			);

			// Save previous values
			prevState = stateValue.GetAsInt();
			prevSoundex1 = PackSoundex(soundex1Value.GetAsString().c_str());
			prevStreetNameID1 = streetNameID1Value.GetAsInt();
			prevSoundex2 = PackSoundex(soundex2Value.GetAsString().c_str());
			prevStreetNameID2 = streetNameID2Value.GetAsInt();

			chunkCount++;
			numberOfOutputRecords++;
			if (chunkCount == StreetIntersectionSoundexChunkSize) {
				chunkCount = 0;
			}
		}

		// Flush bitstream to byte-align it.
		positionIndexBitStream.Flush();
		// Write the total number of records at the end of the position index.
		positionIndexBitStream.WriteBitsFromInt(32, numberOfOutputRecords);
		positionIndexBitStream.Flush();
		positionIndexFile.Close();

		dataBitStream.Flush();
		dataFile.Close();

		// Write out the huffman code tables.
		struct FreqTableFiledef {
			FreqTableFiledef(
				FreqTable<int>& table_,
				const char* filename_
			) : table(table_), filename(filename_)
			{}
			FreqTable<int>& table;
			TsString filename;
		} freqTableFiledefs[] = {
			FreqTableFiledef(stateFreqTable, STREET_INTERSECTION_STATE_HUFF_FILE),
			FreqTableFiledef(soundex1FreqTable, STREET_INTERSECTION_SOUNDEX1_HUFF_FILE),
			FreqTableFiledef(streetNameID1FreqTable,STREET_INTERSECTION_STREET_NAME_ID1_HUFF_FILE),
			FreqTableFiledef(streetSegmentOffset1FreqTable, STREET_INTERSECTION_STREET_SEGMENT_OFFSET1_HUFF_FILE),
			FreqTableFiledef(soundex2FreqTable, STREET_INTERSECTION_SOUNDEX2_HUFF_FILE),
			FreqTableFiledef(streetNameID2FreqTable, STREET_INTERSECTION_STREET_NAME_ID2_HUFF_FILE),
			FreqTableFiledef(streetSegmentOffset2FreqTable, STREET_INTERSECTION_STREET_SEGMENT_OFFSET2_HUFF_FILE)
		};

		for (
			int freqTableIdx = 0; 
			freqTableIdx < sizeof(freqTableFiledefs)/sizeof(freqTableFiledefs[0]); 
			freqTableIdx++
		) {
			std::fstream fs;
			TsString filename = outdir + "/" + freqTableFiledefs[freqTableIdx].filename;
			fs.open(filename.c_str(), std::ios_base::out | std::ios_base::trunc);
			if (fs.fail() || !freqTableFiledefs[freqTableIdx].table.Save(fs)) {
				throw TsString("Cannot write " + filename);
			}
			fs.close();
		}
	}
Ejemplo n.º 5
0
void EncodeBuffer::encodeCachedValue(unsigned int value,
                                     unsigned int numBits,
                                     PixelCache & cache,
                                     HuffmanCoder & escapeCoder0,
                                     HuffmanCoder & escapeCoder1)
{
    unsigned int index;

    DBG("encodePixelCache.\n");

    if (cache.lookup(value, index))
    {
        encodeIndex(index);
    }
    else
    {
        encodeEscapeIndex();
        // To transmit the value, use run-length coding with the static
        // Huffman code implemented by the supplied "escapeCoder" object
        //X encodeValue(value, numBits, numBits);
        unsigned int srcMask = 0x1;
        unsigned int pixelValue = ((value & srcMask) ? 1 : 0);

        encodeDirect(pixelValue, 1);
        for (unsigned int x = 0; x < numBits;)
        {
            unsigned int runStart = x;

            if (pixelValue)
            {
                while (x < numBits)
                {
                    if (!(value & srcMask))
                        break;
                    srcMask <<= 1;
                    x++;
                }
            }
            else
            {
                while (x < numBits)
                {
                    if (value & srcMask)
                        break;
                    srcMask <<= 1;
                    x++;
                }
            }
            unsigned int runLength = x - runStart;

            if (pixelValue)
            {
                escapeCoder1.encode(runLength - 1, *this);
                pixelValue = 0;
            }
            else
            {
                escapeCoder0.encode(runLength - 1, *this);
                pixelValue = 1;
            }
        }
    }
}