int main(int argc, char ** argv) {
if(argc!=4) {
cout << "Checks the array class generating <length> elements between 0 and <maxv> using <seed> as seed for the numbers generation" << endl << endl;
cout << "usage: " << argv[0] << " <seed> <length> <maxv>" << endl;
return 0;
}
srand(transform(string(argv[1])));
uint len = transform(string(argv[2]));
uint maxv = transform(string(argv[3]));
//cout << "maxv = " << maxv << endl;
//cout << "len = " << len << endl;
Array a(len,maxv);
for(uint i=0;i<len;i++) {
a.setField(i,rand()%maxv);
cout << "s[" << i << "]=" << a[i] << endl;
}
HuffmanCoder * hc = new HuffmanCoder(a);
cout << "Max length: " << hc->maxLength() << endl;
for(size_t i=0;i<len;i++) {
size_t ptr = 0;
uint stream[10];
cout << "coding " << a[i] << ": ";
ptr = hc->encode(a[i],stream,ptr);
for(size_t k=0;k<ptr;k++)
cout << bitget(stream,k);
cout << endl;
}
return 0;
}
int main() {
std::ifstream file("text.txt");
std::string message, line;
while (std::getline(file, line))
message += line;
HuffmanCoder hf;
const auto& a = hf.encodeMessage(message);
const auto& b = hf.decodeMessage(a);
std::cout << "Original Message: " << message << "\n\nEncoded message: " << a << "\n\nDecoded Message: " << b;
}
int _tmain(int argc, _TCHAR* argv[])
{
HuffmanCoder H;
//string lorem = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
//string lorem = "Lorem ipsum dolor sit amet, \
consectetur adipiscing elit. \
Videamus animi partes, \
quarum est conspectus illustrior; \
Quae cum essent dicta, \
finem fecimus et ambulandi et disputandi. \
Ad corpus diceres pertinere, \
sed ea, quae dixi, ad corpusne refers? \
Duo Reges: constructio interrete. \
Expectoque quid ad id, \
quod quaerebam, respondeas. \
Nunc ita separantur, ut disiuncta sint, \
quo nihil potest esse perversius. \
At miser, si in flagitiosa et \
vitiosa vita afflueret voluptatibus.";
string lorem;
ifstream myfile("C:\\Users\\Sam\\Desktop\\Compressor\\Genesis.txt");
for(int i = 0; myfile && i<16000; i++)
{
lorem += myfile.get();
}
myfile.close();
cout << "read" << endl;
string coded = H.encode(lorem);
cout << lorem.length()*8 << "->" << coded.length() << endl;
PAUSE
return 0;
}
///////////////////////////////////////////////////////////////////////////////
// Process the records for a terminal node.
// Return value:
// bool true on success, false on error or abort
///////////////////////////////////////////////////////////////////////////////
void GeoLoadStreetIntersectionSoundex::Process()
{
numberOfOutputRecords = 0;
// Get the first record from upstream first, to allow complex processing
// to complete before opening the output file.
bool inputOK = m_readCSV.ReadRecord();
if (!inputOK)
throw TsString("No records to read");
// Frequency-counting tables.
FreqTable<int> stateFreqTable;
FreqTable<int> soundex1FreqTable;
FreqTable<int> streetNameID1FreqTable;
FreqTable<int> streetSegmentOffset1FreqTable;
FreqTable<int> soundex2FreqTable;
FreqTable<int> streetNameID2FreqTable;
FreqTable<int> streetSegmentOffset2FreqTable;
// Some variables used in processing
int tmpInt;
unsigned int tmpUInt;
int prevState;
int prevSoundex1;
int prevStreetNameID1;
int prevSoundex2;
int prevStreetNameID2;
// Get local variable equivalents of bound fields.
FieldAccessor stateValue = m_mapFieldAccessors["STATE"];
FieldAccessor soundex1Value = m_mapFieldAccessors["SOUNDEX1"];
FieldAccessor streetNameID1Value = m_mapFieldAccessors["STREET_NAME_ID1"];
FieldAccessor streetSegmentOffset1Value = m_mapFieldAccessors["STREET_SEGMENT_OFFSET1"];
FieldAccessor soundex2Value = m_mapFieldAccessors["SOUNDEX2"];
FieldAccessor streetNameID2Value = m_mapFieldAccessors["STREET_NAME_ID2"];
FieldAccessor streetSegmentOffset2Value = m_mapFieldAccessors["STREET_SEGMENT_OFFSET2"];
// Read all records and analyze frequency counts for Huffman coding.
// Buffer the records to a temp file so we can read them back later.
{
int chunkCount = 0;
do {
unsigned char tmpBuf[10];
int varIntLength;
// State
tmpInt = stateValue.GetAsInt();
if (chunkCount == 0) {
// no freq count
} else {
int diff = tmpInt - prevState;
varIntLength = IntToVarLengthBuf(diff, tmpBuf);
AddArrayToFreqTable(stateFreqTable, tmpBuf, varIntLength);
}
// Soundex1
tmpInt = PackSoundex(soundex1Value.GetAsString().c_str());
if (chunkCount == 0) {
// no freq count
} else {
int diff = tmpInt - prevSoundex1;
varIntLength = IntToVarLengthBuf(diff, tmpBuf);
AddArrayToFreqTable(soundex1FreqTable, tmpBuf, varIntLength);
}
// StreetNameID1
tmpInt = streetNameID1Value.GetAsInt();
if (chunkCount == 0) {
// no freq count
} else {
int diff = tmpInt - prevStreetNameID1;
varIntLength = IntToVarLengthBuf(diff, tmpBuf);
AddArrayToFreqTable(streetNameID1FreqTable, tmpBuf, varIntLength);
}
// StreetSegmentOffset1
tmpUInt = streetSegmentOffset1Value.GetAsInt();
varIntLength = IntToVarLengthBuf(tmpUInt, tmpBuf);
AddArrayToFreqTable(streetSegmentOffset1FreqTable, tmpBuf, varIntLength);
// Soundex2
tmpInt = PackSoundex(soundex2Value.GetAsString().c_str());
if (chunkCount == 0) {
// no freq count
} else {
int diff = tmpInt - prevSoundex2;
varIntLength = IntToVarLengthBuf(diff, tmpBuf);
AddArrayToFreqTable(soundex2FreqTable, tmpBuf, varIntLength);
}
// StreetNameID2
tmpInt = streetNameID2Value.GetAsInt();
if (chunkCount == 0) {
// no freq count
} else {
int diff = tmpInt - prevStreetNameID2;
varIntLength = IntToVarLengthBuf(diff, tmpBuf);
AddArrayToFreqTable(streetNameID2FreqTable, tmpBuf, varIntLength);
}
// StreetSegmentOffset2
tmpUInt = streetSegmentOffset2Value.GetAsInt();
varIntLength = IntToVarLengthBuf(tmpUInt, tmpBuf);
AddArrayToFreqTable(streetSegmentOffset2FreqTable, tmpBuf, varIntLength);
// Save previous values
prevState = stateValue.GetAsInt();
prevSoundex1 = PackSoundex(soundex1Value.GetAsString().c_str());
prevStreetNameID1 = streetNameID1Value.GetAsInt();
prevSoundex2 = PackSoundex(soundex2Value.GetAsString().c_str());
prevStreetNameID2 = streetNameID2Value.GetAsInt();
chunkCount++;
if (chunkCount == StreetIntersectionSoundexChunkSize) {
chunkCount = 0;
}
} while (m_readCSV.ReadRecord());
}
// Open files
File dataFile;
TsString filename = outdir + "/" + STREET_INTERSECTION_SOUNDEX_FILE;
if (!dataFile.Open(File::CreateAndWrite, filename, FileBufferSize)) {
throw TsString(
"Cannot open file " + filename + " for output"
);
}
File positionIndexFile;
filename = outdir + "/" + STREET_INTERSECTION_SOUNDEX_POSITION_INDEX_FILE;
if (!positionIndexFile.Open(File::CreateAndWrite, filename, FileBufferSize)) {
throw TsString(
"Cannot open file " + filename + " for output"
);
}
// Create an output BitStream for the data file
BitStreamWrite dataBitStream(new FileBitStreamAdaptor(dataFile));
// Create an output BitStream for the position-index file
BitStreamWrite positionIndexBitStream(new FileBitStreamAdaptor(positionIndexFile));
// Set up Huffman coding.
HuffmanCoder<int, std::less<int> > stateCoder;
HuffmanCoder<int, std::less<int> > soundex1Coder;
HuffmanCoder<int, std::less<int> > streetNameID1Coder;
HuffmanCoder<int, std::less<int> > streetSegmentOffset1Coder;
HuffmanCoder<int, std::less<int> > soundex2Coder;
HuffmanCoder<int, std::less<int> > streetNameID2Coder;
HuffmanCoder<int, std::less<int> > streetSegmentOffset2Coder;
// Populate the huffman coder tables
stateCoder.AddEntries(stateFreqTable);
soundex1Coder.AddEntries(soundex1FreqTable);
streetNameID1Coder.AddEntries(streetNameID1FreqTable);
streetSegmentOffset1Coder.AddEntries(streetSegmentOffset1FreqTable);
soundex2Coder.AddEntries(soundex2FreqTable);
streetNameID2Coder.AddEntries(streetNameID2FreqTable);
streetSegmentOffset2Coder.AddEntries(streetSegmentOffset2FreqTable);
// Generate Huffman codes
stateCoder.MakeCodes();
soundex1Coder.MakeCodes();
streetNameID1Coder.MakeCodes();
streetSegmentOffset1Coder.MakeCodes();
soundex2Coder.MakeCodes();
streetNameID2Coder.MakeCodes();
streetSegmentOffset2Coder.MakeCodes();
int chunkCount = 0;
m_readCSV.ReOpen();
m_readCSV.ReadRecord();
while (m_readCSV.ReadRecord())
{
// Write position-index records.
// Must be written before the data file, to get the correct bit offset.
if (chunkCount == 0) {
__int64 bitsWritten = dataBitStream.GetNumberOfBitsWritten();
if (bitsWritten >= ((__int64)1 << StreetIntersectionPositionIndexBitSize)) {
throw TsString("StreetIntersectionSoundex position index uses " + FormatInteger(StreetIntersectionPositionIndexBitSize) + " bits but must be larger");
}
if (StreetIntersectionPositionIndexBitSize > 32) {
throw TsString("StreetIntersectionSoundex position index uses " + FormatInteger(StreetIntersectionPositionIndexBitSize) + " bits; must recode to use __int64 variables");
}
positionIndexBitStream.WriteBitsFromInt(StreetIntersectionPositionIndexBitSize, (int)bitsWritten);
}
// Write the fields of the data record.
// state
tmpInt = stateValue.GetAsInt();
if (chunkCount == 0) {
dataBitStream.WriteBitsFromInt(StreetIntersectionStateBitSize, tmpInt);
} else {
CodeVarLengthIntToBitStream(
tmpInt - prevState,
stateCoder,
dataBitStream
);
}
// Soundex1
tmpInt = PackSoundex(soundex1Value.GetAsString().c_str());
if (chunkCount == 0) {
dataBitStream.WriteBitsFromInt(StreetIntersectionSoundexBitSize, tmpInt);
} else {
CodeVarLengthIntToBitStream(
tmpInt - prevSoundex1,
soundex1Coder,
dataBitStream
);
}
// StreetNameID1
tmpInt = streetNameID1Value.GetAsInt();
if (chunkCount == 0) {
dataBitStream.WriteBitsFromInt(StreetIntersectionStreetNameIDBitSize, tmpInt);
} else {
CodeVarLengthIntToBitStream(
tmpInt - prevStreetNameID1,
streetNameID1Coder,
dataBitStream
);
}
// StreetSegmentOffset1
tmpUInt = streetSegmentOffset1Value.GetAsInt();
CodeVarLengthIntToBitStream(
tmpUInt,
streetSegmentOffset1Coder,
dataBitStream
);
// Soundex2
tmpInt = PackSoundex(soundex2Value.GetAsString().c_str());
if (chunkCount == 0) {
dataBitStream.WriteBitsFromInt(StreetIntersectionSoundexBitSize, tmpInt);
} else {
CodeVarLengthIntToBitStream(
tmpInt - prevSoundex2,
soundex2Coder,
dataBitStream
);
}
// StreetNameID2
tmpInt = streetNameID2Value.GetAsInt();
if (chunkCount == 0) {
dataBitStream.WriteBitsFromInt(StreetIntersectionStreetNameIDBitSize, tmpInt);
} else {
CodeVarLengthIntToBitStream(
tmpInt - prevStreetNameID2,
streetNameID2Coder,
dataBitStream
);
}
// StreetSegmentOffset2
tmpUInt = streetSegmentOffset2Value.GetAsInt();
CodeVarLengthIntToBitStream(
tmpUInt,
streetSegmentOffset2Coder,
dataBitStream
);
// Save previous values
prevState = stateValue.GetAsInt();
prevSoundex1 = PackSoundex(soundex1Value.GetAsString().c_str());
prevStreetNameID1 = streetNameID1Value.GetAsInt();
prevSoundex2 = PackSoundex(soundex2Value.GetAsString().c_str());
prevStreetNameID2 = streetNameID2Value.GetAsInt();
chunkCount++;
numberOfOutputRecords++;
if (chunkCount == StreetIntersectionSoundexChunkSize) {
chunkCount = 0;
}
}
// Flush bitstream to byte-align it.
positionIndexBitStream.Flush();
// Write the total number of records at the end of the position index.
positionIndexBitStream.WriteBitsFromInt(32, numberOfOutputRecords);
positionIndexBitStream.Flush();
positionIndexFile.Close();
dataBitStream.Flush();
dataFile.Close();
// Write out the huffman code tables.
struct FreqTableFiledef {
FreqTableFiledef(
FreqTable<int>& table_,
const char* filename_
) : table(table_), filename(filename_)
{}
FreqTable<int>& table;
TsString filename;
} freqTableFiledefs[] = {
FreqTableFiledef(stateFreqTable, STREET_INTERSECTION_STATE_HUFF_FILE),
FreqTableFiledef(soundex1FreqTable, STREET_INTERSECTION_SOUNDEX1_HUFF_FILE),
FreqTableFiledef(streetNameID1FreqTable,STREET_INTERSECTION_STREET_NAME_ID1_HUFF_FILE),
FreqTableFiledef(streetSegmentOffset1FreqTable, STREET_INTERSECTION_STREET_SEGMENT_OFFSET1_HUFF_FILE),
FreqTableFiledef(soundex2FreqTable, STREET_INTERSECTION_SOUNDEX2_HUFF_FILE),
FreqTableFiledef(streetNameID2FreqTable, STREET_INTERSECTION_STREET_NAME_ID2_HUFF_FILE),
FreqTableFiledef(streetSegmentOffset2FreqTable, STREET_INTERSECTION_STREET_SEGMENT_OFFSET2_HUFF_FILE)
};
for (
int freqTableIdx = 0;
freqTableIdx < sizeof(freqTableFiledefs)/sizeof(freqTableFiledefs[0]);
freqTableIdx++
) {
std::fstream fs;
TsString filename = outdir + "/" + freqTableFiledefs[freqTableIdx].filename;
fs.open(filename.c_str(), std::ios_base::out | std::ios_base::trunc);
if (fs.fail() || !freqTableFiledefs[freqTableIdx].table.Save(fs)) {
throw TsString("Cannot write " + filename);
}
fs.close();
}
}
void EncodeBuffer::encodeCachedValue(unsigned int value,
unsigned int numBits,
PixelCache & cache,
HuffmanCoder & escapeCoder0,
HuffmanCoder & escapeCoder1)
{
unsigned int index;
DBG("encodePixelCache.\n");
if (cache.lookup(value, index))
{
encodeIndex(index);
}
else
{
encodeEscapeIndex();
// To transmit the value, use run-length coding with the static
// Huffman code implemented by the supplied "escapeCoder" object
//X encodeValue(value, numBits, numBits);
unsigned int srcMask = 0x1;
unsigned int pixelValue = ((value & srcMask) ? 1 : 0);
encodeDirect(pixelValue, 1);
for (unsigned int x = 0; x < numBits;)
{
unsigned int runStart = x;
if (pixelValue)
{
while (x < numBits)
{
if (!(value & srcMask))
break;
srcMask <<= 1;
x++;
}
}
else
{
while (x < numBits)
{
if (value & srcMask)
break;
srcMask <<= 1;
x++;
}
}
unsigned int runLength = x - runStart;
if (pixelValue)
{
escapeCoder1.encode(runLength - 1, *this);
pixelValue = 0;
}
else
{
escapeCoder0.encode(runLength - 1, *this);
pixelValue = 1;
}
}
}
}