void benchFilter(const std::vector<byte>& data) {
CM<kCMTypeMax> comp(6);
// data = randomArray(kBenchDataSize);
check(!data.empty());
const uint64_t expected_sum = std::accumulate(data.begin(), data.end(), 0UL);
std::vector<byte> out_data;
out_data.resize(20 * MB + static_cast<uint32_t>(data.size() * FilterType::getMaxExpansion() * 1.2));
uint64_t start = clock();
uint64_t write_count;
uint64_t old_sum = 0;
uint32_t best_size = std::numeric_limits<uint32_t>::max();
uint32_t best_spec;
for (uint32_t i = 0; i < kIterations; ++i) {
WriteMemoryStream wms(&out_data[0]);
ReadMemoryStream rms(&data);
FilterType f(&rms); // , 1 + (i & 3), 1 + (i / 4));
// f.setSpecific(100 + i);
comp.setOpt(i);
clock_t start = clock();
comp.compress(&f, &wms, std::numeric_limits<uint64_t>::max());
write_count = wms.tell();
f.dumpInfo();
if (write_count < best_size) {
best_size = static_cast<uint32_t>(write_count);
best_spec = i;
}
std::cout << "Cur=" << i << " size=" << write_count << " best(" << best_spec << ")=" << best_size << " time=" << clock() - start << std::endl;
#if 0
uint64_t checksum = std::accumulate(&out_data[0], &out_data[write_count], 0UL);
if (old_sum != 0) {
check(old_sum == checksum);
}
old_sum = checksum;
#endif
}
std::cout << "Forward: " << data.size() << "->" << write_count << " rate=" << prettySize(computeRate(data.size() * kIterations, clock() - start)) << "/S" << std::endl;
std::vector<byte> result;
result.resize(data.size());
start = clock();
for (uint32_t i = 0; i < kIterations; ++i) {
WriteMemoryStream wvs(&result[0]);
FilterType reverse_filter(&wvs);
comp.decompress(&ReadMemoryStream(&out_data[0], &out_data[0] + write_count), &reverse_filter, std::numeric_limits<uint64_t>::max());
reverse_filter.flush();
}
uint64_t rate = computeRate(data.size() * kIterations, clock() - start);
std::cout << "Reverse: " << prettySize(rate) << "/S" << std::endl;
// Check tht the shit matches.
check(result.size() == data.size());
for (uint32_t i = 0; i < data.size(); ++i) {
check(result[i] == data[i]);
}
}
void CCoder::readBlocks(ISequentialInStream *inStream, UInt64 *inSizeProcessed) {
size_t metadata_size = (size_t)read_int(inStream, inSizeProcessed);
size_t compressed_size = (size_t)read_int(inStream, inSizeProcessed);
std::vector<uint8_t> compressed_buffer;
compressed_buffer.assign(compressed_size, 0);
if (FAILED(ReadStream(inStream, compressed_buffer.data(), &compressed_size)))
return;
ReadMemoryStream rms_compressed(compressed_buffer.data(), compressed_buffer.data() + compressed_size);
std::unique_ptr<Compressor> c(createMetaDataCompressor());
std::vector<uint8_t> metadata;
WriteVectorStream wvs(&metadata);
c->decompress(&rms_compressed, &wvs, metadata_size);
auto cmp = read_int(inStream, inSizeProcessed);
check(cmp == 1234u);
ReadMemoryStream rms(&metadata);
blocks_.read(&rms);
}
void Archive::writeBlocks() {
std::vector<uint8_t> temp;
WriteVectorStream wvs(&temp);
// Write out the blocks into temp.
blocks_.write(&wvs);
size_t blocks_size = wvs.tell();
files_.write(&wvs);
size_t files_size = wvs.tell() - blocks_size;
// Compress overhead.
std::unique_ptr<Compressor> c(createMetaDataCompressor());
ReadMemoryStream rms(&temp[0], &temp[0] + temp.size());
auto start_pos = stream_->tell();
stream_->leb128Encode(temp.size());
c->compress(&rms, stream_);
stream_->leb128Encode(static_cast<uint64_t>(1234u));
std::cout << "(flist=" << files_size << "+" << "blocks=" << blocks_size << ")=" << temp.size() << " -> " << stream_->tell() - start_pos << std::endl << std::endl;
}
void Archive::readBlocks() {
if (!files_.empty()) {
// Already read.
return;
}
auto metadata_size = stream_->leb128Decode();
std::cout << "Metadata size=" << metadata_size << std::endl;
// Decompress overhead.
std::unique_ptr<Compressor> c(createMetaDataCompressor());
std::vector<uint8_t> metadata;
WriteVectorStream wvs(&metadata);
auto start_pos = stream_->tell();
c->decompress(stream_, &wvs, metadata_size);
auto cmp = stream_->leb128Decode();
check(cmp == 1234u);
ReadMemoryStream rms(&metadata);
blocks_.read(&rms);
files_.read(&rms);
}
void CCoder::writeBlocks(ISequentialOutStream *outStream, UInt64 *outSizeProcessed) {
std::vector<uint8_t> temp;
WriteVectorStream wvs(&temp);
blocks_.write(&wvs);
std::unique_ptr<Compressor> c(createMetaDataCompressor());
ReadMemoryStream rms(&temp[0], &temp[0] + temp.size());
std::vector<uint8_t> compressed_buffer;
WriteVectorStream wvs_compressed(&compressed_buffer);
c->compress(&rms, &wvs_compressed);
if (FAILED(NCompress::NPackjpg::write_int(outStream, temp.size(), outSizeProcessed)))
return;
if (FAILED(NCompress::NPackjpg::write_int(outStream, compressed_buffer.size(), outSizeProcessed)))
return;
if (FAILED(WriteStream(outStream, compressed_buffer.data(), compressed_buffer.size())))
return;
*outSizeProcessed += compressed_buffer.size();
NCompress::NPackjpg::write_int(outStream, 1234u, outSizeProcessed);
}
void testFilter() {
Store store_comp;
std::vector<byte> data;
uint32_t size = (rand() * 7654321 + rand()) % kDataSize;
for (uint32_t i = 0; i < size; ++i) {
data.push_back(rand() % 256);
}
std::vector<byte> out_data;
store_comp.compress(&FilterType(&ReadMemoryStream(&data)), &WriteVectorStream(&out_data), std::numeric_limits<uint64_t>::max());
std::vector<byte> result;
WriteVectorStream wvs(&result);
FilterType reverse_filter(&wvs);
store_comp.decompress(&ReadMemoryStream(&out_data), &reverse_filter, std::numeric_limits<uint64_t>::max());
reverse_filter.flush();
// Check tht the shit matches.
check(result.size() == data.size());
for (uint32_t i = 0; i < data.size(); ++i) {
byte a = result[i];
byte b = data[i];
check(a == b);
}
}
