Data Gzip::Decompress(const void *data, size_t size) {
//TODO Change interface to taking cpputils::Data objects (needs changing blockstore so we can read their "class Data", because this is called from CompressedBlock::Decompress()).
CryptoPP::Gunzip zipper;
zipper.Put((byte *) data, size);
zipper.MessageEnd();
Data decompressed(zipper.MaxRetrievable());
zipper.Get((byte *) decompressed.data(), decompressed.size());
return decompressed;
}
Utils::ByteBuffer
Decompressor::decompress(char* src, unsigned long src_len)
{
Utils::ByteBuffer dst;
unsigned dst_len = 128 * 1024;
dst.setSize(dst_len);
decompress(dst.getBufferSigned(), dst.getSize(), src, src_len);
dst.setSize(decompressed());
return dst;
}
DEF_TEST(SkDeflateWStream, r) {
SkRandom random(123456);
for (int i = 0; i < 50; ++i) {
uint32_t size = random.nextULessThan(10000);
SkAutoTMalloc<uint8_t> buffer(size);
for (uint32_t j = 0; j < size; ++j) {
buffer[j] = random.nextU() & 0xff;
}
SkDynamicMemoryWStream dynamicMemoryWStream;
{
SkDeflateWStream deflateWStream(&dynamicMemoryWStream);
uint32_t j = 0;
while (j < size) {
uint32_t writeSize =
SkTMin(size - j, random.nextRangeU(1, 400));
if (!deflateWStream.write(&buffer[j], writeSize)) {
ERRORF(r, "something went wrong.");
return;
}
j += writeSize;
}
}
SkAutoTDelete<SkStreamAsset> compressed(
dynamicMemoryWStream.detachAsStream());
SkAutoTDelete<SkStreamAsset> decompressed(stream_inflate(compressed));
if (decompressed->getLength() != size) {
ERRORF(r, "Decompression failed to get right size [%d]."
" %u != %u", i, (unsigned)(decompressed->getLength()),
(unsigned)size);
SkString s = SkStringPrintf("/tmp/deftst_compressed_%d", i);
SkFILEWStream o(s.c_str());
o.writeStream(compressed.get(), compressed->getLength());
compressed->rewind();
s = SkStringPrintf("/tmp/deftst_input_%d", i);
SkFILEWStream o2(s.c_str());
o2.write(&buffer[0], size);
continue;
}
uint32_t minLength = SkTMin(size,
(uint32_t)(decompressed->getLength()));
for (uint32_t i = 0; i < minLength; ++i) {
uint8_t c;
SkDEBUGCODE(size_t rb =)decompressed->read(&c, sizeof(uint8_t));
SkASSERT(sizeof(uint8_t) == rb);
if (buffer[i] != c) {
ERRORF(r, "Decompression failed at byte %u.", (unsigned)i);
break;
}
}
}
}
std::vector<unsigned char> GZip::decompress(
const std::vector<unsigned char> &compressed)
{
::z_stream zstream;
std::memset(&zstream, 0, sizeof(zstream));
zstream.next_in = const_cast<unsigned char *>(compressed.data());
zstream.avail_in = compressed.size();
auto rc = ::inflateInit2(&zstream, WINDOW_SIZE + WINSIZE_OFFSET_GZIP);
kulloAssert(rc == Z_OK);
std::vector<unsigned char> decompressed(1 * 1024 * 1024, '\0');
bool done = false;
while (!done)
{
zstream.next_out = decompressed.data() + zstream.total_out;
zstream.avail_out = decompressed.size() - zstream.total_out;
rc = ::inflate(&zstream, Z_NO_FLUSH);
switch (rc)
{
case Z_OK:
continue;
case Z_BUF_ERROR:
if (zstream.avail_out > 0)
{
throw GZipStreamError("Z_BUF_ERROR but avail_out > 0");
}
// increase size heuristically to 150%
decompressed.resize(decompressed.size() * 3 / 2);
break;
case Z_STREAM_END:
// shrink to needed size
decompressed.resize(zstream.total_out);
done = true;
break;
default:
throw GZipStreamError(
std::string("inflate error ") + std::to_string(rc)
+ ": " + zstream.msg);
}
}
rc = ::inflateEnd(&zstream);
kulloAssert(rc == Z_OK);
return decompressed;
}
void ImageDecoder::decompress() {
if (!_compressed)
return;
for (MipMaps::iterator m = _mipMaps.begin(); m != _mipMaps.end(); ++m) {
MipMap decompressed(this);
decompress(decompressed, **m, _formatRaw);
decompressed.swap(**m);
}
_format = kPixelFormatRGBA;
_formatRaw = kPixelFormatRGBA8;
_dataType = kPixelDataType8;
_compressed = false;
}
byteArray getTileHolderDatabaseMaster(const blub::vector3int32& id) const
{
soci::indicator indicator = soci::i_null;
std::string selectData;
m_databaseConnection << "select data from voxel_tiles where x = :x and y = :y and z = :z"
, soci::into(selectData, indicator), blub::database::use(id.x), blub::database::use(id.y), blub::database::use(id.z);
if (indicator == soci::i_ok)
{
byteArray decompressed(base64::decode(selectData));
if (m_funcDecompress)
{
decompressed = m_funcDecompress(decompressed);
}
return decompressed;
}
return byteArray();
}
std::unique_ptr<std::vector<char>> Compression::decompress(
const std::vector<char>& data,
const Schema& schema,
const std::size_t numPoints)
{
const std::size_t decompressedSize(numPoints * schema.pointSize());
DecompressionStream decompressionStream(data);
pdal::LazPerfDecompressor<DecompressionStream> decompressor(
decompressionStream,
schema.pdalLayout().dimTypes());
std::unique_ptr<std::vector<char>> decompressed(
new std::vector<char>(decompressedSize));
decompressor.decompress(decompressed->data(), decompressed->size());
return decompressed;
}
std::unique_ptr<std::vector<char>> Compression::decompress(
const std::vector<char>& data,
const Schema& nativeSchema,
const Schema* const wantedSchema,
const std::size_t numPoints)
{
if (!wantedSchema || *wantedSchema == nativeSchema)
{
return decompress(data, nativeSchema, numPoints);
}
// Get decompressor in the native schema.
DecompressionStream decompressionStream(data);
pdal::LazPerfDecompressor<DecompressionStream> decompressor(
decompressionStream,
nativeSchema.pdalLayout().dimTypes());
// Allocate room for a single point in the native schema.
std::vector<char> nativePoint(nativeSchema.pointSize());
BinaryPointTable table(nativeSchema, nativePoint.data());
pdal::PointRef pointRef(table, 0);
// Get our result space, in the desired schema, ready.
std::unique_ptr<std::vector<char>> decompressed(
new std::vector<char>(numPoints * wantedSchema->pointSize(), 0));
char* pos(decompressed->data());
const char* end(pos + decompressed->size());
while (pos < end)
{
decompressor.decompress(nativePoint.data(), nativePoint.size());
for (const auto& d : wantedSchema->dims())
{
pointRef.getField(pos, d.id(), d.type());
pos += d.size();
}
}
return decompressed;
}
void
Decompressor::decompress(Utils::ByteBuffer& dst, Utils::ByteBuffer& src)
{
decompress(dst.getBufferSigned(), dst.getSize(), src.getBufferSigned(), src.getSize());
dst.setSize(decompressed());
}
///////////////////////////////////////////////////////////
//
// Returns a data handler for the waveform
//
///////////////////////////////////////////////////////////
ptr<handlers::dataHandler> waveform::getIntegerData(std::uint32_t channel, std::int32_t paddingValue)
{
PUNTOEXE_FUNCTION_START(L"waveform::getIntegerData");
static std::int32_t uLawDecompressTable[256] =
{
-32124,-31100,-30076,-29052,-28028,-27004,-25980,-24956,
-23932,-22908,-21884,-20860,-19836,-18812,-17788,-16764,
-15996,-15484,-14972,-14460,-13948,-13436,-12924,-12412,
-11900,-11388,-10876,-10364, -9852, -9340, -8828, -8316,
-7932, -7676, -7420, -7164, -6908, -6652, -6396, -6140,
-5884, -5628, -5372, -5116, -4860, -4604, -4348, -4092,
-3900, -3772, -3644, -3516, -3388, -3260, -3132, -3004,
-2876, -2748, -2620, -2492, -2364, -2236, -2108, -1980,
-1884, -1820, -1756, -1692, -1628, -1564, -1500, -1436,
-1372, -1308, -1244, -1180, -1116, -1052, -988, -924,
-876, -844, -812, -780, -748, -716, -684, -652,
-620, -588, -556, -524, -492, -460, -428, -396,
-372, -356, -340, -324, -308, -292, -276, -260,
-244, -228, -212, -196, -180, -164, -148, -132,
-120, -112, -104, -96, -88, -80, -72, -64,
-56, -48, -40, -32, -24, -16, -8, 0,
32124, 31100, 30076, 29052, 28028, 27004, 25980, 24956,
23932, 22908, 21884, 20860, 19836, 18812, 17788, 16764,
15996, 15484, 14972, 14460, 13948, 13436, 12924, 12412,
11900, 11388, 10876, 10364, 9852, 9340, 8828, 8316,
7932, 7676, 7420, 7164, 6908, 6652, 6396, 6140,
5884, 5628, 5372, 5116, 4860, 4604, 4348, 4092,
3900, 3772, 3644, 3516, 3388, 3260, 3132, 3004,
2876, 2748, 2620, 2492, 2364, 2236, 2108, 1980,
1884, 1820, 1756, 1692, 1628, 1564, 1500, 1436,
1372, 1308, 1244, 1180, 1116, 1052, 988, 924,
876, 844, 812, 780, 748, 716, 684, 652,
620, 588, 556, 524, 492, 460, 428, 396,
372, 356, 340, 324, 308, 292, 276, 260,
244, 228, 212, 196, 180, 164, 148, 132,
120, 112, 104, 96, 88, 80, 72, 64,
56, 48, 40, 32, 24, 16, 8, 0
};
static std::int32_t aLawDecompressTable[256] =
{
-5504, -5248, -6016, -5760, -4480, -4224, -4992, -4736,
-7552, -7296, -8064, -7808, -6528, -6272, -7040, -6784,
-2752, -2624, -3008, -2880, -2240, -2112, -2496, -2368,
-3776, -3648, -4032, -3904, -3264, -3136, -3520, -3392,
-22016,-20992,-24064,-23040,-17920,-16896,-19968,-18944,
-30208,-29184,-32256,-31232,-26112,-25088,-28160,-27136,
-11008,-10496,-12032,-11520,-8960, -8448, -9984, -9472,
-15104,-14592,-16128,-15616,-13056,-12544,-14080,-13568,
-344, -328, -376, -360, -280, -264, -312, -296,
-472, -456, -504, -488, -408, -392, -440, -424,
-88, -72, -120, -104, -24, -8, -56, -40,
-216, -200, -248, -232, -152, -136, -184, -168,
-1376, -1312, -1504, -1440, -1120, -1056, -1248, -1184,
-1888, -1824, -2016, -1952, -1632, -1568, -1760, -1696,
-688, -656, -752, -720, -560, -528, -624, -592,
-944, -912, -1008, -976, -816, -784, -880, -848,
5504, 5248, 6016, 5760, 4480, 4224, 4992, 4736,
7552, 7296, 8064, 7808, 6528, 6272, 7040, 6784,
2752, 2624, 3008, 2880, 2240, 2112, 2496, 2368,
3776, 3648, 4032, 3904, 3264, 3136, 3520, 3392,
22016, 20992, 24064, 23040, 17920, 16896, 19968, 18944,
30208, 29184, 32256, 31232, 26112, 25088, 28160, 27136,
11008, 10496, 12032, 11520, 8960, 8448, 9984, 9472,
15104, 14592, 16128, 15616, 13056, 12544, 14080, 13568,
344, 328, 376, 360, 280, 264, 312, 296,
472, 456, 504, 488, 408, 392, 440, 424,
88, 72, 120, 104, 24, 8, 56, 40,
216, 200, 248, 232, 152, 136, 184, 168,
1376, 1312, 1504, 1440, 1120, 1056, 1248, 1184,
1888, 1824, 2016, 1952, 1632, 1568, 1760, 1696,
688, 656, 752, 720, 560, 528, 624, 592,
944, 912, 1008, 976, 816, 784, 880, 848
};
// Lock the dataset during the interpretation of the
// dataset
///////////////////////////////////////////////////////////
lockObject lockDataSet(m_pDataSet);
// Get the original data
///////////////////////////////////////////////////////////
ptr<handlers::dataHandler> waveformData(m_pDataSet->getDataHandler(0x5400, 0x0, 0x1010, 0, false));
std::string sourceDataType(waveformData->getDataType());
// Get the interpretation, number of channels, number of
// samples
///////////////////////////////////////////////////////////
std::string waveformInterpretation(getInterpretation());
std::uint32_t numChannels(getChannels());
std::uint32_t numSamples(getSamples());
std::uint32_t originalPaddingValue(0);
bool bPaddingValueExists(false);
ptr<handlers::dataHandler> paddingTagHandler(m_pDataSet->getDataHandler(0x5400, 0, 0x100A, 0, false));
if(paddingTagHandler != 0)
{
originalPaddingValue = paddingTagHandler->getUnsignedLong(0);
bPaddingValueExists = true;
}
// Allocate a buffer for the destination data
///////////////////////////////////////////////////////////
ptr<buffer> waveformBuffer(new buffer(0, "SL"));
ptr<handlers::dataHandler> destinationHandler(waveformBuffer->getDataHandler(true, numSamples));
// Copy the data to the destination for unsigned values
///////////////////////////////////////////////////////////
std::uint32_t waveformPointer(channel);
std::uint32_t destinationPointer(0);
if(sourceDataType == "UB" || sourceDataType == "US")
{
for(std::uint32_t copySamples (numSamples); copySamples != 0; --copySamples)
{
std::uint32_t unsignedData(waveformData->getUnsignedLong(waveformPointer));
waveformPointer += numChannels;
if(bPaddingValueExists && unsignedData == originalPaddingValue)
{
destinationHandler->setSignedLong(destinationPointer++, paddingValue);
continue;
}
destinationHandler->setUnsignedLong(destinationPointer++, unsignedData);
}
return destinationHandler;
}
// Copy the data to the destination for signed values
///////////////////////////////////////////////////////////
int highBit(getBitsAllocated() - 1);
std::uint32_t testBit = ((std::uint32_t)1) << highBit;
std::uint32_t orBits = ((std::uint32_t)((std::int32_t)-1)) << highBit;
for(std::uint32_t copySamples (numSamples); copySamples != 0; --copySamples)
{
std::uint32_t unsignedData = waveformData->getUnsignedLong(waveformPointer);
waveformPointer += numChannels;
if(bPaddingValueExists && unsignedData == originalPaddingValue)
{
destinationHandler->setSignedLong(destinationPointer++, paddingValue);;
continue;
}
if((unsignedData & testBit) != 0)
{
unsignedData |= orBits;
}
destinationHandler->setSignedLong(destinationPointer++, (std::int32_t)unsignedData);
}
// Now decompress uLaw or aLaw
if(waveformInterpretation == "AB") // 8bits aLaw
{
for(std::uint32_t aLawSamples(0); aLawSamples != numSamples; ++aLawSamples)
{
std::uint32_t compressed(destinationHandler->getUnsignedLong(aLawSamples));
if(bPaddingValueExists && compressed == originalPaddingValue)
{
continue;
}
std::int32_t decompressed(aLawDecompressTable[compressed]);
destinationHandler->setSignedLong(aLawSamples, decompressed);
}
}
// Now decompress uLaw or aLaw
if(waveformInterpretation == "MB") // 8bits aLaw
{
for(std::uint32_t uLawSamples(0); uLawSamples != numSamples; ++uLawSamples)
{
std::uint32_t compressed(destinationHandler->getUnsignedLong(uLawSamples));
if(bPaddingValueExists && compressed == originalPaddingValue)
{
continue;
}
std::int32_t decompressed(uLawDecompressTable[compressed]);
destinationHandler->setSignedLong(uLawSamples, decompressed);
}
}
return destinationHandler;
PUNTOEXE_FUNCTION_END();
}
