void Message::uncompress() {
if (!isCompressed())
return;
#ifdef BUILD_WITH_COMPRESSION_MINIZ
int cmp_status;
mz_ulong actualSize = strTo<size_t>(_meta["um.compressed"]);
uint8_t *pUncmp;
pUncmp = (mz_uint8 *)malloc((size_t)actualSize);
cmp_status = mz_uncompress(pUncmp, &actualSize, (const unsigned char *)_data.get(), _size);
_size = actualSize;
_data = SharedPtr<char>((char*)pUncmp);
_meta.erase("um.compressed");
#elif defined(BUILD_WITH_COMPRESSION_FASTLZ)
int actualSize = strTo<size_t>(_meta["um.compressed"]);
void* uncompressed = malloc((size_t)actualSize);
// returns the size of the decompressed block.
actualSize = fastlz_decompress(_data.get(), _size, uncompressed, actualSize);
// If error occurs, e.g. the compressed data is corrupted or the output buffer is not large enough, then 0
_size = actualSize;
_data = SharedPtr<char>((char*)uncompressed);
_meta.erase("um.compressed");
#endif
}
static int lmz_uncompress(lua_State* L)
{
int ret;
size_t in_len, out_len;
const unsigned char* inb;
unsigned char* outb;
in_len = 0;
inb = (const unsigned char*)luaL_checklstring(L, 1, &in_len);
out_len = in_len;
outb = malloc(out_len);
ret = mz_uncompress(outb, &out_len, inb, in_len);
switch (ret) {
case MZ_OK:
lua_pushlstring(L, (const char*)outb, out_len);
ret = 1;
break;
default:
lua_pushnil(L);
lua_pushstring(L, mz_error(ret));
ret = 2;
break;
}
free(outb);
return ret;
}
int benchmark()
{
volatile int cnt=0;
int len = strlen(text);
unsigned long slen, dlen;
dlen = 1200;
mz_compress2(compressed, &dlen, (const unsigned char *) text, len, 1);
slen = 1200;
mz_uncompress(tocompress, &slen, compressed, dlen);
dlen = 1200;
mz_compress2(compressed, &dlen, (const unsigned char *) text, len, 7);
slen = 1200;
mz_uncompress(tocompress, &slen, compressed, dlen);
return cnt;
}
int decompress(uint8_t * dest, unsigned long * destSizeBytes,
const uint8_t * source, const unsigned long sourceSizeBytes)
{
assert(dest != nullptr);
assert(destSizeBytes != nullptr && *destSizeBytes != 0);
assert(source != nullptr);
assert(sourceSizeBytes != 0);
return mz_uncompress(dest, destSizeBytes, source, sourceSizeBytes);
}
static inline
bool decode(std::vector<uint8_t>& decompressed, const uint8_t* src, size_t length)
{
const SWF::Header* header = (const SWF::Header*) src;
decompressed.resize(header->fileLength);
uint8_t* pBuff = &decompressed[0];
memcpy(pBuff, src, sizeof(SWF::Header));
mz_ulong sz = header->fileLength - sizeof(SWF::Header);
int ret = mz_uncompress(pBuff + sizeof(SWF::Header), &sz, (const unsigned char*)(header+1), length - sizeof(SWF::Header));
if (ret) {
puts(mz_error(ret));
return false;
}
pBuff[0] = 'F';
return true;
}
void TiledMapLoader::loadLayerTiles(Map &map, Layer &layer, rapidxml::xml_node<> &layerDataNode) {
rapidxml::xml_node<> *tileNode = nullptr;
XMLElement layerDataElement(layerDataNode);
unsigned mapIterator = 0;
unsigned gid = 0;
if (!strncmp(layerDataElement.getString("encoding"), "base64", 6) && !strncmp(layerDataElement.getString("compression"), "zlib", 4)) {
auto base64Tiles = Base64::decode(trim(layerDataElement.getValue()));
unsigned numberOfGids = layer.getWidth() *layer.getHeight();
unsigned tileIndex = 0;
mz_ulong uncompressSize = numberOfGids << 2;
auto gids = new unsigned char[uncompressSize];
if (!gids) {
throw std::logic_error("Uncompression failed: can't allocate memory");
}
if (mz_uncompress(gids, &uncompressSize, (unsigned char *) base64Tiles.c_str(), base64Tiles.length()) != MZ_OK) {
throw std::logic_error("Zlib error: uncompression failed");
}
while (mapIterator < numberOfGids) {
gid = gids[tileIndex] | gids[tileIndex + 1] << 8 | gids[tileIndex + 2] << 16 | gids[tileIndex + 3] << 24;
if (gid)
createTileFromGid(map, layer, gid, mapIterator);
tileIndex += 4;
++mapIterator;
}
delete[] gids;
}
else {
tileNode = layerDataNode.first_node("tile");
if (!tileNode) {
throw std::logic_error("Invalid tiled map: no tiles (only plain XML or ZLIB supported)");
}
while (tileNode) {
XMLElement tileElement(*tileNode);
gid = tileElement.getUnsignedInt("gid");
if (gid) {
createTileFromGid(map, layer, gid, mapIterator);
}
tileNode = tileNode->next_sibling("tile");
++mapIterator;
}
}
}
unsigned char *ramdisk_load (const char *filename, int32_t *outlen)
{
ramdisk_t *ramfile = ramdisk_data;
unsigned char *out;
char *alt_filename;
alt_filename = 0;
/*
* If the file is on disk and is newer than the program, use that in
* preference.
*/
if (file_exists(filename)) {
if (strstr(filename, "data/")) {
if (file_exists_and_is_newer_than(filename,
EXEC_FULL_PATH_AND_NAME)) {
out = file_read_if_exists(filename, outlen);
if (out) {
DBG("Locdisk %s (newer than exec)", filename);
return (out);
}
}
if (file_exists_and_is_newer_than(filename, ".o/ramdisk_data.o")) {
out = file_read_if_exists(filename, outlen);
if (out) {
DBG("Locdisk %s (newer than build)", filename);
return (out);
}
}
if (file_exists_and_is_newer_than(filename, "src/.o/ramdisk_data.o")) {
out = file_read_if_exists(filename, outlen);
if (out) {
DBG("Locdisk %s (newer than src build)", filename);
return (out);
}
}
} else {
out = file_read_if_exists(filename, outlen);
if (out) {
DBG("Locdisk %s (exists locally)", filename);
return (out);
}
}
}
if (EXEC_DIR) {
alt_filename = strprepend(filename, EXEC_DIR);
if (file_exists(alt_filename)) {
if (file_exists_and_is_newer_than(alt_filename,
EXEC_FULL_PATH_AND_NAME)) {
out = file_read_if_exists(alt_filename, outlen);
if (out) {
DBG("Locdisk %s", filename);
myfree(alt_filename);
return (out);
}
}
if (file_exists_and_is_newer_than(alt_filename,
".o/ramdisk_data.o")) {
out = file_read_if_exists(alt_filename, outlen);
if (out) {
DBG("Locdisk %s", filename);
myfree(alt_filename);
return (out);
}
}
if (file_exists_and_is_newer_than(alt_filename,
"src/.o/ramdisk_data.o")) {
out = file_read_if_exists(alt_filename, outlen);
if (out) {
DBG("Locdisk %s", filename);
myfree(alt_filename);
return (out);
}
}
}
}
while (ramfile->filename) {
if (strcmp(ramfile->filename, filename)) {
ramfile++;
continue;
}
if (outlen) {
*outlen = (int)ramfile->len;
}
DBG("Ramdisk %s, %d bytes", filename, ramfile->len);
if (alt_filename) {
myfree(alt_filename);
}
uint8_t *copy = (TYPEOF(copy))
mymalloc((int)ramfile->len + 1, "ramdisk load");
if (!copy) {
DBG("no memory for loading ramdisk copy, %s", filename);
return (0);
}
memcpy(copy, (unsigned char*)ramfile->data, (int)ramfile->len);
*(copy + (int)ramfile->len) = 0;
return (copy);
#ifdef USE_ZLIB
int32_t err;
out = mymalloc(ramfile->orig_len, "RAMDISK scratchpad");
if (!out) {
ERR("no memory for ramdisk, %s", filename);
}
outlenl = ramfile->orig_len;
err = uncompress(out, &outlenl,
(unsigned char *)ramfile->data,
ramfile->len);
if (err) {
ERR("file failed to decompress from ramdisk, %s, "
"orig len %d, compressed len %d, err %d",
filename, ramfile->orig_len, ramfile->len, err);
}
if (outlen) {
*outlen = (int)outlenl;
}
#endif
#ifdef USE_MINIZ
int32_t err;
out = (TYPEOF(out)) mymalloc(ramfile->orig_len, "RAMDISK scratchpad");
if (!out) {
ERR("no memory for ramdisk, %s", filename);
}
outlenl = ramfile->orig_len;
err = mz_uncompress(out, &outlenl,
(unsigned char *)ramfile->data,
ramfile->len);
if (err) {
ERR("file failed to decompress from ramdisk, %s, err %d",
filename, err);
}
if (outlen) {
*outlen = (int32_t)outlenl;
}
#endif
#ifdef USE_STB_IMAGE
out = (TYPEOF(out))
stbi_zlib_decode_malloc((const char *)ramfile->data,
ramfile->len, &outlenl);
if (!out) {
ERR("file failed to decompress from ramdisk, %s", filename);
}
if (outlen) {
*outlen = (int)outlenl;
}
#endif
return (out);
}
/*
* Fallback to the disk.
*/
out = file_read_if_exists(filename, outlen);
if (out) {
DBG("Locdisk %s", filename);
if (alt_filename) {
myfree(alt_filename);
}
return (out);
}
out = file_read_if_exists(alt_filename, outlen);
if (out) {
DBG("Locdisk %s", filename);
if (alt_filename) {
myfree(alt_filename);
}
return (out);
}
if (alt_filename) {
myfree(alt_filename);
}
alt_filename = mybasename(filename, "strip dir");
out = file_read_if_exists(alt_filename, outlen);
if (out) {
DBG("Locdisk %s", alt_filename);
if (alt_filename) {
myfree(alt_filename);
}
return (out);
}
/*
* Fail. Caller should whinge.
*
char *popup_str = dynprintf("Filename was not found on ramdisk or "
"on the local disk, %s", filename);
MSG_BOX("%s", popup_str);
myfree(popup_str);
*/
LOG("File not found \"%s\"", filename);
if (alt_filename) {
myfree(alt_filename);
}
return (0);
}
void _parse_header(pTHX_ srl_splitter_t *splitter) {
int magic_string = 1;
int high_magic_string = 1;
U8 version_encoding;
U8 version;
U8 encoding_flags;
UV header_len;
int is_zlib_encoded = 0;
int is_snappy_encoded = 0;
int is_snappyincr_encoded = 0;
// SRL_MAGIC_STRLEN + PROTOCOL_LENGTH + OPTIONAL-HEADER-SIZE(at least 1 byte) + DATA(at least 1 byte)
if (splitter->input_len < SRL_MAGIC_STRLEN + 1 + 1 + 1){
croak("input Sereal string lacks data");
} else if ( (high_magic_string = strncmp(splitter->input_str, SRL_MAGIC_STRING, SRL_MAGIC_STRLEN))
&& (magic_string = strncmp(splitter->input_str, SRL_MAGIC_STRING_HIGHBIT, SRL_MAGIC_STRLEN)) ) {
croak("input Sereal string has wrong Sereal magic");
}
splitter->pos += SRL_MAGIC_STRLEN;
version_encoding = (U8)*(splitter->pos);
version = (U8)(version_encoding & SRL_PROTOCOL_VERSION_MASK);
encoding_flags = (U8)(version_encoding & SRL_PROTOCOL_ENCODING_MASK);
if ( version <= 0
|| ( version < 3 && high_magic_string )
|| ( version > 2 && magic_string ) ) {
croak("unsupported Sereal versions/protocol");
}
switch(encoding_flags) {
case SRL_PROTOCOL_ENCODING_RAW:
/* no op */
SRL_SPLITTER_TRACE("encoding is raw %s", "");
break;
case SRL_PROTOCOL_ENCODING_SNAPPY:
SRL_SPLITTER_TRACE("encoding is snappy %s", "");
is_snappy_encoded = 1;
break;
case SRL_PROTOCOL_ENCODING_SNAPPY_INCREMENTAL:
SRL_SPLITTER_TRACE("encoding is snappy incr %s", "");
is_snappy_encoded = is_snappyincr_encoded = 1;
break;
case SRL_PROTOCOL_ENCODING_ZLIB:
SRL_SPLITTER_TRACE("encoding is zlib %s", "");
is_zlib_encoded = 1;
break;
default:
croak("Sereal document encoded in an unknown format");
}
SRL_SPLITTER_TRACE("header version is %hhu", version);
// move after protocol version
splitter->pos += 1;
header_len= _read_varint_uv_nocheck(splitter);
SRL_SPLITTER_TRACE("header len is %lu", header_len);
//TODO: add code for processing the header
splitter->pos += header_len;
if (version < 2) {
splitter->input_body_pos = splitter->input_str;
} else {
splitter->input_body_pos = splitter->pos;
}
if (is_snappy_encoded) {
UV compressed_len;
uint32_t uncompressed_len;
int decompress_ok;
char * new_input_str;
if (is_snappyincr_encoded) {
compressed_len = _read_varint_uv_nocheck(splitter);
} else {
compressed_len = splitter->input_len - (splitter->pos - splitter->input_str);
}
SRL_SPLITTER_TRACE("snappy compressed len %"UVuf, compressed_len);
// splitter->pos is now at start of compressed payload
int snappy_header_len;
char *old_pos;
old_pos = splitter->pos;
snappy_header_len = csnappy_get_uncompressed_length(
(char *)old_pos,
compressed_len,
&uncompressed_len
);
if (snappy_header_len == CSNAPPY_E_HEADER_BAD) {
croak("invalid Snappy header in Snappy-compressed Sereal packet");
}
// allocate a new SV for uncompressed data
SvREFCNT_dec(splitter->input_sv);
splitter->input_sv = newSVpvs("");
new_input_str = SvGROW(splitter->input_sv, uncompressed_len);
decompress_ok = csnappy_decompress_noheader((char *) (old_pos + snappy_header_len),
compressed_len - snappy_header_len,
(char *) new_input_str,
&uncompressed_len);
if ( decompress_ok != 0 ) {
croak("Snappy decompression of Sereal packet payload failed");
}
splitter->input_str = new_input_str;
SRL_SPLITTER_TRACE(" decompress OK: uncompressed length: %d\n", uncompressed_len);
splitter->pos = splitter->input_str;;
splitter->input_len = uncompressed_len;
splitter->input_body_pos = splitter->pos;
} else if (is_zlib_encoded) {
UV uncompressed_len = _read_varint_uv_nocheck(splitter);
UV compressed_len = _read_varint_uv_nocheck(splitter);
char * new_input_str;
// splitter->pos is now at start of compressed payload
SRL_SPLITTER_TRACE("unzipping %s", "");
SRL_SPLITTER_TRACE("compressed_len : %" UVuf, compressed_len);
SRL_SPLITTER_TRACE("uncompressed_len : %" UVuf, uncompressed_len);
mz_ulong tmp = uncompressed_len;
// allocate a new SV for uncompressed data
SvREFCNT_dec(splitter->input_sv);
splitter->input_sv = newSVpvs("");
new_input_str = SvGROW(splitter->input_sv, uncompressed_len);
char *compressed = splitter->pos;
int decompress_ok = mz_uncompress( (unsigned char *) new_input_str,
&tmp,
(const unsigned char *) compressed,
compressed_len );
if (decompress_ok != Z_OK)
croak("ZLIB decompression of Sereal packet payload failed");
splitter->input_str = new_input_str;
SRL_SPLITTER_TRACE(" decompress OK: length %lu\n", uncompressed_len);
splitter->pos = splitter->input_str;
splitter->input_len = (STRLEN)tmp;
splitter->input_body_pos = splitter->pos;
}
}
void Message::uncompress() {
if (!isCompressed())
return;
std::string errorStr = "Unsupported compression";
size_t actualSize;
std::string comprType;
std::string comprIdent = _meta["um.compressed"];
size_t colon = comprIdent.find_first_of(':');
if (colon == std::string::npos) {
errorStr = "No colon found in um.compressed meta field";
goto DECOMPRESS_ERROR;
}
actualSize = strTo<size_t>(comprIdent.substr(0, colon));
comprType = comprIdent.substr(colon + 1);
_meta["um.compressRatio"] = toStr((double)_size / (double)actualSize);
// std::cout << _size << " vs " << actualSize << std::endl;
if (false) {}
#ifdef BUILD_WITH_COMPRESSION_MINIZ
else if (comprType == "miniz") {
int cmp_status;
uint8_t *pUncmp;
pUncmp = (mz_uint8 *)malloc((size_t)actualSize);
cmp_status = mz_uncompress(pUncmp, (mz_ulong*)&actualSize, (const unsigned char *)_data.get(), _size);
if (cmp_status != MZ_OK) {
errorStr = mz_error(cmp_status);
goto DECOMPRESS_ERROR;
}
_size = actualSize;
_data = SharedPtr<char>((char*)pUncmp);
_meta.erase("um.compressed");
return;
}
#endif
#ifdef BUILD_WITH_COMPRESSION_FASTLZ
else if (comprType == "fastlz") {
void* uncompressed = malloc((size_t)actualSize);
// returns the size of the decompressed block.
actualSize = fastlz_decompress(_data.get(), _size, uncompressed, actualSize);
// If error occurs, e.g. the compressed data is corrupted or the output buffer is not large enough, then 0
if (actualSize == 0) {
errorStr = "fastlz_decompress returned 0";
goto DECOMPRESS_ERROR;
}
_size = actualSize;
_data = SharedPtr<char>((char*)uncompressed);
_meta.erase("um.compressed");
return;
}
#endif
#ifdef BUILD_WITH_COMPRESSION_LZ4
else if (comprType == "lz4") {
#ifdef LZ4_FRAME
LZ4F_errorCode_t n;
LZ4F_decompressionContext_t ctx;
void* uncompressed = malloc((size_t)actualSize);
n = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
if (LZ4F_isError(n)) {
errorStr = LZ4F_getErrorName(n);
goto DECOMPRESS_ERROR;
}
n = LZ4F_decompress(ctx, uncompressed, &actualSize, _data.get(), &_size, NULL);
if (LZ4F_isError(n)) {
errorStr = LZ4F_getErrorName(n);
goto DECOMPRESS_ERROR;
}
_size = actualSize;
_data = SharedPtr<char>((char*)uncompressed);
_meta.erase("um.compressed");
LZ4F_freeDecompressionContext(ctx);
#else
char* uncompressed = (char*)malloc((size_t)actualSize);
int n = LZ4_decompress_fast(_data.get(), uncompressed, actualSize);
if (n < 0) {
errorStr = "Decompression failed";
goto DECOMPRESS_ERROR;
}
_size = actualSize;
_data = SharedPtr<char>((char*)uncompressed);
_meta.erase("um.compressed");
#endif
return;
}
#endif
DECOMPRESS_ERROR:
UM_LOG_WARN("Could not decompress message: %s", errorStr.c_str());
}
void gettitle(unsigned char *buf)
{
uLongf uncomplen=UTITLELEN;
mz_uncompress(buf, &uncomplen, title_gz, CTITLELEN);
}