void dnApplyDelta( const snapshot_t *olds, snapshot_t *news, const snapshotDelta_t *delta ) {
const char *streamPtr;
const char *oldp = (const char*)olds;
char *newp = (char*)news;
unsigned char *bitmask;
bitmask = (unsigned char*)malloc( sizeof( snapshot_t ) / 8 );
LZ4_decompress_fast( delta->bitmask, (char*)bitmask, sizeof( snapshot_t ) / 8 );
streamPtr = &delta->stream[0];
for ( int i = 0; i < sizeof( snapshot_t ); i++, oldp++, newp++ ) {
if ( bitmask[ i/8 ] & ( 1 << i%8 )) {
*newp = *streamPtr;
streamPtr++;
} else {
*newp = *oldp;
}
}
free( bitmask );
printf( "stream size: %d\n", delta->streamSize );
printf( "read %d bytes\n", streamPtr - &delta->stream[0] );
}
/*
* Class: net_jpountz_lz4_LZ4
* Method: LZ4_decompress
* Signature: ([BI[BII)I
*/
JNIEXPORT jint JNICALL Java_net_jpountz_lz4_LZ4JNI_LZ4_1decompress_1fast
(JNIEnv *env, jclass cls, jbyteArray src, jint srcOff, jbyteArray dest, jint destOff, jint destLen) {
char* in;
char* out;
jint compressed;
in = (char*) (*env)->GetPrimitiveArrayCritical(env, src, 0);
if (in == NULL) {
throw_OOM(env);
return 0;
}
out = (char*) (*env)->GetPrimitiveArrayCritical(env, dest, 0);
if (out == NULL) {
throw_OOM(env);
return 0;
}
compressed = LZ4_decompress_fast(in + srcOff, out + destOff, destLen);
(*env)->ReleasePrimitiveArrayCritical(env, src, in, 0);
(*env)->ReleasePrimitiveArrayCritical(env, src, out, 0);
return compressed;
}
bool decompress(Compression compression, char *&data, std::uint32_t in_size, std::uint32_t out_size)
{
(void)in_size; // Unreferenced parameter
switch (compression)
{
case COMPRESS_NONE:
{
return true;
}
#ifdef ZAP_COMPRESS_LZ4
case COMPRESS_LZ4:
{
char *out_data = new char[out_size];
if (LZ4_decompress_fast(data, out_data, out_size) > 0)
{
delete[] data;
data = out_data;
return true;
}
else
{
return false;
}
}
#endif
default:
{
return false;
}
}
}
unsigned DecompressData(void* dest, const void* src, unsigned destSize)
{
if (!dest || !src || !destSize)
return 0;
else
return LZ4_decompress_fast((const char*)src, (char*)dest, destSize);
}
/* Return value must be an intptr_t instead of an int because pthread returns
* a void*-sized value */
static intptr_t decompress(const decompress_args* args) {
int actual_compressed_size = LZ4_decompress_fast(
args->compressed,
args->decompress_start,
args->decompressed_size);
return args->compressed_size == actual_compressed_size;
}
int _ol_get_value_from_bucket(const ol_database *db, const ol_bucket *bucket,
unsigned char **data, size_t *dsize) {
check(bucket != NULL, "Cannot retrieve value from NULL bucket.");
/* Allocate memory to store memcpy'd data into. */
*data = malloc(bucket->original_size + 1);
check_mem(*data);
(*data)[bucket->original_size] = '\0';
if (dsize != NULL) {
/* "memcpy never fails!" */
size_t *ret = memcpy(dsize, &bucket->original_size, sizeof(size_t));
check(ret == dsize, "Could not copy data size into input data_size param.");
}
unsigned char *data_ptr = db->values + bucket->data_offset;
/* Decomperss with LZ4 if enabled */
if (db->is_enabled(OL_F_LZ4, &db->feature_set)) {
int processed = 0;
processed = LZ4_decompress_fast((char *)data_ptr,
(char *)*data,
bucket->original_size);
check(processed == bucket->data_size, "Could not decompress data.");
} else {
/* We know data isn't NULL by this point. */
unsigned char *ret = memcpy(*data, data_ptr, bucket->original_size);
check(ret == *data, "Could not copy data into output data param.");
}
return 0;
error:
return 1;
}
void Decompress(const unsigned char* data, size_t size, std::vector<T>& out)
{
ssize_t countBS = sizeof(T)*out.size();
int32_t r = LZ4_decompress_fast((const char*)data, (char*)&out.front(), countBS);
if(r < 0) {
Util::fire_exception("lz4 decompress failed", r);
}
}
void decompress(void* dest, size_t destSize, const void* source, size_t sourceSize)
{
if (sourceSize == 0 && destSize == 0) return;
int result = LZ4_decompress_fast(static_cast<const char*>(source), static_cast<char*>(dest), destSize);
assert(result >= 0);
assert(static_cast<size_t>(result) == sourceSize);
}
void lz4Decompress(std::shared_ptr<uint8_t> src, std::shared_ptr<uint8_t>& dst,
size_t uncompressedBytes)
{
if (uncompressedBytes > size_t(LZ4_MAX_INPUT_SIZE))
throw std::runtime_error("Expected output data too big for LZ4 (max LZ4_MAX_INPUT_SIZE)");
int const outputSize = static_cast<int>(uncompressedBytes);
int readBytes = LZ4_decompress_fast((const char*)src.get(),
(char*)dst.get(),
outputSize);
if (readBytes < 0)
throw std::runtime_error(std::string("LZ4_decompress_fast failed: faulty input "
"byte at position ") +
SysTools::ToString(-readBytes));
}
void decompress(char * to, size_t size_decompressed, size_t size_compressed_without_checksum)
{
UInt8 method = compressed_buffer[0]; /// See CompressedWriteBuffer.h
if (method == static_cast<UInt8>(CompressionMethodByte::LZ4))
{
//LZ4::statistics(compressed_buffer + COMPRESSED_BLOCK_HEADER_SIZE, to, size_decompressed, stat);
if (variant == LZ4_REFERENCE)
{
if (LZ4_decompress_fast(compressed_buffer + COMPRESSED_BLOCK_HEADER_SIZE, to, size_decompressed) < 0)
throw Exception("Cannot LZ4_decompress_fast", ErrorCodes::CANNOT_DECOMPRESS);
}
else
LZ4::decompress(compressed_buffer + COMPRESSED_BLOCK_HEADER_SIZE, to, size_compressed_without_checksum, size_decompressed, perf_stat);
}
else
throw Exception("Unknown compression method: " + toString(method), ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
}
/* Decompress and bitunshuffle a single block. */
int64_t bshuf_decompress_lz4_block(ioc_chain *C_ptr,
const size_t size, const size_t elem_size) {
int64_t nbytes, count;
size_t this_iter;
void *in = ioc_get_in(C_ptr, &this_iter);
int32_t nbytes_from_header = bshuf_read_uint32_BE(in);
ioc_set_next_in(C_ptr, &this_iter,
(void*) ((char*) in + nbytes_from_header + 4));
void *out = ioc_get_out(C_ptr, &this_iter);
ioc_set_next_out(C_ptr, &this_iter,
(void *) ((char *) out + size * elem_size));
void* tmp_buf = malloc(size * elem_size);
if (tmp_buf == NULL) return -1;
#ifdef BSHUF_LZ4_DECOMPRESS_FAST
nbytes = LZ4_decompress_fast((const char*) in + 4, (char*) tmp_buf, size * elem_size);
CHECK_ERR_FREE_LZ(nbytes, tmp_buf);
if (nbytes != nbytes_from_header) {
free(tmp_buf);
return -91;
}
#else
nbytes = LZ4_decompress_safe((const char*) in + 4, (char *) tmp_buf, nbytes_from_header,
size * elem_size);
CHECK_ERR_FREE_LZ(nbytes, tmp_buf);
if (nbytes != size * elem_size) {
free(tmp_buf);
return -91;
}
nbytes = nbytes_from_header;
#endif
count = bshuf_untrans_bit_elem(tmp_buf, out, size, elem_size);
CHECK_ERR_FREE(count, tmp_buf);
nbytes += 4;
free(tmp_buf);
return nbytes;
}
void hh_load_dep_table(value in_filename) {
CAMLparam1(in_filename);
struct timeval tv;
gettimeofday(&tv, NULL);
FILE* fp = fopen(String_val(in_filename), "rb");
if (fp == NULL) {
unix_error(errno, "fopen", in_filename);
}
fread_header(fp);
int compressed_size = 0;
read_all(fileno(fp), (void*)&compressed_size, sizeof compressed_size);
char* compressed = malloc(compressed_size * sizeof(char));
assert(compressed != NULL);
read_all(fileno(fp), compressed, compressed_size * sizeof(char));
int actual_compressed_size = LZ4_decompress_fast(
compressed,
(char*)deptbl,
DEP_SIZE_B);
assert(compressed_size == actual_compressed_size);
tv = log_duration("Loading file", tv);
uintptr_t slot = 0;
unsigned long hash = 0;
for (slot = 0; slot < DEP_SIZE; ++slot) {
hash = deptbl[slot];
if (hash != 0) {
htable_add(deptbl_bindings, hash, hash);
}
}
fclose(fp);
log_duration("Bindings", tv);
CAMLreturn0;
}
int ol_cas(ol_database *db, const char *key, const size_t klen,
unsigned char *value, size_t vsize,
const unsigned char *ovalue, const size_t ovsize) {
char _key[KEY_SIZE] = {'\0'};
size_t _klen = 0;
ol_bucket *bucket = ol_get_bucket(db, key, klen, &_key, &_klen);
check(_klen > 0, "Key length of zero not allowed.");
if (bucket == NULL)
return OL_FAILURE;
/* Quick fail if the two sizes don't match */
if (bucket->original_size != ovsize)
return OL_FAILURE;
/* ATOMIC, GOOOO! */
const unsigned char *data_ptr = db->values + bucket->data_offset;
if (db->is_enabled(OL_F_LZ4, &db->feature_set)) {
char decompressed[bucket->original_size];
memset(decompressed, '\0', bucket->original_size);
int processed = 0;
processed = LZ4_decompress_fast((char *)data_ptr,
(char *)decompressed,
bucket->original_size);
check(processed == bucket->data_size, "Could not decompress data.");
if (memcmp(decompressed, ovalue, ovsize) == 0)
return ol_jar(db, key, klen, value, vsize);
} else {
if (memcmp(data_ptr, ovalue, ovsize) == 0)
return ol_jar(db, key, klen, value, vsize);
}
return OL_FAILURE;
error:
return OL_FAILURE;
}
bool DecompressStream(Serializer& dest, Deserializer& src)
{
if (src.IsEof())
return false;
unsigned destSize = src.ReadUInt();
unsigned srcSize = src.ReadUInt();
if (!srcSize || !destSize)
return true; // No data
if (srcSize > src.GetSize())
return false; // Illegal source (packed data) size reported, possibly not valid data
SharedArrayPtr<unsigned char> srcBuffer(new unsigned char[srcSize]);
SharedArrayPtr<unsigned char> destBuffer(new unsigned char[destSize]);
if (src.Read(srcBuffer, srcSize) != srcSize)
return false;
LZ4_decompress_fast((const char*)srcBuffer.Get(), (char*)destBuffer.Get(), destSize);
return dest.Write(destBuffer, destSize) == destSize;
}
void SDRdaemonBufferOld::writeToRawBufferLZ4()
{
uint64_t crc64 = m_crc64.calculate_crc(m_lz4InBuffer, m_lz4InSize);
if (memcmp(&crc64, &m_dataCRC, 8) == 0)
{
m_nbLz4CRCOK++;
}
else
{
return;
}
int compressedSize = LZ4_decompress_fast((const char*) m_lz4InBuffer, (char*) m_lz4OutBuffer, m_frameSize);
m_nbLz4Decodes++;
if (compressedSize == m_lz4InSize)
{
m_nbLz4SuccessfulDecodes++;
writeToRawBufferUncompressed((const char *) m_lz4OutBuffer, m_frameSize);
}
}
int FUZ_SecurityTest()
{
char* output;
char* input;
int i, r;
// Overflow test, by Ludwig Strigeus
printf("Overflow test (issue 52)...");
input = (char*) malloc (20<<20);
output = (char*) malloc (20<<20);
input[0] = 0x0F;
input[1] = 0x00;
input[2] = 0x00;
for(i = 3; i < 16840000; i++)
input[i] = 0xff;
r = LZ4_decompress_fast(input, output, 20<<20);
free(input);
free(output);
printf(" Passed (return = %i < 0)\n",r);
return 0;
}
// Test that we can compress data larger than 4096 bytes (MAX_INPUT_SIZE).
void test_large_data()
{
std::string lorem_ipsum = "Lorem ipsum dolor sit amet, soluta accusamus ei vix, quo an quas quidam scribentur, cu illud atqui mea. No mel adipisci repudiandae. Congue doctus cu vix, dolorum accusata id ius, ad dicat recusabo voluptatum his. Nam ludus choro ut, quo justo accommodare ea. Cu odio civibus dolores duo, et eos cibo reprimique, simul quaestio signiferumque ei pri. Impedit consetetur te vel. Altera pericula ut his, ut cum offendit prodesset referrentur. Falli blandit te vel, sed te assum consectetuer vituperatoribus. Qui odio nobis labores ei. Eu iudico appareat eum, in sed causae definitionem, vocent tritani volutpat est eu. Te vis atqui nominavi voluptatum. Quem alienum rationibus ad eum. Ignota laboramus honestatis ad quo. Vel eu virtute veritus, et brute euripidis ius. Tollit consequat scriptorem ad eam, in vix vituperata honestatis, no vitae recusabo sea. Congue graeci laoreet et eam, quo ex modus suscipit contentiones, oratio accusamus pri et. Cu mea legendos molestiae prodesset. Ius ubique dolores conclusionemque at. An vel malis consequat. Mel an suas voluptua. Nec te aliquam mandamus. Eum ut aliquam albucius, ad vis sonet intellegebat, no pro detracto fabellas. Vocent fuisset deleniti te quo, audiam contentiones mei ne. Ea tempor invenire quaerendum est, eu possim instructior eum. Ut aliquam evertitur tincidunt duo, impetus nostrud nam ei, eos affert oporteat persequeris in. Ne justo integre omittantur vel. Per epicuri fastidii at, usu causae delicatissimi id. Ne ius consul repudiare. Dicat appetere ius eu, odio nobis oportere cu est. Ea natum numquam per, molestiae deseruisse vix ex, denique singulis has no. Ut novum adipiscing duo, nec cu ornatus appareat. Ei placerat verterem tractatos eos. Cu doctus civibus has, eam volumus splendide at. Erant insolens invenire eu nec, augue scripta est ne. Wisi partem has in, civibus albucius pro ea, his stet postea no. Postea aliquip vituperata sed no, eos eu erat repudiare. Cu invenire salutatus deterruisset vis, sea ei nulla impetus, ea alii noster vivendo ius. Et vel lorem graecis iudicabit. Ei mea utamur dolorem, tantas impedit vivendum in cum. Quis fuisset moderatius sit eu, vide meis assueverit sit te, quo errem impetus repudiandae te. Mel agam quas graecis ea. Duo doming mollis latine et. Putant equidem inimicus mea et, te purto lorem vix, eum choro clita accusamus at. Eros alienum insolens ex eum, vitae equidem iudicabit te eam. Eam tale duis in. His no illud antiopam, erat sale te vix, agam impedit quo ei. Ut mei albucius referrentur. Ea mel solum accusam, quodsi everti vulputate te mei. Tota putent nemore eam te, saepe fastidii ea cum. Legimus reprimique vel an, at qui detracto definiebas sadipscing, ut quod nostrud sadipscing nec. Vel id vidit fierent, ex sed tale platonem. Vel eu deleniti perfecto dissentiet, ut mei quodsi detracto, mea id novum suavitate. Ne viris postea dictas sed, erat iisque necessitatibus in has. Mel ei iisque meliore, ne wisi tamquam est. At appareat scriptorem mea, inermis perpetua ne ius, id vitae eligendi ius. Soleat prodesset no vis, hinc munere appellantur eum te. Tale laoreet principes his an, instructior complectitur voluptatibus vel eu. Expetenda efficiendi per an. Essent expetendis quo ut, his sapientem ocurreret ei. Per graece animal iudicabit et. Pri debet vituperata ne. Porro senserit sed ne, at has sint ocurreret. Vis populo essent expetenda an, graece aliquando ne usu, his ei noster omnium habemus. Ex ius corrumpit expetendis. Ne his nullam evertitur eloquentiam. Ad vero eruditi cum, malis maiestatis et mei. Vim dicit deserunt definitionem ad, facete malorum molestiae in vim, cum te libris referrentur efficiantur. No tation animal omnesque sea. Usu illud petentium in, blandit torquatos mnesarchum his at, eum an audire blandit. Mel illum ponderum prodesset an, qui ea summo postulant gubergren, est graeci ullamcorper te. Prima equidem deseruisse vim in, altera iuvaret nec no, ea putant nusquam corrumpit per. Liber scriptorem nam cu, mea quis corpora ea, sea vivendum indoctum forensibus te. Vix alia sapientem et, pri an esse mucius. Movet vocent nostrum ex quo. His ne civibus ponderum quaestio, et habeo tation gubergren his. In modo everti qui. Probatus atomorum ne has, duo utinam mandamus voluptatibus eu, id option pertinax assueverit has. Mei ad solet iudicabit. Mea ei minim harum iusto, eu nam affert dicunt philosophia. Homero cetero no quo, eum ad qualisque posidonium. Nec falli regione nonumes no, eu vim illud nominati. Facete nonumes eu mei, per ex labores expetenda. Eu harum epicuri atomorum sit, dicat latine inermis et mei. Dolorem periculis efficiantur vim id. Ne omnis impetus ornatus nec. Nec in fuisset interesset, an eam tation legere. Ne mei dicit expetenda. Et mel affert prompta, zril altera bonorum vel an, ullum nonumy vituperata et eum. At eum singulis accommodare, ius in sumo quot. Est ea harum patrioque comprehensam. Eros fierent qui an. Quo in mazim perpetua deterruisset, doming postulant constituto ex has, utinam consectetuer nam ex. Luptatum atomorum nec ex, lorem splendide repudiandae ei eos. Fugit molestie eum an, odio eripuit mea id, sed ea choro nominati. Docendi evertitur eum id. Ei alii facete hendrerit mei, at quot graeco integre nam, in qui soluta contentiones. An eius suscipiantur per, nam in quaeque percipit, odio intellegam eum id. Zril accusam cu pro, cu mei sint nibh corpora. Vis at ornatus nonumes sensibus, adipisci scribentur sit ut, quis molestie vim ei. Vel labores appetere repudiandae ea. Te discere omittam qui, regione omnesque sapientem his ne. Ut sea errem nostro labores.";
SAS::Event event(1, 2, 3);
event.add_compressed_param(lorem_ipsum, &lz4_dict_profile);
std::string bytes = event.to_string();
SasTest::Event expected;
expected.parse(bytes);
// Attempt to decompress the compressed var param.
char decompressed_data[8192] = {0};
int rc = LZ4_decompress_fast(expected.var_params[0].data(), decompressed_data, lorem_ipsum.size());
// LZ4_decompress_fast returns the number of bytes if successful, or a negative number if unsuccessful.
ASSERT(rc > 0);
std::string decompressed_data_str = decompressed_data;
// The decompressed data should be equal to the original data.
ASSERT(lorem_ipsum == decompressed_data_str);
}
/*
* Class: net_jpountz_lz4_LZ4
* Method: LZ4_decompress
* Signature: ([BI[BII)I
*/
JNIEXPORT jint JNICALL Java_net_jpountz_lz4_LZ4JNI_LZ4_1decompress_1fast
(JNIEnv *env, jclass cls, jbyteArray srcArray, jobject srcBuffer, jint srcOff, jbyteArray destArray, jobject destBuffer, jint destOff, jint destLen) {
char* in;
char* out;
jint compressed;
if (srcArray != NULL) {
in = (char*) (*env)->GetPrimitiveArrayCritical(env, srcArray, 0);
} else {
in = (char*) (*env)->GetDirectBufferAddress(env, srcBuffer);
}
if (in == NULL) {
throw_OOM(env);
return 0;
}
if (destArray != NULL) {
out = (char*) (*env)->GetPrimitiveArrayCritical(env, destArray, 0);
} else {
out = (char*) (*env)->GetDirectBufferAddress(env, destBuffer);
}
if (out == NULL) {
throw_OOM(env);
return 0;
}
compressed = LZ4_decompress_fast(in + srcOff, out + destOff, destLen);
if (destArray != NULL) {
(*env)->ReleasePrimitiveArrayCritical(env, destArray, out, 0);
}
if (srcArray != NULL) {
(*env)->ReleasePrimitiveArrayCritical(env, srcArray, in, 0);
}
return compressed;
}
bool LZ4CompressedReaderAdapter::fill_buffer()
{
size_t buffer_size;
if (read_uint64(m_file, buffer_size) == 0)
return false;
ensure_minimum_size(m_buffer, buffer_size);
size_t compressed_buffer_size;
read_uint64(m_file, compressed_buffer_size);
ensure_minimum_size(m_compressed_buffer, compressed_buffer_size);
m_file.read(&m_compressed_buffer[0], compressed_buffer_size);
LZ4_decompress_fast(
reinterpret_cast<const char*>(&m_compressed_buffer[0]),
reinterpret_cast<char*>(&m_buffer[0]),
static_cast<int>(buffer_size));
m_buffer_index = 0;
m_buffer_end = buffer_size;
return true;
}
int main (int argc, char *argv[])
{
FILE *in;
uint8_t *input;
uint32_t isize = 0;
uint32_t ipos = 0;
FILE *out;
uint32_t osize = 0;
uint8_t *output;
int32_t decompressed;
uint32_t n;
if (argc != 3) {
fprintf(stderr, "Mozilla LZ4 4a version decompressor.\n"
"Usage: unmoz inputfile outputfile\n"
);
return 1;
}
in = fopen(argv[1], "rb");
if (!in) { perror("Error opening input file"); return 1; }
isize = fsize(argv[1]);
if (isize < 12 || isize > 134217728) {
fprintf(stderr, "Size is %" PRIu32 " bytes, must be >12 bytes and <128MB\n", isize);
return 1;
}
input = malloc(isize * sizeof(uint8_t));
if (input == NULL) { perror("Cannot allocate memory"); return 1; }
memset(input, 0, isize*sizeof(uint8_t));
n = fread(input, sizeof(uint8_t), isize, in);
if (n != isize) {
fprintf(stderr, "Only got %" PRIu32 " input bytes. Quitting.\n", n);
perror("Read error");
return 1;
}
/* check magic number */
if (strncmp((char *)(input+ipos), MOZLZ4_MAGIC, strlen(MOZLZ4_MAGIC)+1) != 0) {
fprintf(stderr, "Magic number mismatch.\n"
"Expected:\n\t"
);
for (int i=0; i<strlen(MOZLZ4_MAGIC)+1; i++) { fprintf(stderr, "%4c", MOZLZ4_MAGIC[i]); }
fprintf(stderr, "\n\t");
for (int i=0; i<strlen(MOZLZ4_MAGIC)+1; i++) { fprintf(stderr, "%4u", MOZLZ4_MAGIC[i]); }
fprintf(stderr, "\nGot:\n\t");
for (int i=0; i<strlen(MOZLZ4_MAGIC)+1; i++) { fprintf(stderr, "%4c", input[i+ipos]); }
fprintf(stderr, "\n\t");
for (int i=0; i<strlen(MOZLZ4_MAGIC)+1; i++) { fprintf(stderr, "%4u", input[i+ipos]); }
fprintf(stderr, "\n");
return 1;
}
printf("Magic number ok!\n");
ipos += strlen(MOZLZ4_MAGIC)+1;
/* read output size - big endian */
osize = input[ipos] + (input[ipos + 1] << 8) + (input[ipos + 2] << 16) + (input[ipos + 3] << 24);
printf("Output size: %" PRIu32 "\n", osize);
ipos += sizeof(uint32_t);
output = malloc(osize * sizeof(uint8_t));
if (input == NULL) { perror("Cannot allocate memory"); return 1; }
memset(output, 0, osize*sizeof(uint8_t));
/* decompress */
decompressed = LZ4_decompress_fast((char *)(input+ipos), (char *)output, osize);
if (decompressed < 0) {
perror("Decompression problem");
return 1;
}
printf("Decompressed %" PRIu32 "/%" PRIu32 " input bytes.\n", decompressed, isize-ipos);
/* write */
out = fopen(argv[2], "wb");
if (!out) { perror("Error opening output file"); return 1; }
n = fwrite(output, sizeof(uint8_t), osize, out);
if (n != osize) {
fprintf(stderr, "Wrote %" PRIu32 "/%" PRIu32 " output bytes.\n", n, osize);
perror("Cannot write output file");
return 1;
}
fclose(in);
fclose(out);
printf("Done.\n");
return 0;
}
static int
lsm_lz4_xUncompress(void *pCtx, void *pOut, int *pnOut, const void *pIn, int nIn)
{
int rc = LZ4_decompress_fast((const char*)pIn, (char*)pOut, *pnOut);
return (rc > 0 ? LSM_OK : LSM_ERROR);
}
int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
unsigned long long bytes = 0;
unsigned long long cbytes = 0;
unsigned long long hcbytes = 0;
unsigned long long ccbytes = 0;
void* CNBuffer;
char* compressedBuffer;
char* decodedBuffer;
# define FUZ_max LZ4_COMPRESSBOUND(LEN)
unsigned int randState=seed;
int ret, cycleNb;
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \
printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; }
# define FUZ_DISPLAYTEST { testNb++; ((displayLevel<3) || no_prompt) ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); }
void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
void* LZ4continue;
LZ4_stream_t LZ4dict;
U32 crcOrig, crcCheck;
int displayRefresh;
// init
memset(&LZ4dict, 0, sizeof(LZ4dict));
// Create compressible test buffer
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
// display refresh rate
switch(displayLevel)
{
case 0: displayRefresh = nbCycles+1; break;
case 1: displayRefresh=FUZ_MAX(1, nbCycles / 100); break;
case 2: displayRefresh=89; break;
default : displayRefresh=1;
}
// move to startCycle
for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
{
// synd rand & dict
int dictSize, blockSize, blockStart;
char* dict;
char* block;
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
if (dictSize > blockStart) dictSize = blockStart;
block = ((char*)CNBuffer) + blockStart;
dict = block - dictSize;
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
}
// Test loop
for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
{
int testNb = 0;
char* dict;
char* block;
int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize;
int blockContinueCompressedSize;
if ((cycleNb%displayRefresh) == 0)
{
printf("\r%7i /%7i - ", cycleNb, nbCycles);
fflush(stdout);
}
// Select block to test
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
if (dictSize > blockStart) dictSize = blockStart;
block = ((char*)CNBuffer) + blockStart;
dict = block - dictSize;
/* Compression tests */
// Test compression HC
FUZ_DISPLAYTEST;
ret = LZ4_compressHC(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
HCcompressedSize = ret;
// Test compression HC using external state
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
// Test compression using external state
FUZ_DISPLAYTEST;
ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
// Test compression
FUZ_DISPLAYTEST;
ret = LZ4_compress(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
compressedSize = ret;
/* Decompression tests */
crcOrig = XXH32(block, blockSize, 0);
// Test decoding with output size being exactly what's necessary => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
// Test decoding with one byte missing => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
// Test decoding with one byte too much => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
// Test decoding with output size exactly what's necessary => must work
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
// Test decoding with more than enough output size => must work
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
decodedBuffer[blockSize+1] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space");
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
//FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ?
FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
// Test decoding with output size being one byte too short => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
// Test decoding with output size being 10 bytes too short => must fail
FUZ_DISPLAYTEST;
if (blockSize>10)
{
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short");
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size");
}
// Test decoding with input size being one byte too short => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
// Test decoding with input size being one byte too large => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
// Test partial decoding with target output size being max/2 => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
// Test partial decoding with target output size being just below max => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
/* Test Compression with limited output size */
// Test compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
// Test compression with output size being exactly what's necessary and external state (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compress_limitedOutput_withState(stateLZ4, block, compressedBuffer, blockSize, compressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space");
// Test HC compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
// Test HC compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space");
// Test compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
compressedBuffer[compressedSize-1] = 0;
ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize-1);
FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compress_limitedOutput overran output buffer")
// Test HC compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
compressedBuffer[compressedSize-1] = 0;
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1);
FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer")
/* Dictionary tests */
// Compress using dictionary
FUZ_DISPLAYTEST;
LZ4continue = LZ4_create (dict);
LZ4_compress_continue (LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
blockContinueCompressedSize = LZ4_compress_continue (LZ4continue, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
LZ4_free (LZ4continue);
// Decompress with dictionary as prefix
FUZ_DISPLAYTEST;
memcpy(decodedBuffer, dict, dictSize);
ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
// Compress using External dictionary
FUZ_DISPLAYTEST;
dict -= 9; // Separation, so it is an ExtDict
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
// Decompress with dictionary as external
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
if (blockSize > 10)
{
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-10, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-10 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-10 byte) (blockSize=%i)", blockSize);
}
// Fill stats
bytes += blockSize;
cbytes += compressedSize;
hcbytes += HCcompressedSize;
ccbytes += blockContinueCompressedSize;
}
printf("\r%7i /%7i - ", cycleNb, nbCycles);
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100);
// unalloc
if(!no_prompt) getchar();
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
free(stateLZ4);
free(stateLZ4HC);
return 0;
_output_error:
if(!no_prompt) getchar();
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
free(stateLZ4);
free(stateLZ4HC);
return 1;
}
void Message::uncompress() {
if (!isCompressed())
return;
std::string errorStr = "Unsupported compression";
size_t actualSize;
std::string comprType;
std::string comprIdent = _meta["um.compressed"];
size_t colon = comprIdent.find_first_of(':');
if (colon == std::string::npos) {
errorStr = "No colon found in um.compressed meta field";
goto DECOMPRESS_ERROR;
}
actualSize = strTo<size_t>(comprIdent.substr(0, colon));
comprType = comprIdent.substr(colon + 1);
_meta["um.compressRatio"] = toStr((double)_size / (double)actualSize);
// std::cout << _size << " vs " << actualSize << std::endl;
if (false) {}
#ifdef BUILD_WITH_COMPRESSION_MINIZ
else if (comprType == "miniz") {
int cmp_status;
uint8_t *pUncmp;
pUncmp = (mz_uint8 *)malloc((size_t)actualSize);
cmp_status = mz_uncompress(pUncmp, (mz_ulong*)&actualSize, (const unsigned char *)_data.get(), _size);
if (cmp_status != MZ_OK) {
errorStr = mz_error(cmp_status);
goto DECOMPRESS_ERROR;
}
_size = actualSize;
_data = SharedPtr<char>((char*)pUncmp);
_meta.erase("um.compressed");
return;
}
#endif
#ifdef BUILD_WITH_COMPRESSION_FASTLZ
else if (comprType == "fastlz") {
void* uncompressed = malloc((size_t)actualSize);
// returns the size of the decompressed block.
actualSize = fastlz_decompress(_data.get(), _size, uncompressed, actualSize);
// If error occurs, e.g. the compressed data is corrupted or the output buffer is not large enough, then 0
if (actualSize == 0) {
errorStr = "fastlz_decompress returned 0";
goto DECOMPRESS_ERROR;
}
_size = actualSize;
_data = SharedPtr<char>((char*)uncompressed);
_meta.erase("um.compressed");
return;
}
#endif
#ifdef BUILD_WITH_COMPRESSION_LZ4
else if (comprType == "lz4") {
#ifdef LZ4_FRAME
LZ4F_errorCode_t n;
LZ4F_decompressionContext_t ctx;
void* uncompressed = malloc((size_t)actualSize);
n = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
if (LZ4F_isError(n)) {
errorStr = LZ4F_getErrorName(n);
goto DECOMPRESS_ERROR;
}
n = LZ4F_decompress(ctx, uncompressed, &actualSize, _data.get(), &_size, NULL);
if (LZ4F_isError(n)) {
errorStr = LZ4F_getErrorName(n);
goto DECOMPRESS_ERROR;
}
_size = actualSize;
_data = SharedPtr<char>((char*)uncompressed);
_meta.erase("um.compressed");
LZ4F_freeDecompressionContext(ctx);
#else
char* uncompressed = (char*)malloc((size_t)actualSize);
int n = LZ4_decompress_fast(_data.get(), uncompressed, actualSize);
if (n < 0) {
errorStr = "Decompression failed";
goto DECOMPRESS_ERROR;
}
_size = actualSize;
_data = SharedPtr<char>((char*)uncompressed);
_meta.erase("um.compressed");
#endif
return;
}
#endif
DECOMPRESS_ERROR:
UM_LOG_WARN("Could not decompress message: %s", errorStr.c_str());
}
unsigned File::Read(void* dest, unsigned size)
{
if (!IsOpen())
{
// If file not open, do not log the error further here to prevent spamming the stderr stream
return 0;
}
if (mode_ == FILE_WRITE)
{
ATOMIC_LOGERROR("File not opened for reading");
return 0;
}
if (size + position_ > size_)
size = size_ - position_;
if (!size)
return 0;
#ifdef __ANDROID__
if (assetHandle_ && !compressed_)
{
// If not using a compressed package file, buffer file reads on Android for better performance
if (!readBuffer_)
{
readBuffer_ = new unsigned char[READ_BUFFER_SIZE];
readBufferOffset_ = 0;
readBufferSize_ = 0;
}
unsigned sizeLeft = size;
unsigned char* destPtr = (unsigned char*)dest;
while (sizeLeft)
{
if (readBufferOffset_ >= readBufferSize_)
{
readBufferSize_ = Min(size_ - position_, READ_BUFFER_SIZE);
readBufferOffset_ = 0;
ReadInternal(readBuffer_.Get(), readBufferSize_);
}
unsigned copySize = Min((readBufferSize_ - readBufferOffset_), sizeLeft);
memcpy(destPtr, readBuffer_.Get() + readBufferOffset_, copySize);
destPtr += copySize;
sizeLeft -= copySize;
readBufferOffset_ += copySize;
position_ += copySize;
}
return size;
}
#endif
if (compressed_)
{
unsigned sizeLeft = size;
unsigned char* destPtr = (unsigned char*)dest;
while (sizeLeft)
{
if (!readBuffer_ || readBufferOffset_ >= readBufferSize_)
{
unsigned char blockHeaderBytes[4];
ReadInternal(blockHeaderBytes, sizeof blockHeaderBytes);
MemoryBuffer blockHeader(&blockHeaderBytes[0], sizeof blockHeaderBytes);
unsigned unpackedSize = blockHeader.ReadUShort();
unsigned packedSize = blockHeader.ReadUShort();
if (!readBuffer_)
{
readBuffer_ = new unsigned char[unpackedSize];
inputBuffer_ = new unsigned char[LZ4_compressBound(unpackedSize)];
}
/// \todo Handle errors
ReadInternal(inputBuffer_.Get(), packedSize);
LZ4_decompress_fast((const char*)inputBuffer_.Get(), (char*)readBuffer_.Get(), unpackedSize);
readBufferSize_ = unpackedSize;
readBufferOffset_ = 0;
}
unsigned copySize = Min((readBufferSize_ - readBufferOffset_), sizeLeft);
memcpy(destPtr, readBuffer_.Get() + readBufferOffset_, copySize);
destPtr += copySize;
sizeLeft -= copySize;
readBufferOffset_ += copySize;
position_ += copySize;
}
return size;
}
// Need to reassign the position due to internal buffering when transitioning from writing to reading
if (readSyncNeeded_)
{
SeekInternal(position_ + offset_);
readSyncNeeded_ = false;
}
if (!ReadInternal(dest, size))
{
// Return to the position where the read began
SeekInternal(position_ + offset_);
ATOMIC_LOGERROR("Error while reading from file " + GetName());
return 0;
}
writeSyncNeeded_ = true;
position_ += size;
return size;
}
unsigned File::Read(void* dest, unsigned size)
{
#ifdef ANDROID
if (!handle_ && !assetHandle_)
#else
if (!handle_)
#endif
{
// Do not log the error further here to prevent spamming the stderr stream
return 0;
}
if (mode_ == FILE_WRITE)
{
LOGERROR("File not opened for reading");
return 0;
}
if (size + position_ > size_)
size = size_ - position_;
if (!size)
return 0;
#ifdef ANDROID
if (assetHandle_)
{
unsigned sizeLeft = size;
unsigned char* destPtr = (unsigned char*)dest;
while (sizeLeft)
{
if (readBufferOffset_ >= readBufferSize_)
{
readBufferSize_ = Min((int)size_ - position_, (int)READ_BUFFER_SIZE);
readBufferOffset_ = 0;
SDL_RWread(assetHandle_, readBuffer_.Get(), readBufferSize_, 1);
}
unsigned copySize = Min((int)(readBufferSize_ - readBufferOffset_), (int)sizeLeft);
memcpy(destPtr, readBuffer_.Get() + readBufferOffset_, copySize);
destPtr += copySize;
sizeLeft -= copySize;
readBufferOffset_ += copySize;
position_ += copySize;
}
return size;
}
#endif
if (compressed_)
{
unsigned sizeLeft = size;
unsigned char* destPtr = (unsigned char*)dest;
while (sizeLeft)
{
if (!readBuffer_ || readBufferOffset_ >= readBufferSize_)
{
unsigned char blockHeaderBytes[4];
fread(blockHeaderBytes, sizeof blockHeaderBytes, 1, (FILE*)handle_);
MemoryBuffer blockHeader(&blockHeaderBytes[0], sizeof blockHeaderBytes);
unsigned unpackedSize = blockHeader.ReadUShort();
unsigned packedSize = blockHeader.ReadUShort();
if (!readBuffer_)
{
readBuffer_ = new unsigned char[unpackedSize];
inputBuffer_ = new unsigned char[LZ4_compressBound(unpackedSize)];
}
/// \todo Handle errors
fread(inputBuffer_.Get(), packedSize, 1, (FILE*)handle_);
LZ4_decompress_fast((const char*)inputBuffer_.Get(), (char *)readBuffer_.Get(), unpackedSize);
readBufferSize_ = unpackedSize;
readBufferOffset_ = 0;
}
unsigned copySize = Min((int)(readBufferSize_ - readBufferOffset_), (int)sizeLeft);
memcpy(destPtr, readBuffer_.Get() + readBufferOffset_, copySize);
destPtr += copySize;
sizeLeft -= copySize;
readBufferOffset_ += copySize;
position_ += copySize;
}
return size;
}
size_t ret = fread(dest, size, 1, (FILE*)handle_);
if (ret != 1)
{
// Return to the position where the read began
fseek((FILE*)handle_, position_ + offset_, SEEK_SET);
LOGERROR("Error while reading from file " + GetName());
return 0;
}
position_ += size;
return size;
}
/*!
Get a decompressed resource data by it's head.
\param[in] rsc_head : resource head
\return : if failed, return NULL, otherwise return the buffer address of the data
*/
static u8 *rsc_decompress_data(handle_t rsc_handle, rsc_comm_type_t type)
{
u32 *p_buf_addr = NULL;
u32 data_addr = 0, org_size = 0, cmp_size = 0, buf_size = 0, out_size = 0;
rsc_bmp_info_t *p_bmp_info = NULL;
rsc_head_t *p_hdr_rsc = NULL;
rsc_comm_info_t *p_comm_info = NULL;
u8 *p_fake = NULL;
p_comm_info = _get_rsc_comm_info(rsc_handle, type);
if(p_comm_info == NULL)
{
OS_PRINTF("rsc_decompress_data: can NOT get comm_info(type = %d), ERROR\n",
type);
return NULL;
}
p_hdr_rsc = _get_rsc_comm_head(p_comm_info);
p_buf_addr = &p_comm_info->unzip_buf_addr;
buf_size = p_comm_info->unzip_buf_size;
switch(type)
{
case RSC_COMM_FONT:
data_addr = p_comm_info->rsc_hdr_addr + sizeof(rsc_font_t);
break;
case RSC_COMM_PAL:
data_addr = p_comm_info->rsc_hdr_addr + sizeof(rsc_palette_t);
break;
case RSC_COMM_BMP:
data_addr = p_comm_info->rsc_hdr_addr + sizeof(rsc_bitmap_t);
break;
case RSC_COMM_STRTAB:
data_addr = p_comm_info->rsc_hdr_addr + sizeof(rsc_string_t);
break;
case RSC_COMM_SCRIPT:
data_addr = p_comm_info->rsc_hdr_addr + sizeof(rsc_script_t);
break;
default:
MT_ASSERT(0);
return NULL;
}
org_size = p_hdr_rsc->org_size;
cmp_size = p_hdr_rsc->cmp_size;
switch(p_hdr_rsc->c_mode)
{
case CMP_MODE_NONE:
if(buf_size > 0)
{
/* buffer is ok, copy to buf */
if(org_size > buf_size)
{
RSC_PRINTF(
"RSC: buf_size is non-zero(%d), but NOT enough(%d), ERROR!\n",
org_size, buf_size);
MT_ASSERT(0);
}
else
{
rsc_read_data(rsc_handle, data_addr, (void *)(*p_buf_addr), org_size);
}
}
else
{
/* no buffer, return address on FLASH */
*p_buf_addr = data_addr;
}
break;
case CMP_MODE_RLE:
//os_mutex_lock(lock);
if(org_size > buf_size)
{
RSC_PRINTF(
"RSC: buf_size is non-zero(%d), but NOT enough(%d), ERROR!\n",
org_size, buf_size);
MT_ASSERT(0);
}
else
{
rsc_rle_decode(rsc_handle, (u8 *)data_addr, cmp_size, (u8 *)(*p_buf_addr));
}
break;
#if 0
case CMP_MODE_ZIP:
rsc_unzip(data, output);
break;
#endif
case CMP_MODE_LZ77:
rsc_lz77_decode(rsc_handle, (u8 *)data_addr, cmp_size, (u8 *)(*p_buf_addr));
break;
case CMP_MODE_LZMA:
out_size = buf_size;
p_fake = mtos_malloc(cmp_size);
MT_ASSERT(p_fake != NULL);
init_fake_mem_lzma(p_fake, cmp_size);
lzma_decompress((unsigned long *)(*p_buf_addr), &out_size, (void *)data_addr, cmp_size);
mtos_free(p_fake);
break;
case CMP_MODE_LZ4:
LZ4_decompress_fast((const char *)data_addr, (char *)(*p_buf_addr), org_size);
break;
case CMP_MODE_JPG: /*only for bitmap*/
if(RSC_TYPE_BITMAP != p_hdr_rsc->type)
{
return NULL;
}
p_bmp_info = (rsc_bmp_info_t *)_get_rsc_comm_info(rsc_handle, RSC_COMM_BMP);
MT_ASSERT(p_bmp_info != NULL);
p_bmp_info->hdr_bmp.pitch =
rsc_jpg_decode(rsc_handle, (u8 *)data_addr, cmp_size, (u8 *)(*p_buf_addr));
break;
default:
return NULL;
}
return (u8 *)(*p_buf_addr);
}
int ol_aol_restore(ol_database *db)
{
ol_string *command = NULL,
*key = NULL,
*value = NULL,
*ct = NULL,
*read_data_size = NULL,
*read_org_size = NULL;
FILE *fd = fopen(db->aol_file, "r");
check(fd, "Error opening file");
while (!feof(fd)) {
command = _ol_read_data(fd);
check(command, "Error reading");
/* Kind of a hack to check for EOF. If the struct is blank, then we
* read past EOF in _ol_read_data. feof is rarely useful I guess... */
if (command->data == NULL) {
free(command);
break;
}
key = _ol_read_data(fd);
check(key, "Error reading"); /* Everything needs a key */
if (strncmp(command->data, "JAR", 3) == 0) {
ct = _ol_read_data(fd);
check(ct, "Error reading");
read_org_size = _ol_read_data(fd);
check(read_org_size, "Error reading");
read_data_size = _ol_read_data(fd);
check(read_data_size, "Error reading");
value = _ol_read_data(fd);
check(value, "Error reading");
size_t original_size = (size_t)strtol(read_org_size->data, NULL, 10);
size_t compressed_size = (size_t)strtol(read_data_size->data, NULL, 10);
size_t data_offset = (size_t)strtol(value->data, NULL, 10);
/* Pointer in the values file to where the data for this command
* should be. */
unsigned char *data_ptr = db->values + data_offset;
/* Short circuit check to see if the memory in the location is all
* null. */
int memory_is_not_null = 0;
int i = 0;
for (; i < compressed_size; i++) {
if ('\0' != data_ptr[i]) {
debug("Data is not null on %zu.", data_offset + i);
memory_is_not_null = 1;
break;
}
}
if (memory_is_not_null) {
/* Turns out that in rare cases LZ4 will compress to exactly
* the same size as it's starting string. This means we can't
* just check to see if original_size != compressed_size, so
* instead we first attempt to decompress and check how many
* chars were processed.
*/
char tmp_data[original_size];
char *ret = memset(&tmp_data, 0, original_size);
check(ret == tmp_data, "Could not initialize tmp_data parameter.");
int processed = LZ4_decompress_fast((const char*)data_ptr, (char *)tmp_data, original_size);
if (processed == compressed_size)
ol_jar_ct(db, key->data, key->dlen, (unsigned char*)tmp_data, original_size, ct->data, ct->dlen);
else {
if (original_size != compressed_size)
ol_log_msg(LOG_WARN, "Could not decompress data that is probably compressed. Data may have been deleted.");
/* Now that we've tried to decompress and failed, send off the raw data instead. */
ol_jar_ct(db, key->data, key->dlen, data_ptr, compressed_size, ct->data, ct->dlen);
}
}
#ifdef DEBUG
/* This happens a lot and isn't bad, so I'm commenting it out. */
else
ol_log_msg(LOG_WARN, "No data in values file that corresponds with this key. Key has been deleted or updated.");
#endif
/* Important: Set the new offset to compressed_size + data_offset.
* We need to do this because compaction/squishing will leave holes
* in the data that we need to account for during replay.
*/
db->val_size = compressed_size + data_offset;
ol_string_free(&read_org_size);
ol_string_free(&read_data_size);
ol_string_free(&ct);
ol_string_free(&value);
} else if (strncmp(command->data, "SCOOP", 5) == 0)
ol_scoop(db, key->data, key->dlen);
else if (strncmp(command->data, "SPOIL", 5) == 0) {
ol_string *spoil = _ol_read_data(fd);
check(spoil != NULL, "Could not read the rest of SPOIL command for AOL.");
struct tm time = {0};
_deserialize_time(&time, spoil->data);
check(spoil, "Error reading");
ol_spoil(db, key->data, key->dlen, &time);
ol_string_free(&spoil);
}
/* Strip the newline char after each "record" */
char c;
check(fread(&c, 1, 1, fd) != 0, "Error reading");
check(c == '\n', "Could not strip newline");
ol_string_free(&command);
ol_string_free(&key);
}
fclose(fd);
return 0;
error:
ol_log_msg(LOG_ERR, "Restore failed. Corrupt AOL?");
/* Free all the stuff */
ol_string_free(&command);
ol_string_free(&key);
ol_string_free(&value);
ol_string_free(&ct);
ol_string_free(&read_org_size);
ol_string_free(&read_data_size);
if (fd != NULL)
fclose(fd);
return -1;
}
