inline void flua_uncompress(BPMutableVector<Byte>* compressed, BPMutableVector<Byte>* out) { int numBytes = LZ4_uncompress_unknownOutputSize( compressed->_start, out->_start, compressed->_end - compressed->_start, // getLength out->_endOfStorage - out->_start // getSize ); out->_end = out->_start + numBytes; }
bool Lz4CompressionTcpSocket::decompressDataWithoutHeader(const QByteArray &source,QByteArray *destination,int *isize,int *osize) { int returnCode=LZ4_uncompress_unknownOutputSize(source.constData(),destination->data(),isize,*osize); if(returnCode<0) return false; else { *osize=returnCode; return true; } }
/*ARGSUSED*/ int lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { const char *src = s_start; uint32_t bufsiz = BE_IN32(src); /* invalid compressed buffer size encoded at start */ if (bufsiz + sizeof (bufsiz) > s_len) return (1); /* * Returns 0 on success (decompression function returned non-negative) * and non-zero on failure (decompression function returned negative. */ return (LZ4_uncompress_unknownOutputSize(&src[sizeof (bufsiz)], d_start, bufsiz, d_len) < 0); }
/* ARGSUSED */ static int lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int dummy __unused) { const uint8_t *src = s_start; uint32_t bufsiz = htonl(*(uint32_t *)src); /* invalid compressed buffer size encoded at start */ if (bufsiz + 4 > s_len) return (1); /* * Returns 0 on success (decompression function returned non-negative) * and non-zero on failure (decompression function returned negative). */ return (LZ4_uncompress_unknownOutputSize(s_start + 4, d_start, bufsiz, d_len) < 0); }
int lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len) { const uint8_t *src = s_start; uint32_t bufsiz = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3]; /* invalid compressed buffer size encoded at start */ if (bufsiz + 4 > s_len) return (1); /* * Returns 0 on success (decompression function returned non-negative) * and non-zero on failure (decompression function returned negative). */ return (LZ4_uncompress_unknownOutputSize(s_start + 4, d_start, bufsiz, d_len) < 0); }
/* * Class: net_jpountz_lz4_LZ4 * Method: LZ4_decompress_unknownOutputSize * Signature: ([BII[BI)I */ JNIEXPORT jint JNICALL Java_net_jpountz_lz4_LZ4JNI_LZ4_1decompress_1unknownOutputSize (JNIEnv *env, jclass cls, jbyteArray src, jint srcOff, jint srcLen, jbyteArray dest, jint destOff, jint maxDestLen) { char* in = (char*) (*env)->GetPrimitiveArrayCritical(env, src, 0); if (in == NULL) { throw_OOM(env); return 0; } char* out = (char*) (*env)->GetPrimitiveArrayCritical(env, dest, 0); if (out == NULL) { throw_OOM(env); return 0; } jint decompressed = LZ4_uncompress_unknownOutputSize(in + srcOff, out + destOff, srcLen, maxDestLen); (*env)->ReleasePrimitiveArrayCritical(env, src, in, 0); (*env)->ReleasePrimitiveArrayCritical(env, src, out, 0); return decompressed; }
JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_decompressBytesDirect (JNIEnv *env, jobject thisj){ const char *compressed_bytes; char *uncompressed_bytes; // Get members of Lz4Decompressor jobject clazz = (*env)->GetStaticObjectField(env,thisj, Lz4Decompressor_clazz); jobject compressed_direct_buf = (*env)->GetObjectField(env,thisj, Lz4Decompressor_compressedDirectBuf); jint compressed_direct_buf_len = (*env)->GetIntField(env,thisj, Lz4Decompressor_compressedDirectBufLen); jobject uncompressed_direct_buf = (*env)->GetObjectField(env,thisj, Lz4Decompressor_uncompressedDirectBuf); size_t uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, Lz4Decompressor_directBufferSize); // Get the input direct buffer LOCK_CLASS(env, clazz, "Lz4Decompressor"); compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); UNLOCK_CLASS(env, clazz, "Lz4Decompressor"); if (compressed_bytes == 0) { return (jint)0; } // Get the output direct buffer LOCK_CLASS(env, clazz, "Lz4Decompressor"); uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); UNLOCK_CLASS(env, clazz, "Lz4Decompressor"); if (uncompressed_bytes == 0) { return (jint)0; } uncompressed_direct_buf_len = LZ4_uncompress_unknownOutputSize(compressed_bytes, uncompressed_bytes, compressed_direct_buf_len, uncompressed_direct_buf_len); if (uncompressed_direct_buf_len < 0) { THROW(env, "java/lang/InternalError", "LZ4_uncompress_unknownOutputSize failed."); } (*env)->SetIntField(env, thisj, Lz4Decompressor_compressedDirectBufLen, 0); return (jint)uncompressed_direct_buf_len; }
static inline bool decompress(I begin,I end,O out) { volatile char i32c[4]; void *const i32cp = (void *)i32c; unsigned int bufLen = LZ4_compressBound(ZT_COMPRESSION_BLOCK_SIZE); char *buf = new char[bufLen * 2]; char *buf2 = buf + bufLen; try { I inp(begin); while (inp != end) { i32c[0] = (char)*inp; if (++inp == end) { delete [] buf; return false; } i32c[1] = (char)*inp; if (++inp == end) { delete [] buf; return false; } i32c[2] = (char)*inp; if (++inp == end) { delete [] buf; return false; } i32c[3] = (char)*inp; if (++inp == end) { delete [] buf; return false; } unsigned int originalSize = ntoh(*((const uint32_t *)i32cp)); i32c[0] = (char)*inp; if (++inp == end) { delete [] buf; return false; } i32c[1] = (char)*inp; if (++inp == end) { delete [] buf; return false; } i32c[2] = (char)*inp; if (++inp == end) { delete [] buf; return false; } i32c[3] = (char)*inp; if (++inp == end) { delete [] buf; return false; } uint32_t _compressedSize = ntoh(*((const uint32_t *)i32cp)); unsigned int compressedSize = _compressedSize & 0x7fffffff; if (compressedSize) { if (compressedSize > bufLen) { delete [] buf; return false; } unsigned int readLen = 0; while ((readLen < compressedSize)&&(inp != end)) { buf[readLen++] = (char)*inp; ++inp; } if (readLen != compressedSize) { delete [] buf; return false; } if (LZ4_uncompress_unknownOutputSize(buf,buf2,compressedSize,bufLen) != (int)originalSize) { delete [] buf; return false; } else out((const void *)buf2,(unsigned int)originalSize); } else { // stored if (originalSize > bufLen) { delete [] buf; return false; } unsigned int readLen = 0; while ((readLen < originalSize)&&(inp != end)) { buf[readLen++] = (char)*inp; ++inp; } if (readLen != originalSize) { delete [] buf; return false; } out((const void *)buf,(unsigned int)originalSize); } } delete [] buf; return true; } catch ( ... ) { delete [] buf; throw; } }
int decode_file(char* input_filename, char* output_filename) { U64 filesize = 0; char* in_buff; char* out_buff; char stdinout[] = "std"; FILE* finput; if (!strcmp (input_filename, stdinout)) { fprintf(stderr, "Using stdin for input\n"); finput = stdin; } else { finput = fopen( input_filename, "rb" ); } FILE* foutput; if (!strcmp (output_filename, stdinout)) { fprintf(stderr, "Using stdout for output\n"); foutput = stdout; } else { foutput = fopen( output_filename, "wb" ); } size_t uselessRet; int sinkint; U32 nextSize; if (finput==0 ) { printf("Pb opening %s\n", input_filename); return 4; } if (foutput==0) { printf("Pb opening %s\n", output_filename); return 5; } // Allocate Memory in_buff = malloc(OUT_CHUNKSIZE); out_buff = malloc(CHUNKSIZE); // Check Archive Header uselessRet = fread(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, finput); if (*(U32*)out_buff != ARCHIVE_MAGICNUMBER) { printf("Wrong file : cannot be decoded\n"); return 6; } uselessRet = fread(in_buff, 1, 4, finput); nextSize = *(U32*)in_buff; // Main Loop while (1) { // Read Block uselessRet = fread(in_buff, 1, nextSize, finput); // Check Next Block uselessRet = (U32) fread(&nextSize, 1, 4, finput); if( uselessRet==0 ) break; // Decode Block sinkint = LZ4_uncompress(in_buff, out_buff, CHUNKSIZE); filesize += CHUNKSIZE; // Write Block fwrite(out_buff, 1, CHUNKSIZE, foutput); } // Last Block uselessRet = fread(in_buff, 1, nextSize, finput); sinkint = LZ4_uncompress_unknownOutputSize(in_buff, out_buff, nextSize, CHUNKSIZE); filesize += sinkint; fwrite(out_buff, 1, sinkint, foutput); // Status fprintf(stderr, "Successfully decoded %llu bytes \n", (unsigned long long)filesize); fclose(finput); fclose(foutput); return 0; }
/* decompression backend for LZ4 */ static int lz4_backend_decompress(const void* input, int length, void* output, int maxout) { return LZ4_uncompress_unknownOutputSize(input, output, length, maxout); /* return LZ4_uncompress(input, output, maxout); */ }
void teDecodeFile(const teString & from, const teString & to, u32 chunkSize, c8 * chunkInputBuffer, u32 chunkInputBufferSize, c8 * chunkOutputBuffer, u32 chunkOutputBufferSize, u1 localPath) { TE_ASSERT(chunkInputBufferSize >= LZ4_compressBound(chunkSize)); TE_ASSERT(chunkOutputBufferSize >= chunkSize); IBuffer * fileInput = GetFileManager()->OpenFile(from, CFileBuffer::FWM_READ, localPath); if(!fileInput) return; IBuffer * fileOutput = GetFileManager()->OpenFile(to, CFileBuffer::FWM_WRITE, localPath); if(!fileOutput) { TE_SAFE_DROP(fileInput); return; } fileInput->SetStreamMode(true); fileOutput->SetStreamMode(true); fileInput->Lock(BLT_READ); fileInput->SetPosition(0); u32 magicNumber = u32Max; fileInput->Read(&magicNumber, ARCHIVE_MAGICNUMBER_SIZE); TE_ASSERT(magicNumber == ARCHIVE_MAGICNUMBER); fileOutput->Lock(BLT_WRITE); fileOutput->SetPosition(0); c8 * inBuffer = chunkInputBuffer; c8 * outBuffer = chunkOutputBuffer; while(true) { if((fileInput->GetSize() - fileInput->GetPosition()) < ARCHIVE_MAGICNUMBER_SIZE) break; u32 chunkSizeInFile; fileInput->Read(&chunkSizeInFile, sizeof(u32)); if(chunkSizeInFile == ARCHIVE_MAGICNUMBER) continue; fileInput->Read(inBuffer, chunkSizeInFile); s32 outputSize = LZ4_uncompress_unknownOutputSize(inBuffer, outBuffer, chunkSizeInFile, chunkSize); if(outputSize < 0) { TE_LOG_ERR("lz4 decoding failed, corrupted input\n"); fileInput->Unlock(); fileOutput->Unlock(); TE_SAFE_DROP(fileInput); TE_SAFE_DROP(fileOutput); return; } fileOutput->Write(outBuffer, (u32)outputSize); } fileInput->Unlock(); fileOutput->Unlock(); TE_SAFE_DROP(fileInput); TE_SAFE_DROP(fileOutput); }
void teDecodeData(c8 * input, u32 inputSize, c8 * output, u32 outputSize, u32 & resultSize) { resultSize = LZ4_uncompress_unknownOutputSize(input + 4, output, *((u32*)input + 0), outputSize); }
BOOL CNktDvDbMerge::LoadInputDbs() { CDbMergeDvDatabase *lpDb; HRESULT hRes; TNktAutoFreePtr<BYTE> aCompressedData; HANDLE hFile = NULL; SIZE_T i, k, nCount, nCompressedSize; Fnv64_t nHashValue, nStoredHash; DWORD dw; LPBYTE s; PrintAndLog(L"Loading input databases... "); for (k=0; k<aInputDatabases.GetCount(); k++) { lpDb = new CDbMergeDvDatabase; if (lpDb == NULL) { err_nomem: if (hFile != NULL) ::CloseHandle(hFile); PrintAndLogNoMemoryError(); return FALSE; } if (aInputDbs.AddElement(lpDb) == FALSE) { delete lpDb; goto err_nomem; } //open database hFile = ::CreateFileW(aInputDatabases[k], GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile == NULL || hFile == INVALID_HANDLE_VALUE) { hFile = NULL; dbload_err: hRes = NKT_HRESULT_FROM_LASTERROR(); dbload_err2: if (hRes == E_OUTOFMEMORY) goto err_nomem; PrintAndLog(L"\nError: Cannot load database [%s] (%08X).\n", aInputDatabases[k], (ULONG)hRes); return FALSE; } //load into memory nCompressedSize = (SIZE_T)::GetFileSize(hFile, NULL); if (nCompressedSize <= (17 + sizeof(ULONG) + sizeof(Fnv64_t)) || nCompressedSize >= 0x0FFFFFFF) { dbload_err_fail: hRes = E_FAIL; goto dbload_err2; } aCompressedData.Reset(); aCompressedData.Attach((LPBYTE)nktMemMalloc(nCompressedSize)); if (aCompressedData == NULL) goto err_nomem; if (::ReadFile(hFile, aCompressedData.Get(), (DWORD)nCompressedSize, &dw, NULL) == FALSE) goto dbload_err; if (nCompressedSize != (SIZE_T)dw) goto dbload_err; nCompressedSize -= (17 + sizeof(ULONG) + sizeof(Fnv64_t)); //check signature and hash s = aCompressedData.Get(); if (nktMemCompare(s, "Deviare Database\x1A", 17) != 0) goto dbload_err_fail; s += 17; dw = *((ULONG NKT_UNALIGNED *)s); lpDb->nDecompressedSize = (SIZE_T)dw; s += sizeof(ULONG); nStoredHash = *((Fnv64_t NKT_UNALIGNED *)s); s += sizeof(Fnv64_t); nHashValue = fnv_64a_buf(&dw, sizeof(dw), FNV1A_64_INIT); nHashValue = fnv_64a_buf(s, nCompressedSize, nHashValue); if (nHashValue != nStoredHash) goto dbload_err_fail; //hash mismatch //create shared memory for decompressed data lpDb->aDecompressedData.Attach((LPBYTE)nktMemMalloc(lpDb->nDecompressedSize)); if (lpDb->aDecompressedData == NULL) goto err_nomem; try { i = (SIZE_T)LZ4_uncompress_unknownOutputSize((char*)s, (char*)(lpDb->aDecompressedData.Get()), (int)nCompressedSize, (int)(lpDb->nDecompressedSize)); } catch (...) { i = NKT_SIZE_T_MAX; //force fail on error } if (lpDb->nDecompressedSize != i) goto dbload_err_fail; //close file ::CloseHandle(hFile); hFile = NULL; //initialize internal pointers s = lpDb->aDecompressedData.Get(); //read super string nktMemCopy(&dw, s, sizeof(dw)); s += sizeof(dw); lpDb->szSuperStringW = (LPWSTR)s; lpDb->nSuperStringSize = (SIZE_T)dw; s += lpDb->nSuperStringSize; //read DbObjects count nktMemCopy(&dw, s, sizeof(dw)); s += sizeof(dw); lpDb->nDbObjectsListCount = (SIZE_T)dw; if (lpDb->nDbObjectsListCount == 0) { hRes = NKT_DVERR_CannotLoadDatabase; goto dbload_err2; } lpDb->aStoredDbObjects.Attach((CNktDvDbObjectNoRef::LPSTORED_DATA*)nktMemMalloc( lpDb->nDbObjectsListCount * sizeof(CNktDvDbObjectNoRef::LPSTORED_DATA))); lpDb->aStoredDbObjects_ChildsCount.Attach((SIZE_T*)nktMemMalloc(lpDb->nDbObjectsListCount * sizeof(SIZE_T))); lpDb->aStoredDbObjects_FirstChild.Attach((CNktDvDbObjectNoRef::LPSTORED_CHILD_ITEM*)nktMemMalloc( lpDb->nDbObjectsListCount * sizeof(CNktDvDbObjectNoRef::LPSTORED_CHILD_ITEM))); if (lpDb->aStoredDbObjects == NULL || lpDb->aStoredDbObjects_ChildsCount == NULL || lpDb->aStoredDbObjects_FirstChild == NULL) goto err_nomem; //count DbObject childs for (i=nCount=0; i<lpDb->nDbObjectsListCount; i++) { lpDb->aStoredDbObjects[i] = (CNktDvDbObjectNoRef::LPSTORED_DATA)s; s += sizeof(CNktDvDbObjectNoRef::STORED_DATA); //read DbObject childs count dw = *((DWORD NKT_UNALIGNED *)s); s += sizeof(dw); lpDb->aStoredDbObjects_ChildsCount[i] = (SIZE_T)dw; nCount += (SIZE_T)dw; lpDb->aStoredDbObjects_FirstChild[i] = (CNktDvDbObjectNoRef::LPSTORED_CHILD_ITEM)s; s += (SIZE_T)dw * sizeof(CNktDvDbObjectNoRef::STORED_CHILD_ITEM); } /* //process DbObject's for (i=nCount=0; i<nDbObjectsListCount; i++) { { aDbObjectChildsList[nCount].lpStoredChildItem = (CNktDvDbObjectNoRef::LPSTORED_CHILD_ITEM)s; s += sizeof(CNktDvDbObjectNoRef::STORED_CHILD_ITEM); aDbObjectChildsList[nCount].szNameW = szSuperStringW + (SIZE_T)(aDbObjectChildsList[nCount].lpStoredChildItem->nNameOffset); nCount++; dw--; } } //replace child object ids with pointers for (i=0; i<nDbObjectsListCount; i++) { nCount = lpDbObjectsList[i].nChildsCount; for (j=0; j<nCount; j++) { nDbObjId = lpDbObjectsList[i].lpFirstChild[j].lpStoredChildItem->nDbObjectId; if (nDbObjId != 0) { //search object lpDbObj = (CNktDvDbObjectNoRef*)bsearch_s(&nDbObjId, lpDbObjectsList, nDbObjectsListCount, sizeof(CNktDvDbObjectNoRef), SearchDbObjectById, NULL); if (lpDbObj == NULL) goto init_err_dbloadfail; } else { lpDbObj = NULL; } lpDbObjectsList[i].lpFirstChild[j].lpObject = lpDbObj; } } */ //read DB_MODULEs count dw = *((DWORD NKT_UNALIGNED *)s); s += sizeof(dw); lpDb->nDbModulesListCount = (SIZE_T)dw; if (lpDb->nDbModulesListCount == 0) { hRes = NKT_DVERR_CannotLoadDatabase; goto dbload_err2; } lpDb->aStoredDbModules.Attach((CNktDvDbObjectNoRef::LPSTORED_MODULE*)nktMemMalloc( lpDb->nDbModulesListCount * sizeof(CNktDvDbObjectNoRef::LPSTORED_MODULE))); if (lpDb->aStoredDbModules == NULL) goto err_nomem; for (i=0; i<lpDb->nDbModulesListCount; i++) { lpDb->aStoredDbModules[i] = (CNktDvDbObjectNoRef::LPSTORED_MODULE)s; s += sizeof(CNktDvDbObjectNoRef::STORED_MODULE); } //read DB_MODULE_FUNCTIONs count dw = *((DWORD NKT_UNALIGNED *)s); s += sizeof(dw); lpDb->nDbModuleFunctionsCount = (SIZE_T)dw; if (lpDb->nDbModuleFunctionsCount > 0) { lpDb->aStoredDbModuleFunctions.Attach((ULONG*)nktMemMalloc(lpDb->nDbModuleFunctionsCount * 2 * sizeof(ULONG))); if (lpDb->aStoredDbModuleFunctions == NULL) goto err_nomem; } for (i=0; i<lpDb->nDbModuleFunctionsCount; i++) { //read DB_MODULE_FUNCTION lpDb->aStoredDbModuleFunctions[i<<1] = *((ULONG NKT_UNALIGNED *)s); s += sizeof(ULONG); lpDb->aStoredDbModuleFunctions[(i<<1)+1] = *((ULONG NKT_UNALIGNED *)s); s += sizeof(ULONG); } /* //sort modules by name cDbModuleNameIndex.Attach((ULONG*)nktMemMalloc(nDbModulesListCount*sizeof(ULONG))); if (cDbModuleNameIndex == NULL) goto err_nomem; for (i=0; i<nDbModulesListCount; i++) cDbModuleNameIndex[i] = (ULONG)i; qsort_s(cDbModuleNameIndex.Get(), nDbModulesListCount, sizeof(ULONG), SortDbModuleByName, lpDbModulesList); //sort module functions by name for (i=0; i<nDbModulesListCount; i++) { qsort_s(lpDbModulesList[i].aFunctionsList.GetBuffer(), lpDbModulesList[i].aFunctionsList.GetCount(), sizeof(CNktDvDbObjectNoRef*), SortDbModuleFunctionsByName, NULL); } //sort objects by name cDbObjectNameIndex.Attach((ULONG*)nktMemMalloc(nDbObjectsListCount*sizeof(ULONG))); if (cDbObjectNameIndex == NULL) goto err_nomem; for (i=0; i<nDbObjectsListCount; i++) cDbObjectNameIndex[i] = (ULONG)i; qsort_s(cDbObjectNameIndex.Get(), nDbObjectsListCount, sizeof(ULONG), SortDbObjectByName, lpDbObjectsList); //sort objects by class and name cDbObjectClassAndNameIndex.Attach((ULONG*)nktMemMalloc(nDbObjectsListCount*sizeof(ULONG))); if (cDbObjectClassAndNameIndex == NULL) goto err_nomem; for (i=0; i<nDbObjectsListCount; i++) cDbObjectClassAndNameIndex[i] = (ULONG)i; qsort_s(cDbObjectClassAndNameIndex.Get(), nDbObjectsListCount, sizeof(ULONG), SortDbObjectByClassAndName, lpDbObjectsList); //get the first item of each class nktMemSet(&sDbObjectClassPos, 0, sizeof(sDbObjectClassPos)); nLastClass = -1; for (i=0; i<nDbObjectsListCount; i++) { nCurrClass = (int)(lpDbObjectsList[cDbObjectClassAndNameIndex[i]].lpStoredData->nClass); if (nCurrClass != nLastClass) { sDbObjectClassPos[nCurrClass+1].nStart = (ULONG)i; sDbObjectClassPos[nCurrClass+1].nCount = 1; nLastClass = nCurrClass; } else { sDbObjectClassPos[nLastClass+1].nCount++; } } */ } PrintAndLog(L"OK\n"); return TRUE; }