inline void flua_uncompress(BPMutableVector<Byte>* compressed, BPMutableVector<Byte>* out) {
int numBytes = LZ4_uncompress_unknownOutputSize(
compressed->_start,
out->_start,
compressed->_end - compressed->_start, // getLength
out->_endOfStorage - out->_start // getSize
);
out->_end = out->_start + numBytes;
}
bool Lz4CompressionTcpSocket::decompressDataWithoutHeader(const QByteArray &source,QByteArray *destination,int *isize,int *osize)
{
int returnCode=LZ4_uncompress_unknownOutputSize(source.constData(),destination->data(),isize,*osize);
if(returnCode<0)
return false;
else
{
*osize=returnCode;
return true;
}
}
/*ARGSUSED*/
int
lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n)
{
const char *src = s_start;
uint32_t bufsiz = BE_IN32(src);
/* invalid compressed buffer size encoded at start */
if (bufsiz + sizeof (bufsiz) > s_len)
return (1);
/*
* Returns 0 on success (decompression function returned non-negative)
* and non-zero on failure (decompression function returned negative.
*/
return (LZ4_uncompress_unknownOutputSize(&src[sizeof (bufsiz)],
d_start, bufsiz, d_len) < 0);
}
/* ARGSUSED */
static int
lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int dummy __unused)
{
const uint8_t *src = s_start;
uint32_t bufsiz = htonl(*(uint32_t *)src);
/* invalid compressed buffer size encoded at start */
if (bufsiz + 4 > s_len)
return (1);
/*
* Returns 0 on success (decompression function returned non-negative)
* and non-zero on failure (decompression function returned negative).
*/
return (LZ4_uncompress_unknownOutputSize(s_start + 4, d_start, bufsiz,
d_len) < 0);
}
int
lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len)
{
const uint8_t *src = s_start;
uint32_t bufsiz = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) |
src[3];
/* invalid compressed buffer size encoded at start */
if (bufsiz + 4 > s_len)
return (1);
/*
* Returns 0 on success (decompression function returned non-negative)
* and non-zero on failure (decompression function returned negative).
*/
return (LZ4_uncompress_unknownOutputSize(s_start + 4, d_start, bufsiz,
d_len) < 0);
}
/*
* Class: net_jpountz_lz4_LZ4
* Method: LZ4_decompress_unknownOutputSize
* Signature: ([BII[BI)I
*/
JNIEXPORT jint JNICALL Java_net_jpountz_lz4_LZ4JNI_LZ4_1decompress_1unknownOutputSize
(JNIEnv *env, jclass cls, jbyteArray src, jint srcOff, jint srcLen, jbyteArray dest, jint destOff, jint maxDestLen) {
char* in = (char*) (*env)->GetPrimitiveArrayCritical(env, src, 0);
if (in == NULL) {
throw_OOM(env);
return 0;
}
char* out = (char*) (*env)->GetPrimitiveArrayCritical(env, dest, 0);
if (out == NULL) {
throw_OOM(env);
return 0;
}
jint decompressed = LZ4_uncompress_unknownOutputSize(in + srcOff, out + destOff, srcLen, maxDestLen);
(*env)->ReleasePrimitiveArrayCritical(env, src, in, 0);
(*env)->ReleasePrimitiveArrayCritical(env, src, out, 0);
return decompressed;
}
JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_decompressBytesDirect
(JNIEnv *env, jobject thisj){
const char *compressed_bytes;
char *uncompressed_bytes;
// Get members of Lz4Decompressor
jobject clazz = (*env)->GetStaticObjectField(env,thisj, Lz4Decompressor_clazz);
jobject compressed_direct_buf = (*env)->GetObjectField(env,thisj, Lz4Decompressor_compressedDirectBuf);
jint compressed_direct_buf_len = (*env)->GetIntField(env,thisj, Lz4Decompressor_compressedDirectBufLen);
jobject uncompressed_direct_buf = (*env)->GetObjectField(env,thisj, Lz4Decompressor_uncompressedDirectBuf);
size_t uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, Lz4Decompressor_directBufferSize);
// Get the input direct buffer
LOCK_CLASS(env, clazz, "Lz4Decompressor");
compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
UNLOCK_CLASS(env, clazz, "Lz4Decompressor");
if (compressed_bytes == 0) {
return (jint)0;
}
// Get the output direct buffer
LOCK_CLASS(env, clazz, "Lz4Decompressor");
uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
UNLOCK_CLASS(env, clazz, "Lz4Decompressor");
if (uncompressed_bytes == 0) {
return (jint)0;
}
uncompressed_direct_buf_len = LZ4_uncompress_unknownOutputSize(compressed_bytes, uncompressed_bytes, compressed_direct_buf_len, uncompressed_direct_buf_len);
if (uncompressed_direct_buf_len < 0) {
THROW(env, "java/lang/InternalError", "LZ4_uncompress_unknownOutputSize failed.");
}
(*env)->SetIntField(env, thisj, Lz4Decompressor_compressedDirectBufLen, 0);
return (jint)uncompressed_direct_buf_len;
}
static inline bool decompress(I begin,I end,O out)
{
volatile char i32c[4];
void *const i32cp = (void *)i32c;
unsigned int bufLen = LZ4_compressBound(ZT_COMPRESSION_BLOCK_SIZE);
char *buf = new char[bufLen * 2];
char *buf2 = buf + bufLen;
try {
I inp(begin);
while (inp != end) {
i32c[0] = (char)*inp; if (++inp == end) { delete [] buf; return false; }
i32c[1] = (char)*inp; if (++inp == end) { delete [] buf; return false; }
i32c[2] = (char)*inp; if (++inp == end) { delete [] buf; return false; }
i32c[3] = (char)*inp; if (++inp == end) { delete [] buf; return false; }
unsigned int originalSize = ntoh(*((const uint32_t *)i32cp));
i32c[0] = (char)*inp; if (++inp == end) { delete [] buf; return false; }
i32c[1] = (char)*inp; if (++inp == end) { delete [] buf; return false; }
i32c[2] = (char)*inp; if (++inp == end) { delete [] buf; return false; }
i32c[3] = (char)*inp; if (++inp == end) { delete [] buf; return false; }
uint32_t _compressedSize = ntoh(*((const uint32_t *)i32cp));
unsigned int compressedSize = _compressedSize & 0x7fffffff;
if (compressedSize) {
if (compressedSize > bufLen) {
delete [] buf;
return false;
}
unsigned int readLen = 0;
while ((readLen < compressedSize)&&(inp != end)) {
buf[readLen++] = (char)*inp;
++inp;
}
if (readLen != compressedSize) {
delete [] buf;
return false;
}
if (LZ4_uncompress_unknownOutputSize(buf,buf2,compressedSize,bufLen) != (int)originalSize) {
delete [] buf;
return false;
} else out((const void *)buf2,(unsigned int)originalSize);
} else { // stored
if (originalSize > bufLen) {
delete [] buf;
return false;
}
unsigned int readLen = 0;
while ((readLen < originalSize)&&(inp != end)) {
buf[readLen++] = (char)*inp;
++inp;
}
if (readLen != originalSize) {
delete [] buf;
return false;
}
out((const void *)buf,(unsigned int)originalSize);
}
}
delete [] buf;
return true;
} catch ( ... ) {
delete [] buf;
throw;
}
}
int decode_file(char* input_filename, char* output_filename)
{
U64 filesize = 0;
char* in_buff;
char* out_buff;
char stdinout[] = "std";
FILE* finput;
if (!strcmp (input_filename, stdinout)) {
fprintf(stderr, "Using stdin for input\n");
finput = stdin;
} else {
finput = fopen( input_filename, "rb" );
}
FILE* foutput;
if (!strcmp (output_filename, stdinout)) {
fprintf(stderr, "Using stdout for output\n");
foutput = stdout;
} else {
foutput = fopen( output_filename, "wb" );
}
size_t uselessRet;
int sinkint;
U32 nextSize;
if (finput==0 ) { printf("Pb opening %s\n", input_filename); return 4; }
if (foutput==0) { printf("Pb opening %s\n", output_filename); return 5; }
// Allocate Memory
in_buff = malloc(OUT_CHUNKSIZE);
out_buff = malloc(CHUNKSIZE);
// Check Archive Header
uselessRet = fread(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, finput);
if (*(U32*)out_buff != ARCHIVE_MAGICNUMBER) { printf("Wrong file : cannot be decoded\n"); return 6; }
uselessRet = fread(in_buff, 1, 4, finput);
nextSize = *(U32*)in_buff;
// Main Loop
while (1)
{
// Read Block
uselessRet = fread(in_buff, 1, nextSize, finput);
// Check Next Block
uselessRet = (U32) fread(&nextSize, 1, 4, finput);
if( uselessRet==0 ) break;
// Decode Block
sinkint = LZ4_uncompress(in_buff, out_buff, CHUNKSIZE);
filesize += CHUNKSIZE;
// Write Block
fwrite(out_buff, 1, CHUNKSIZE, foutput);
}
// Last Block
uselessRet = fread(in_buff, 1, nextSize, finput);
sinkint = LZ4_uncompress_unknownOutputSize(in_buff, out_buff, nextSize, CHUNKSIZE);
filesize += sinkint;
fwrite(out_buff, 1, sinkint, foutput);
// Status
fprintf(stderr, "Successfully decoded %llu bytes \n", (unsigned long long)filesize);
fclose(finput);
fclose(foutput);
return 0;
}
/* decompression backend for LZ4 */
static int lz4_backend_decompress(const void* input, int length, void* output,
int maxout) {
return LZ4_uncompress_unknownOutputSize(input, output, length, maxout);
/* return LZ4_uncompress(input, output, maxout); */
}
void teDecodeFile(const teString & from, const teString & to, u32 chunkSize, c8 * chunkInputBuffer, u32 chunkInputBufferSize, c8 * chunkOutputBuffer, u32 chunkOutputBufferSize, u1 localPath)
{
TE_ASSERT(chunkInputBufferSize >= LZ4_compressBound(chunkSize));
TE_ASSERT(chunkOutputBufferSize >= chunkSize);
IBuffer * fileInput = GetFileManager()->OpenFile(from, CFileBuffer::FWM_READ, localPath);
if(!fileInput)
return;
IBuffer * fileOutput = GetFileManager()->OpenFile(to, CFileBuffer::FWM_WRITE, localPath);
if(!fileOutput)
{
TE_SAFE_DROP(fileInput);
return;
}
fileInput->SetStreamMode(true);
fileOutput->SetStreamMode(true);
fileInput->Lock(BLT_READ);
fileInput->SetPosition(0);
u32 magicNumber = u32Max;
fileInput->Read(&magicNumber, ARCHIVE_MAGICNUMBER_SIZE);
TE_ASSERT(magicNumber == ARCHIVE_MAGICNUMBER);
fileOutput->Lock(BLT_WRITE);
fileOutput->SetPosition(0);
c8 * inBuffer = chunkInputBuffer;
c8 * outBuffer = chunkOutputBuffer;
while(true)
{
if((fileInput->GetSize() - fileInput->GetPosition()) < ARCHIVE_MAGICNUMBER_SIZE)
break;
u32 chunkSizeInFile;
fileInput->Read(&chunkSizeInFile, sizeof(u32));
if(chunkSizeInFile == ARCHIVE_MAGICNUMBER)
continue;
fileInput->Read(inBuffer, chunkSizeInFile);
s32 outputSize = LZ4_uncompress_unknownOutputSize(inBuffer, outBuffer, chunkSizeInFile, chunkSize);
if(outputSize < 0)
{
TE_LOG_ERR("lz4 decoding failed, corrupted input\n");
fileInput->Unlock();
fileOutput->Unlock();
TE_SAFE_DROP(fileInput);
TE_SAFE_DROP(fileOutput);
return;
}
fileOutput->Write(outBuffer, (u32)outputSize);
}
fileInput->Unlock();
fileOutput->Unlock();
TE_SAFE_DROP(fileInput);
TE_SAFE_DROP(fileOutput);
}
void teDecodeData(c8 * input, u32 inputSize, c8 * output, u32 outputSize, u32 & resultSize)
{
resultSize = LZ4_uncompress_unknownOutputSize(input + 4, output, *((u32*)input + 0), outputSize);
}
BOOL CNktDvDbMerge::LoadInputDbs()
{
CDbMergeDvDatabase *lpDb;
HRESULT hRes;
TNktAutoFreePtr<BYTE> aCompressedData;
HANDLE hFile = NULL;
SIZE_T i, k, nCount, nCompressedSize;
Fnv64_t nHashValue, nStoredHash;
DWORD dw;
LPBYTE s;
PrintAndLog(L"Loading input databases... ");
for (k=0; k<aInputDatabases.GetCount(); k++)
{
lpDb = new CDbMergeDvDatabase;
if (lpDb == NULL)
{
err_nomem:
if (hFile != NULL)
::CloseHandle(hFile);
PrintAndLogNoMemoryError();
return FALSE;
}
if (aInputDbs.AddElement(lpDb) == FALSE)
{
delete lpDb;
goto err_nomem;
}
//open database
hFile = ::CreateFileW(aInputDatabases[k], GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL, NULL);
if (hFile == NULL || hFile == INVALID_HANDLE_VALUE)
{
hFile = NULL;
dbload_err:
hRes = NKT_HRESULT_FROM_LASTERROR();
dbload_err2:
if (hRes == E_OUTOFMEMORY)
goto err_nomem;
PrintAndLog(L"\nError: Cannot load database [%s] (%08X).\n", aInputDatabases[k], (ULONG)hRes);
return FALSE;
}
//load into memory
nCompressedSize = (SIZE_T)::GetFileSize(hFile, NULL);
if (nCompressedSize <= (17 + sizeof(ULONG) + sizeof(Fnv64_t)) ||
nCompressedSize >= 0x0FFFFFFF)
{
dbload_err_fail:
hRes = E_FAIL;
goto dbload_err2;
}
aCompressedData.Reset();
aCompressedData.Attach((LPBYTE)nktMemMalloc(nCompressedSize));
if (aCompressedData == NULL)
goto err_nomem;
if (::ReadFile(hFile, aCompressedData.Get(), (DWORD)nCompressedSize, &dw, NULL) == FALSE)
goto dbload_err;
if (nCompressedSize != (SIZE_T)dw)
goto dbload_err;
nCompressedSize -= (17 + sizeof(ULONG) + sizeof(Fnv64_t));
//check signature and hash
s = aCompressedData.Get();
if (nktMemCompare(s, "Deviare Database\x1A", 17) != 0)
goto dbload_err_fail;
s += 17;
dw = *((ULONG NKT_UNALIGNED *)s);
lpDb->nDecompressedSize = (SIZE_T)dw;
s += sizeof(ULONG);
nStoredHash = *((Fnv64_t NKT_UNALIGNED *)s);
s += sizeof(Fnv64_t);
nHashValue = fnv_64a_buf(&dw, sizeof(dw), FNV1A_64_INIT);
nHashValue = fnv_64a_buf(s, nCompressedSize, nHashValue);
if (nHashValue != nStoredHash)
goto dbload_err_fail; //hash mismatch
//create shared memory for decompressed data
lpDb->aDecompressedData.Attach((LPBYTE)nktMemMalloc(lpDb->nDecompressedSize));
if (lpDb->aDecompressedData == NULL)
goto err_nomem;
try
{
i = (SIZE_T)LZ4_uncompress_unknownOutputSize((char*)s, (char*)(lpDb->aDecompressedData.Get()),
(int)nCompressedSize, (int)(lpDb->nDecompressedSize));
}
catch (...)
{
i = NKT_SIZE_T_MAX; //force fail on error
}
if (lpDb->nDecompressedSize != i)
goto dbload_err_fail;
//close file
::CloseHandle(hFile);
hFile = NULL;
//initialize internal pointers
s = lpDb->aDecompressedData.Get();
//read super string
nktMemCopy(&dw, s, sizeof(dw));
s += sizeof(dw);
lpDb->szSuperStringW = (LPWSTR)s;
lpDb->nSuperStringSize = (SIZE_T)dw;
s += lpDb->nSuperStringSize;
//read DbObjects count
nktMemCopy(&dw, s, sizeof(dw));
s += sizeof(dw);
lpDb->nDbObjectsListCount = (SIZE_T)dw;
if (lpDb->nDbObjectsListCount == 0)
{
hRes = NKT_DVERR_CannotLoadDatabase;
goto dbload_err2;
}
lpDb->aStoredDbObjects.Attach((CNktDvDbObjectNoRef::LPSTORED_DATA*)nktMemMalloc(
lpDb->nDbObjectsListCount * sizeof(CNktDvDbObjectNoRef::LPSTORED_DATA)));
lpDb->aStoredDbObjects_ChildsCount.Attach((SIZE_T*)nktMemMalloc(lpDb->nDbObjectsListCount *
sizeof(SIZE_T)));
lpDb->aStoredDbObjects_FirstChild.Attach((CNktDvDbObjectNoRef::LPSTORED_CHILD_ITEM*)nktMemMalloc(
lpDb->nDbObjectsListCount * sizeof(CNktDvDbObjectNoRef::LPSTORED_CHILD_ITEM)));
if (lpDb->aStoredDbObjects == NULL || lpDb->aStoredDbObjects_ChildsCount == NULL ||
lpDb->aStoredDbObjects_FirstChild == NULL)
goto err_nomem;
//count DbObject childs
for (i=nCount=0; i<lpDb->nDbObjectsListCount; i++)
{
lpDb->aStoredDbObjects[i] = (CNktDvDbObjectNoRef::LPSTORED_DATA)s;
s += sizeof(CNktDvDbObjectNoRef::STORED_DATA);
//read DbObject childs count
dw = *((DWORD NKT_UNALIGNED *)s);
s += sizeof(dw);
lpDb->aStoredDbObjects_ChildsCount[i] = (SIZE_T)dw;
nCount += (SIZE_T)dw;
lpDb->aStoredDbObjects_FirstChild[i] = (CNktDvDbObjectNoRef::LPSTORED_CHILD_ITEM)s;
s += (SIZE_T)dw * sizeof(CNktDvDbObjectNoRef::STORED_CHILD_ITEM);
}
/*
//process DbObject's
for (i=nCount=0; i<nDbObjectsListCount; i++)
{
{
aDbObjectChildsList[nCount].lpStoredChildItem = (CNktDvDbObjectNoRef::LPSTORED_CHILD_ITEM)s;
s += sizeof(CNktDvDbObjectNoRef::STORED_CHILD_ITEM);
aDbObjectChildsList[nCount].szNameW = szSuperStringW +
(SIZE_T)(aDbObjectChildsList[nCount].lpStoredChildItem->nNameOffset);
nCount++;
dw--;
}
}
//replace child object ids with pointers
for (i=0; i<nDbObjectsListCount; i++)
{
nCount = lpDbObjectsList[i].nChildsCount;
for (j=0; j<nCount; j++)
{
nDbObjId = lpDbObjectsList[i].lpFirstChild[j].lpStoredChildItem->nDbObjectId;
if (nDbObjId != 0)
{
//search object
lpDbObj = (CNktDvDbObjectNoRef*)bsearch_s(&nDbObjId, lpDbObjectsList, nDbObjectsListCount,
sizeof(CNktDvDbObjectNoRef), SearchDbObjectById, NULL);
if (lpDbObj == NULL)
goto init_err_dbloadfail;
}
else
{
lpDbObj = NULL;
}
lpDbObjectsList[i].lpFirstChild[j].lpObject = lpDbObj;
}
}
*/
//read DB_MODULEs count
dw = *((DWORD NKT_UNALIGNED *)s);
s += sizeof(dw);
lpDb->nDbModulesListCount = (SIZE_T)dw;
if (lpDb->nDbModulesListCount == 0)
{
hRes = NKT_DVERR_CannotLoadDatabase;
goto dbload_err2;
}
lpDb->aStoredDbModules.Attach((CNktDvDbObjectNoRef::LPSTORED_MODULE*)nktMemMalloc(
lpDb->nDbModulesListCount * sizeof(CNktDvDbObjectNoRef::LPSTORED_MODULE)));
if (lpDb->aStoredDbModules == NULL)
goto err_nomem;
for (i=0; i<lpDb->nDbModulesListCount; i++)
{
lpDb->aStoredDbModules[i] = (CNktDvDbObjectNoRef::LPSTORED_MODULE)s;
s += sizeof(CNktDvDbObjectNoRef::STORED_MODULE);
}
//read DB_MODULE_FUNCTIONs count
dw = *((DWORD NKT_UNALIGNED *)s);
s += sizeof(dw);
lpDb->nDbModuleFunctionsCount = (SIZE_T)dw;
if (lpDb->nDbModuleFunctionsCount > 0)
{
lpDb->aStoredDbModuleFunctions.Attach((ULONG*)nktMemMalloc(lpDb->nDbModuleFunctionsCount * 2 *
sizeof(ULONG)));
if (lpDb->aStoredDbModuleFunctions == NULL)
goto err_nomem;
}
for (i=0; i<lpDb->nDbModuleFunctionsCount; i++)
{
//read DB_MODULE_FUNCTION
lpDb->aStoredDbModuleFunctions[i<<1] = *((ULONG NKT_UNALIGNED *)s);
s += sizeof(ULONG);
lpDb->aStoredDbModuleFunctions[(i<<1)+1] = *((ULONG NKT_UNALIGNED *)s);
s += sizeof(ULONG);
}
/*
//sort modules by name
cDbModuleNameIndex.Attach((ULONG*)nktMemMalloc(nDbModulesListCount*sizeof(ULONG)));
if (cDbModuleNameIndex == NULL)
goto err_nomem;
for (i=0; i<nDbModulesListCount; i++)
cDbModuleNameIndex[i] = (ULONG)i;
qsort_s(cDbModuleNameIndex.Get(), nDbModulesListCount, sizeof(ULONG), SortDbModuleByName,
lpDbModulesList);
//sort module functions by name
for (i=0; i<nDbModulesListCount; i++)
{
qsort_s(lpDbModulesList[i].aFunctionsList.GetBuffer(), lpDbModulesList[i].aFunctionsList.GetCount(),
sizeof(CNktDvDbObjectNoRef*), SortDbModuleFunctionsByName, NULL);
}
//sort objects by name
cDbObjectNameIndex.Attach((ULONG*)nktMemMalloc(nDbObjectsListCount*sizeof(ULONG)));
if (cDbObjectNameIndex == NULL)
goto err_nomem;
for (i=0; i<nDbObjectsListCount; i++)
cDbObjectNameIndex[i] = (ULONG)i;
qsort_s(cDbObjectNameIndex.Get(), nDbObjectsListCount, sizeof(ULONG), SortDbObjectByName,
lpDbObjectsList);
//sort objects by class and name
cDbObjectClassAndNameIndex.Attach((ULONG*)nktMemMalloc(nDbObjectsListCount*sizeof(ULONG)));
if (cDbObjectClassAndNameIndex == NULL)
goto err_nomem;
for (i=0; i<nDbObjectsListCount; i++)
cDbObjectClassAndNameIndex[i] = (ULONG)i;
qsort_s(cDbObjectClassAndNameIndex.Get(), nDbObjectsListCount, sizeof(ULONG), SortDbObjectByClassAndName,
lpDbObjectsList);
//get the first item of each class
nktMemSet(&sDbObjectClassPos, 0, sizeof(sDbObjectClassPos));
nLastClass = -1;
for (i=0; i<nDbObjectsListCount; i++)
{
nCurrClass = (int)(lpDbObjectsList[cDbObjectClassAndNameIndex[i]].lpStoredData->nClass);
if (nCurrClass != nLastClass)
{
sDbObjectClassPos[nCurrClass+1].nStart = (ULONG)i;
sDbObjectClassPos[nCurrClass+1].nCount = 1;
nLastClass = nCurrClass;
}
else
{
sDbObjectClassPos[nLastClass+1].nCount++;
}
}
*/
}
PrintAndLog(L"OK\n");
return TRUE;
}
