void UncompressInc(
std::vector<unsigned char> &outBuf,
const std::vector<unsigned char> &inBuf)
{
CLzmaDec dec;
UInt64 unpackSize = 0;
for (int i = 0; i < 8; i++)
unpackSize += (UInt64)inBuf[5 + i] << (i * 8);
LzmaDec_Construct(&dec);
SRes res = LzmaDec_Allocate(&dec, &inBuf[0], LZMA_PROPS_SIZE, &SzAllocForLzma);
assert(res == SZ_OK);
LzmaDec_Init(&dec);
//printf("uint64 resLen : %" PRIu64 "\n", unpackSize); //print uint64
outBuf.resize(unpackSize);
unsigned outPos = 0, inPos = LZMA_PROPS_SIZE;
ELzmaStatus status;
const unsigned BUF_SIZE = 10240;
while (outPos < outBuf.size())
{
unsigned destLen = min(BUF_SIZE, outBuf.size() - outPos);
unsigned srcLen = min(BUF_SIZE, inBuf.size() - 8 - inPos);
res = LzmaDec_DecodeToBuf(&dec,
&outBuf[outPos], &destLen,
&inBuf[inPos+8], &srcLen,
(outPos + destLen == outBuf.size())
? LZMA_FINISH_END : LZMA_FINISH_ANY, &status);
assert(res == SZ_OK);
inPos += srcLen;
outPos += destLen;
if (status == LZMA_STATUS_FINISHED_WITH_MARK)
break;
}
LzmaDec_Free(&dec, &g_Alloc);
outBuf.resize(outPos);
FILE *fout = fopen("data33.dc", "wb+");
fwrite(&outBuf[0], 1, outBuf.size(), fout);
fclose(fout);
}
FileReaderLZMA::FileReaderLZMA (FileReader &file, size_t uncompressed_size, bool zip)
: File(file), SawEOF(false)
{
BYTE header[4 + LZMA_PROPS_SIZE];
int err;
assert(zip == true);
Size = uncompressed_size;
OutProcessed = 0;
// Read zip LZMA properties header
if (File.Read(header, sizeof(header)) < (long)sizeof(header))
{
I_Error("FileReaderLZMA: File too shart\n");
}
if (header[2] + header[3] * 256 != LZMA_PROPS_SIZE)
{
I_Error("FileReaderLZMA: LZMA props size is %d (expected %d)\n",
header[2] + header[3] * 256, LZMA_PROPS_SIZE);
}
FillBuffer();
Streamp = new StreamPointer;
LzmaDec_Construct(&Streamp->Stream);
err = LzmaDec_Allocate(&Streamp->Stream, header + 4, LZMA_PROPS_SIZE, &g_Alloc);
if (err != SZ_OK)
{
I_Error("FileReaderLZMA: LzmaDec_Allocate failed: %d\n", err);
}
LzmaDec_Init(&Streamp->Stream);
}
void Lzma2Dec_Init(CLzma2Dec *p)
{
p->state = LZMA2_STATE_CONTROL;
p->needInitDic = True;
p->needInitState = True;
p->needInitProp = True;
LzmaDec_Init(&p->decoder);
}
void CDecoder::SetOutStreamSizeResume(const UInt64 *outSize)
{
_outSizeDefined = (outSize != NULL);
if (_outSizeDefined)
_outSize = *outSize;
_outSizeProcessed = 0;
LzmaDec_Init(&_state);
}
/* should be the first function */
void entry(unsigned long icache_size, unsigned long icache_lsize,
unsigned long dcache_size, unsigned long dcache_lsize,
unsigned long fw_arg0, unsigned long fw_arg1,
unsigned long fw_arg2, unsigned long fw_arg3)
{
CLzmaDec state;
ISzAlloc dummy;
ISeqInStream stream;
ELzmaStatus status;
unsigned int i; /* temp value */
unsigned int osize; /* uncompressed size */
dummy.Alloc = dummy_alloc;
dummy.Free = dummy_free;
stream.Read = read_bytes;
/* look for trx header, 32-bit data access */
for (data = ((unsigned char *)KSEG1ADDR(BCM4710_FLASH));
((struct trx_header *)data)->magic != TRX_MAGIC; data += 65536);
/* compressed kernel is in the partition 1 */
data += ((struct trx_header *)data)->offsets[1];
offset = 0;
/* read lzma stream header */
SeqInStream_Read(&stream, workspace, LZMA_PROPS_SIZE + 8);
/* read the lower half of uncompressed size in the header */
osize = (workspace[LZMA_PROPS_SIZE + 0]) +
(workspace[LZMA_PROPS_SIZE + 1] << 8) +
(workspace[LZMA_PROPS_SIZE + 2] << 16) +
(workspace[LZMA_PROPS_SIZE + 3] << 24);
LzmaDec_Construct(&state);
LzmaDec_AllocateProbs(&state, workspace, LZMA_PROPS_SIZE, &dummy);
state.dic = (unsigned char *)LOADADDR;
state.dicBufSize = osize;
/* decompress kernel */
LzmaDec_Init(&state);
do {
i = LZMA_REQUIRED_INPUT_MAX;
SeqInStream_Read(&stream, workspace, i);
if (LzmaDec_DecodeToDic(&state, osize, workspace, &i,
LZMA_FINISH_ANY, &status) != SZ_OK) {
/* something went wrong */
return;
}
} while (status == LZMA_STATUS_NEEDS_MORE_INPUT);
blast_dcache(dcache_size, dcache_lsize);
blast_icache(icache_size, icache_lsize);
/* jump to load address */
((void (*)(unsigned long, unsigned long, unsigned long,
unsigned long))LOADADDR)(fw_arg0, fw_arg1, fw_arg2, fw_arg3);
}
void Lzma2Dec_Init(CLzma2Dec *p)
{
p->state = LZMA2_STATE_CONTROL;
p->needInitLevel = 0xE0;
p->isExtraMode = False;
p->unpackSize = 0;
// p->decoder.dicPos = 0; // we can use it instead of full init
LzmaDec_Init(&p->decoder);
}
STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize)
{
_outSizeDefined = (outSize != NULL);
if (_outSizeDefined)
_outSize = *outSize;
LzmaDec_Init(&_state);
_inPos = _inSize = 0;
_inSizeProcessed = _outSizeProcessed = 0;
return S_OK;
}
static SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inStream,
UInt64 unpackSize)
{
int thereIsSize = (unpackSize != (UInt64)(Int64)-1);
Byte inBuf[IN_BUF_SIZE];
Byte outBuf[OUT_BUF_SIZE];
size_t inPos = 0, inSize = 0, outPos = 0;
LzmaDec_Init(state);
for (;;)
{
if (inPos == inSize)
{
inSize = IN_BUF_SIZE;
RINOK(inStream->Read(inStream, inBuf, &inSize));
inPos = 0;
}
{
SRes res;
SizeT inProcessed = inSize - inPos;
SizeT outProcessed = OUT_BUF_SIZE - outPos;
ELzmaFinishMode finishMode = LZMA_FINISH_ANY;
ELzmaStatus status;
if (thereIsSize && outProcessed > unpackSize)
{
outProcessed = (SizeT)unpackSize;
finishMode = LZMA_FINISH_END;
}
res = LzmaDec_DecodeToBuf(state, outBuf + outPos, &outProcessed,
inBuf + inPos, &inProcessed, finishMode, &status);
inPos += inProcessed;
outPos += outProcessed;
unpackSize -= outProcessed;
if (outStream)
if (outStream->Write(outStream, outBuf, outPos) != outPos)
return SZ_ERROR_WRITE;
outPos = 0;
if (res != SZ_OK || (thereIsSize && unpackSize == 0))
return res;
if (inProcessed == 0 && outProcessed == 0)
{
if (thereIsSize || status != LZMA_STATUS_FINISHED_WITH_MARK)
return SZ_ERROR_DATA;
return res;
}
}
}
}
static SRes SzDecodeLzma(CSzCoderInfo *coder, CFileSize inSize, ISzInStream *inStream,
Byte *outBuffer, size_t outSize, ISzAlloc *allocMain)
{
CLzmaDec state;
int res = SZ_OK;
size_t _inSize;
Byte *inBuf = NULL;
LzmaDec_Construct(&state);
RINOK(LzmaDec_AllocateProbs(&state, coder->Props.data, (unsigned)coder->Props.size, allocMain));
state.dic = outBuffer;
state.dicBufSize = outSize;
LzmaDec_Init(&state);
_inSize = 0;
for (;;)
{
if (_inSize == 0)
{
_inSize = (1 << 18);
if (_inSize > inSize)
_inSize = (size_t)(inSize);
res = inStream->Read((void *)inStream, (void **)&inBuf, &_inSize);
if (res != SZ_OK)
break;
inSize -= _inSize;
}
{
SizeT inProcessed = _inSize, dicPos = state.dicPos;
ELzmaStatus status;
res = LzmaDec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status);
_inSize -= inProcessed;
inBuf = (Byte *)inBuf + inProcessed;
if (res != SZ_OK)
break;
if (state.dicPos == state.dicBufSize || (inProcessed == 0 && dicPos == state.dicPos))
{
if (state.dicBufSize != outSize || _inSize != 0 ||
(status != LZMA_STATUS_FINISHED_WITH_MARK &&
status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK))
res = SZ_ERROR_DATA;
break;
}
}
}
LzmaDec_FreeProbs(&state, allocMain);
return res;
}
inline TImpl(TInputStream* slave)
: Slave_(slave)
, InPos_(0)
, InSize_(0)
{
LzmaDec_Construct(&H_);
Byte buf[LZMA_PROPS_SIZE];
Slave_->Load(buf, sizeof(buf));
Check(LzmaDec_Allocate(&H_, buf, sizeof(buf), Alloc()));
LzmaDec_Init(&H_);
}
SRes IS_LzmaDec_Init(CLzmaDec *state, size_t stateSize, const Byte *props,
unsigned propsSize, ISzAlloc *alloc)
{
if (stateSize != sizeof(*state)) {
return SZ_ERROR_PARAM;
}
// Not needed; just sets fields to 0, which will leak memory if Init was already called previously
//LzmaDec_Construct(state);
RINOK(LzmaDec_Allocate(state, props, propsSize, alloc));
LzmaDec_Init(state);
return SZ_OK;
}
static SRes SzDecodeLzma(CSzCoderInfo *coder, UInt64 inSize, ILookInStream *inStream,
Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain)
{
CLzmaDec state;
SRes res = SZ_OK;
LzmaDec_Construct(&state);
RINOK(LzmaDec_AllocateProbs(&state, coder->Props.data, (unsigned)coder->Props.size, allocMain));
state.dic = outBuffer;
state.dicBufSize = outSize;
LzmaDec_Init(&state);
for (;;)
{
Byte *inBuf = NULL;
size_t lookahead = (1 << 18);
if (lookahead > inSize)
lookahead = (size_t)inSize;
res = inStream->Look((void *)inStream, (const void **)&inBuf, &lookahead);
if (res != SZ_OK)
break;
{
SizeT inProcessed = (SizeT)lookahead, dicPos = state.dicPos;
ELzmaStatus status;
res = LzmaDec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status);
lookahead -= inProcessed;
inSize -= inProcessed;
if (res != SZ_OK)
break;
if (state.dicPos == state.dicBufSize || (inProcessed == 0 && dicPos == state.dicPos))
{
if (state.dicBufSize != outSize || lookahead != 0 ||
(status != LZMA_STATUS_FINISHED_WITH_MARK &&
status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK))
res = SZ_ERROR_DATA;
break;
}
res = inStream->Skip((void *)inStream, inProcessed);
if (res != SZ_OK)
break;
}
}
LzmaDec_FreeProbs(&state, allocMain);
return res;
}
bool LZMADecoder::CodeTo(const MemoryByteData& data,MemoryByteData& outData) const
{
const byte* inBuffer=data.GetData();
size_t inSize=data.GetSize();
const size_t propSize=5;
ELzmaStatus outStatus;
ISzAlloc myAlloc;
myAlloc.Alloc=LZMAAlloc;
myAlloc.Free=LZMAFree;
CLzmaDec p;
SRes res;
if (inSize < propSize)
return false;
LzmaDec_Construct(&p);
res = LzmaDec_AllocateProbs(&p, inBuffer, propSize, &myAlloc);
uint64 fileSize = 0;
for (int i = 0; i < 8; i++)
fileSize |= ((uint64)inBuffer[propSize + i]) << (8 * i);
if (outData.GetSize()!=(size_t)fileSize)
{
LzmaDec_FreeProbs(&p, &myAlloc);
return false;
}
else
{
LzmaDec_Init(&p);
p.dic = outData.GetData();
p.dicBufSize = (size_t)fileSize;
size_t outSize=inSize-13;
res = LzmaDec_DecodeToDic(&p, (size_t)fileSize, inBuffer+13, &outSize, LZMA_FINISH_ANY, &outStatus);
if (res == SZ_OK && outStatus == LZMA_STATUS_NEEDS_MORE_INPUT)
res = SZ_ERROR_INPUT_EOF;
LzmaDec_FreeProbs(&p, &myAlloc);
return true;
}
}
size_t LZMADecoder::OnCode(const MemoryData& input, MemoryData& output) const
{
RETURN_ZERO_IF_EMPTY(input);
const byte* inBuffer = input.Data();
size_t inSize = input.Size();
ELzmaStatus outStatus;
ISzAlloc myAlloc;
myAlloc.Alloc = LZMAAlloc;
myAlloc.Free = LZMAFree;
CLzmaDec p;
SRes res;
LzmaDec_Construct(&p);
res = LzmaDec_AllocateProbs(&p, inBuffer, LZMA_PROPS_SIZE, &myAlloc);
uint64 fileSize = 0;
for (int i = 0; i < 8; i++)
fileSize |= ((uint64)inBuffer[LZMA_PROPS_SIZE + i]) << (8 * i);
if (output.Size()<(size_t)fileSize)
{
Log::AssertFailedFormat("output size:{} < expected size{}", output.Size(), fileSize);
LzmaDec_FreeProbs(&p, &myAlloc);
return 0;
}
LzmaDec_Init(&p);
p.dic = output.MutableData();
p.dicBufSize = (size_t)fileSize;
size_t outSize = inSize - 13;
res = LzmaDec_DecodeToDic(&p, (size_t)fileSize, inBuffer + 13, &outSize, LZMA_FINISH_ANY, &outStatus);
if (res == SZ_OK && outStatus == LZMA_STATUS_NEEDS_MORE_INPUT)
res = SZ_ERROR_INPUT_EOF;
LzmaDec_FreeProbs(&p, &myAlloc);
return (size_t)fileSize;
}
struct lzma_dec_state *lzma_dec_init(void* props) {
struct lzma_dec_state *state = malloc(sizeof(struct lzma_dec_state));
memset(state, 0, sizeof(*state));
if (!state) {
return NULL;
}
state->alloc.Alloc = &alloc_impl;
state->alloc.Free = &free_impl;
if (LzmaDec_Allocate(&state->decoder, props, 5, &state->alloc)) {
free (state);
return NULL;
}
LzmaDec_Init(&state->decoder);
return state;
}
int cli_LzmaInit(struct CLI_LZMA *L, uint64_t size_override) {
int fail;
if(!L->init) {
L->p_cnt = LZMA_PROPS_SIZE;
if(size_override)
L->usize = size_override;
else
L->s_cnt = 8;
L->init = 1;
} else if(size_override)
cli_warnmsg("cli_LzmaInit: ignoring late size override\n");
if(L->freeme) return LZMA_RESULT_OK;
while(L->p_cnt) {
L->header[LZMA_PROPS_SIZE - L->p_cnt] = lzma_getbyte(L, &fail);
if(fail) return LZMA_RESULT_OK;
L->p_cnt--;
}
while(L->s_cnt) {
uint64_t c = (uint64_t)lzma_getbyte(L, &fail);
if(fail) return LZMA_RESULT_OK;
L->usize = c << (8 * (8 - L->s_cnt));
L->s_cnt--;
}
LzmaDec_Construct(&L->state);
if(LzmaDec_Allocate(&L->state, L->header, LZMA_PROPS_SIZE, &g_Alloc) != SZ_OK)
return LZMA_RESULT_DATA_ERROR;
LzmaDec_Init(&L->state);
L->freeme = 1;
return LZMA_RESULT_OK;
}
int
BCMINITFN(_nvram_read)(void *buf, int idx)
{
uint32 *src, *dst;
uint i;
if (!nvram_header)
return -19; /* -ENODEV */
#if defined(_CFE_) && defined(BCM_DEVINFO)
if ((!devinfo_nvram_header) && (idx == 1)) {
return -19; /* -ENODEV */
}
src = idx == 0 ? (uint32 *) nvram_header : (uint32 *) devinfo_nvram_nvh;
#else
src = (uint32 *) nvram_header;
#endif /* _CFE_ && BCM_DEVINFO */
dst = (uint32 *) buf;
for (i = 0; i < sizeof(struct nvram_header); i += 4)
*dst++ = *src++;
/* Since we know what the first 3 bytes of the lzma properties
* should be based on what we used to compress, check them
* to see if we need to decompress (uncompressed this would show up a
* a single [ and then the end of nvram marker so its invalid in an
* uncompressed nvram block
*/
if ((((unsigned char *)src)[0] == 0x5d) &&
(((unsigned char *)src)[1] == 0) &&
(((unsigned char *)src)[2] == 0)) {
unsigned int dstlen = nvram_header->len;
unsigned int srclen = MAX_NVRAM_SPACE-LZMA_PROPS_SIZE-NVRAM_HEADER_SIZE;
unsigned char *cp = (unsigned char *)src;
CLzmaDec state;
SRes res;
ELzmaStatus status;
LzmaDec_Construct(&state);
res = LzmaDec_Allocate(&state, cp, LZMA_PROPS_SIZE, &g_Alloc);
if (res != SZ_OK) {
printf("Error Initializing LZMA Library\n");
return -19;
}
LzmaDec_Init(&state);
res = LzmaDec_DecodeToBuf(&state,
(unsigned char *)dst, &dstlen,
&cp[LZMA_PROPS_SIZE], &srclen,
LZMA_FINISH_ANY,
&status);
LzmaDec_Free(&state, &g_Alloc);
if (res != SZ_OK) {
printf("Error Decompressing eNVRAM\n");
return -19;
}
} else {
for (; i < nvram_header->len && i < MAX_NVRAM_SPACE; i += 4)
*dst++ = ltoh32(*src++);
}
return 0;
}
int
elzma_decompress_run(elzma_decompress_handle hand,
elzma_read_callback inputStream, void * inputContext,
elzma_write_callback outputStream, void * outputContext,
elzma_file_format format)
{
unsigned long long int totalRead = 0; /* total amount read from stream */
unsigned int crc32 = CRC_INIT_VAL; /* running crc32 (lzip case) */
CLzmaDec dec;
unsigned int errorCode = ELZMA_E_OK;
struct elzma_format_handler formatHandler;
struct elzma_file_header h;
struct elzma_file_footer f;
/* switch between supported formats */
if (format == ELZMA_lzma) {
initializeLZMAFormatHandler(&formatHandler);
} else if (format == ELZMA_lzip) {
CrcGenerateTable();
initializeLZIPFormatHandler(&formatHandler);
} else {
return ELZMA_E_BAD_PARAMS;
}
/* initialize footer */
f.crc32 = 0;
f.uncompressedSize = 0;
/* initialize decoder memory */
memset((void *) &dec, 0, sizeof(dec));
LzmaDec_Init(&dec);
/* decode the header. */
{
unsigned char * hdr =
hand->allocStruct.Alloc(&(hand->allocStruct),
formatHandler.header_size);
size_t sz = formatHandler.header_size;
formatHandler.init_header(&h);
if (inputStream(inputContext, hdr, &sz) != 0 ||
sz != formatHandler.header_size)
{
hand->allocStruct.Free(&(hand->allocStruct), hdr);
return ELZMA_E_INPUT_ERROR;
}
if (0 != formatHandler.parse_header(hdr, &h)) {
hand->allocStruct.Free(&(hand->allocStruct), hdr);
return ELZMA_E_CORRUPT_HEADER;
}
/* the LzmaDec_Allocate call requires 5 bytes which have
* compression properties encoded in them. In the case of
* lzip, the header format does not already contain what
* LzmaDec_Allocate expects, so we must craft it, silly */
{
unsigned char propsBuf[13];
const unsigned char * propsPtr = hdr;
if (format == ELZMA_lzip) {
struct elzma_format_handler lzmaHand;
initializeLZMAFormatHandler(&lzmaHand);
lzmaHand.serialize_header(propsBuf, &h);
propsPtr = propsBuf;
}
/* now we're ready to allocate the decoder */
LzmaDec_Allocate(&dec, propsPtr, 5,
(ISzAlloc *) &(hand->allocStruct));
}
hand->allocStruct.Free(&(hand->allocStruct), hdr);
}
/* perform the decoding */
for (;;)
{
size_t dstLen = ELZMA_DECOMPRESS_OUTPUT_BUFSIZE;
size_t srcLen = ELZMA_DECOMPRESS_INPUT_BUFSIZE;
size_t amt = 0;
size_t bufOff = 0;
ELzmaStatus stat;
if (0 != inputStream(inputContext, hand->inbuf, &srcLen))
{
errorCode = ELZMA_E_INPUT_ERROR;
goto decompressEnd;
}
/* handle the case where the input prematurely finishes */
if (srcLen == 0) {
errorCode = ELZMA_E_INSUFFICIENT_INPUT;
goto decompressEnd;
}
amt = srcLen;
/* handle the case where a single read buffer of compressed bytes
* will translate into multiple buffers of uncompressed bytes,
* with this inner loop */
stat = LZMA_STATUS_NOT_SPECIFIED;
while (bufOff < srcLen) {
SRes r = LzmaDec_DecodeToBuf(&dec, (Byte *) hand->outbuf, &dstLen,
((Byte *) hand->inbuf + bufOff), &amt,
LZMA_FINISH_ANY, &stat);
/* XXX deal with result code more granularly*/
if (r != SZ_OK) {
errorCode = ELZMA_E_DECOMPRESS_ERROR;
goto decompressEnd;
}
/* write what we've read */
{
size_t wt;
/* if decoding lzip, update our crc32 value */
if (format == ELZMA_lzip && dstLen > 0) {
crc32 = CrcUpdate(crc32, hand->outbuf, dstLen);
}
totalRead += dstLen;
wt = outputStream(outputContext, hand->outbuf, dstLen);
if (wt != dstLen) {
errorCode = ELZMA_E_OUTPUT_ERROR;
goto decompressEnd;
}
}
/* do we have more data on the input buffer? */
bufOff += amt;
assert( bufOff <= srcLen );
if (bufOff >= srcLen) break;
amt = srcLen - bufOff;
/* with lzip, we will have the footer left on the buffer! */
if (stat == LZMA_STATUS_FINISHED_WITH_MARK) {
break;
}
}
/* now check status */
if (stat == LZMA_STATUS_FINISHED_WITH_MARK) {
/* read a footer if one is expected and
* present */
if (formatHandler.footer_size > 0 &&
amt >= formatHandler.footer_size &&
formatHandler.parse_footer != NULL)
{
formatHandler.parse_footer(
(unsigned char *) hand->inbuf + bufOff, &f);
}
break;
}
/* for LZMA utils, we don't always have a finished mark */
if (!h.isStreamed && totalRead >= h.uncompressedSize) {
break;
}
}
/* finish the calculated crc32 */
crc32 ^= 0xFFFFFFFF;
/* if we have a footer, check that the calculated crc32 matches
* the encoded crc32, and that the sizes match */
if (formatHandler.footer_size)
{
if (f.crc32 != crc32) {
errorCode = ELZMA_E_CRC32_MISMATCH;
} else if (f.uncompressedSize != totalRead) {
errorCode = ELZMA_E_SIZE_MISMATCH;
}
}
else if (!h.isStreamed)
{
/* if the format does not support a footer and has an uncompressed
* size in the header, let's compare that with how much we actually
* read */
if (h.uncompressedSize != totalRead) {
errorCode = ELZMA_E_SIZE_MISMATCH;
}
}
decompressEnd:
LzmaDec_Free(&dec, (ISzAlloc *) &(hand->allocStruct));
return errorCode;
}
static PyObject *
pylzma_decomp_decompress(CDecompressionObject *self, PyObject *args)
{
PyObject *result=NULL;
unsigned char *data;
Byte *next_in, *next_out;
int length, res, bufsize=BLOCK_SIZE;
SizeT avail_in;
SizeT inProcessed, outProcessed;
ELzmaStatus status;
if (!PyArg_ParseTuple(args, "s#|i", &data, &length, &bufsize)){
return NULL;
}
if (bufsize <= 0) {
PyErr_SetString(PyExc_ValueError, "bufsize must be greater than zero");
return NULL;
}
if (self->unconsumed_length > 0) {
self->unconsumed_tail = (unsigned char *) realloc(self->unconsumed_tail, self->unconsumed_length + length);
next_in = (unsigned char *) self->unconsumed_tail;
memcpy(next_in + self->unconsumed_length, data, length);
} else {
next_in = data;
}
if (self->need_properties) {
if ((self->unconsumed_length + length) < LZMA_PROPS_SIZE) {
// we need enough bytes to read the properties
self->unconsumed_tail = (unsigned char *) realloc(self->unconsumed_tail, self->unconsumed_length + length);
if (self->unconsumed_tail == NULL) {
PyErr_NoMemory();
return NULL;
}
memcpy(self->unconsumed_tail + self->unconsumed_length, data, length);
self->unconsumed_length += length;
return PyString_FromString("");
}
self->unconsumed_length += length;
res = LzmaDec_Allocate(&self->state, next_in, LZMA_PROPS_SIZE, &allocator);
if (res != SZ_OK) {
PyErr_SetString(PyExc_TypeError, "Incorrect stream properties");
return NULL;
}
next_in += LZMA_PROPS_SIZE;
self->unconsumed_length -= LZMA_PROPS_SIZE;
if (self->unconsumed_length > 0) {
if (self->unconsumed_tail == NULL) {
// No remaining data yet
self->unconsumed_tail = (unsigned char *) malloc(self->unconsumed_length);
if (self->unconsumed_tail == NULL) {
PyErr_NoMemory();
return NULL;
}
memcpy(self->unconsumed_tail, next_in, self->unconsumed_length);
next_in = self->unconsumed_tail;
} else {
// Skip properties in remaining data
memmove(self->unconsumed_tail, self->unconsumed_tail+LZMA_PROPS_SIZE, self->unconsumed_length);
self->unconsumed_tail = next_in = (unsigned char *) realloc(self->unconsumed_tail, self->unconsumed_length);
if (self->unconsumed_tail == NULL) {
PyErr_NoMemory();
return NULL;
}
}
} else {
FREE_AND_NULL(self->unconsumed_tail);
}
self->need_properties = 0;
LzmaDec_Init(&self->state);
} else {
self->unconsumed_length += length;
}
avail_in = self->unconsumed_length;
if (avail_in == 0) {
// no more bytes to decompress
return PyString_FromString("");
}
result = PyString_FromStringAndSize(NULL, bufsize);
if (result == NULL) {
PyErr_NoMemory();
goto exit;
}
next_out = (unsigned char *) PyString_AS_STRING(result);
Py_BEGIN_ALLOW_THREADS
// Decompress until EOS marker is reached
inProcessed = avail_in;
outProcessed = bufsize;
res = LzmaDec_DecodeToBuf(&self->state, next_out, &outProcessed,
next_in, &inProcessed, LZMA_FINISH_ANY, &status);
Py_END_ALLOW_THREADS
self->total_out += outProcessed;
next_in += inProcessed;
avail_in -= inProcessed;
if (res != SZ_OK) {
DEC_AND_NULL(result);
PyErr_SetString(PyExc_ValueError, "data error during decompression");
goto exit;
}
// Not all of the compressed data could be accomodated in the output buffer
// of specified size. Return the unconsumed tail in an attribute.
if (avail_in > 0) {
if (self->unconsumed_tail == NULL) {
// data are in "data"
self->unconsumed_tail = (unsigned char *) malloc(avail_in);
if (self->unconsumed_tail == NULL) {
Py_DECREF(result);
PyErr_NoMemory();
goto exit;
}
memcpy(self->unconsumed_tail, next_in, avail_in);
} else {
memmove(self->unconsumed_tail, next_in, avail_in);
self->unconsumed_tail = (unsigned char *) realloc(self->unconsumed_tail, avail_in);
}
} else {
FREE_AND_NULL(self->unconsumed_tail);
}
self->unconsumed_length = avail_in;
_PyString_Resize(&result, outProcessed);
exit:
return result;
}
int _tmain(int argc, _TCHAR* argv[])
{
HANDLE hFileIn = ::CreateFile(_T("C:\\Users\\Christoph\\Desktop\\S.csv.lzma"), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
HANDLE hFileOut = ::CreateFile(_T("C:\\Users\\Christoph\\Desktop\\S.csv.decd"), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
BYTE header[LZMA_PROPS_SIZE + 8];
DWORD dwBytesRead = 0;
DWORD dwBytesWritten = 0;
::ReadFile(hFileIn, header, sizeof(header), &dwBytesRead, NULL);
ELzmaStatus status;
CLzmaDec state;
LzmaDec_Construct(&state);
LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &g_Alloc);
LzmaDec_Init(&state);
__int64 nSize = *reinterpret_cast<__int64*>(header + LZMA_PROPS_SIZE);
_tprintf(_T("Extracted size: %d\n"), nSize);
BYTE *pDataIn = new BYTE[BUFFER_SIZE];
BYTE *pDataOut = new BYTE[BUFFER_SIZE];
size_t nTotalSize = (size_t)nSize;
size_t nInSize = BUFFER_SIZE, nInPos = 0;
size_t nOutSize = BUFFER_SIZE;
while (nTotalSize)
{
if (!::ReadFile(hFileIn, pDataIn, BUFFER_SIZE, &dwBytesRead, NULL)) break;
if (dwBytesRead == 0) break;
nInSize = dwBytesRead;
nInPos = 0;
_tprintf(_T("."));
while (nInPos < nInSize)
{
size_t nInProcessed = nInSize - nInPos;
size_t nOutProcessed = BUFFER_SIZE;
LzmaDec_DecodeToBuf(&state, pDataOut, &nOutProcessed,
pDataIn + nInPos, &nInProcessed,
(nOutProcessed > nTotalSize)? LZMA_FINISH_END : LZMA_FINISH_ANY,
&status);
::WriteFile(hFileOut, pDataOut, nOutProcessed, &dwBytesRead, NULL);
nInPos += nInProcessed;
nTotalSize -= nOutProcessed;
if (nInProcessed == 0 && nOutProcessed == 0)
{
nInPos = nInSize = 0;
nTotalSize = 0;
}
}
}
delete[] pDataOut;
delete[] pDataIn;
LzmaDec_Free(&state, &g_Alloc);
::CloseHandle(hFileOut);
::CloseHandle(hFileIn);
#ifdef _DEBUG
_tsystem(_T("pause"));
#endif
return 0;
}
static int lzmafs_fileop_open(void **ref,void *fsctx_arg,char *filename,int mode)
{
lzmafs_fsctx_t *fsctx;
lzmafs_file_t *file;
int err = 0, i;
/* header: 5 bytes of LZMA properties and 8 bytes of uncompressed size */
unsigned char header[LZMA_PROPS_SIZE + 8];
if (mode != FILE_MODE_READ)
return CFE_ERR_UNSUPPORTED;
fsctx = (lzmafs_fsctx_t *) fsctx_arg;
file = KMALLOC(sizeof(lzmafs_file_t), 0);
if (!file) {
return CFE_ERR_NOMEM;
}
file->lzmafs_fileoffset = 0;
file->lzmafs_fsctx = fsctx;
file->lzmafs_inlen = 0;
file->lzmafs_eofseen = 0;
file->lzmafs_state = KMALLOC(sizeof(CLzmaDec), 0);
if (!file->lzmafs_state)
goto error2;
file->lzmafs_outlen = 0;
/* Open the raw file system */
err = BDOPEN(fsctx->lzmafsctx_subops, &(file->lzmafs_subfile),
fsctx->lzmafsctx_subfsctx, filename);
if (err != 0)
goto error2;
err = BDREAD(file->lzmafs_fsctx->lzmafsctx_subops,
file->lzmafs_subfile,
header,
sizeof(header));
if (err < 0)
goto error;
file->lzmafs_unpackSize = 0;
/* uncompressed size */
for (i = 0; i < 8; i++)
file->lzmafs_unpackSize += header[LZMA_PROPS_SIZE + i] << (i * 8);
LzmaDec_Construct(file->lzmafs_state);
RINOK(LzmaDec_Allocate(file->lzmafs_state, header, LZMA_PROPS_SIZE, &g_Alloc));
file->lzmafs_inbuf = KMALLOC(LZMAFS_INBUFSIZE, 0);
file->lzmafs_outbuf = KMALLOC(LZMAFS_OUTBUFSIZE, 0);
if (!file->lzmafs_inbuf || !file->lzmafs_outbuf) {
err = CFE_ERR_NOMEM;
goto error;
}
file->lzmafs_outptr = file->lzmafs_outbuf;
LzmaDec_Init(file->lzmafs_state);
fsctx->lzmafsctx_refcnt++;
*ref = file;
return 0;
error:
BDCLOSE(file->lzmafs_fsctx->lzmafsctx_subops, file->lzmafs_subfile);
error2:
if (file->lzmafs_inbuf)
KFREE(file->lzmafs_inbuf);
if (file->lzmafs_outbuf)
KFREE(file->lzmafs_outbuf);
if (file->lzmafs_state) {
LzmaDec_Free(file->lzmafs_state, &g_Alloc);
KFREE(file->lzmafs_state);
}
if (file)
KFREE(file);
return err;
}
static int Decode(FILE *inFile, FILE *outFile, char *rs)
{
UInt64 unpackSize;
int thereIsSize; /* = 1, if there is uncompressed size in headers */
int i;
int res = 0;
CLzmaDec state;
/* header: 5 bytes of LZMA properties and 8 bytes of uncompressed size */
unsigned char header[LZMA_PROPS_SIZE + 8];
/* Read and parse header */
if (!MyReadFileAndCheck(inFile, header, sizeof(header)))
return PrintError(rs, kCantReadMessage);
unpackSize = 0;
thereIsSize = 0;
for (i = 0; i < 8; i++)
{
unsigned char b = header[LZMA_PROPS_SIZE + i];
if (b != 0xFF)
thereIsSize = 1;
unpackSize += (UInt64)b << (i * 8);
}
LzmaDec_Construct(&state);
res = LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &g_Alloc);
if (res != SZ_OK)
return res;
{
Byte inBuf[IN_BUF_SIZE];
Byte outBuf[OUT_BUF_SIZE];
size_t inPos = 0, inSize = 0, outPos = 0;
LzmaDec_Init(&state);
for (;;)
{
if (inPos == inSize)
{
inSize = MyReadFile(inFile, inBuf, IN_BUF_SIZE);
inPos = 0;
}
{
SizeT inProcessed = inSize - inPos;
SizeT outProcessed = OUT_BUF_SIZE - outPos;
ELzmaFinishMode finishMode = LZMA_FINISH_ANY;
ELzmaStatus status;
if (thereIsSize && outProcessed > unpackSize)
{
outProcessed = (SizeT)unpackSize;
finishMode = LZMA_FINISH_END;
}
res = LzmaDec_DecodeToBuf(&state, outBuf + outPos, &outProcessed,
inBuf + inPos, &inProcessed, finishMode, &status);
inPos += (UInt32)inProcessed;
outPos += outProcessed;
unpackSize -= outProcessed;
if (outFile != 0)
MyWriteFile(outFile, outBuf, outPos);
outPos = 0;
if (res != SZ_OK || thereIsSize && unpackSize == 0)
break;
if (inProcessed == 0 && outProcessed == 0)
{
if (thereIsSize || status != LZMA_STATUS_FINISHED_WITH_MARK)
res = SZ_ERROR_DATA;
break;
}
}
}
}
LzmaDec_Free(&state, &g_Alloc);
return res;
}
PakEntry Pak::read(uint32_t tag) const
{
int entryIndex = findEntry(tag);
if(entryIndex < 0)
{
return PakEntry(NULL, 0, false);
}
const Header *header = reinterpret_cast<const Header*>(m_pakData);
const Entry *entry = reinterpret_cast<const Entry*>(header + 1) + entryIndex;
size_t entrySize = (entry+1)->offset - entry->offset;
if(entry->offset>m_pakDataSize || entrySize>=4*1024*1024 || entry->offset+entrySize>m_pakDataSize)
{
ERROR("Pak::findEntry: invalid entry offset/size (tag=0x%08x index=%u offset=%u size=%u)", tag, entryIndex, entry->offset, entrySize);
return PakEntry(NULL, 0, false);
}
const uint8_t *entryData = cast<uint8_t>(m_pakData, entry->offset);
const LzmaHeader *lzmaHeader = cast<LzmaHeader>(entryData, 0);
if(memcmp(lzmaHeader->magic, "lzma", 4) != 0)
{
DEBUG("Pak::findEntry: found uncompressed 0x%08x at %u size %zu", tag, entry->offset, entrySize);
return PakEntry(entryData, entrySize, false);
}
if(lzmaHeader->decompressedSize<entrySize/2 || lzmaHeader->decompressedSize>=8*1024*1024)
{
ERROR("Pak::findEntry: invalid uncompressed size (tag=0x%08x size=%u decompressedsize=%u)", tag, entrySize, lzmaHeader->decompressedSize);
return PakEntry(NULL, 0, false);
}
CLzmaDec dec;
LzmaDec_Construct(&dec);
LzmaDec_Init(&dec);
SRes res = LzmaDec_Allocate(&dec, lzmaHeader->props, LZMA_PROPS_SIZE, &lzmaAllocFuncs);
if(res != SZ_OK)
{
ERROR("Pak::findEntry: LzmaDec_Allocate failed: %d (tag=0x%08x)", res, tag);
return PakEntry(NULL, 0, false);
}
void *decdata = malloc(lzmaHeader->decompressedSize);
if(decdata == NULL)
{
ERROR("Pak::findEntry: LzmaDec_Allocate failed to allocate %u bytes", lzmaHeader->decompressedSize);
LzmaDec_Free(&dec, &lzmaAllocFuncs);
return PakEntry(NULL, 0, false);
}
size_t destSize = lzmaHeader->decompressedSize;
size_t srcSize = entrySize - sizeof(lzmaHeader);
ELzmaStatus status;
res = LzmaDec_DecodeToBuf(&dec,
(Byte*)decdata, &destSize,
lzmaHeader->data, &srcSize,
LZMA_FINISH_END, &status);
if(res != SZ_OK)
{
ERROR("Pak::findEntry: LzmaDec_DecodeToBuf failed: %d (tag=0x%08x)", res, tag);
LzmaDec_Free(&dec, &lzmaAllocFuncs);
free(decdata);
return PakEntry(NULL, 0, false);
}
if(!(status==LZMA_STATUS_FINISHED_WITH_MARK || (status==LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK && destSize!=lzmaHeader->decompressedSize)))
{
ERROR("Pak::findEntry: LzmaDec_DecodeToBuf ended with unexpected status: %d (tag=0x%08x)", status, tag);
LzmaDec_Free(&dec, &lzmaAllocFuncs);
free(decdata);
return PakEntry(NULL, 0, false);
}
if(destSize != lzmaHeader->decompressedSize)
{
ERROR("Pak::findEntry: LzmaDec_DecodeToBuf failed to decompress whole buffer: %zu!=%u (tag=0x%08x)", destSize, lzmaHeader->decompressedSize, tag);
LzmaDec_Free(&dec, &lzmaAllocFuncs);
free(decdata);
return PakEntry(NULL, 0, false);
}
LzmaDec_Free(&dec, &lzmaAllocFuncs);
return PakEntry(decdata, lzmaHeader->decompressedSize, true);
}