int64_t lzbench_lzma_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, size_t) { char rs[800] = { 0 }; CLzmaEncProps props; int res; size_t headerSize = LZMA_PROPS_SIZE; SizeT out_len = outsize - LZMA_PROPS_SIZE; LzmaEncProps_Init(&props); props.level = level; LzmaEncProps_Normalize(&props); /* p->level = 5; p->dictSize = p->mc = 0; p->reduceSize = (UInt64)(Int64)-1; p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1; p->writeEndMark = 0; */ res = LzmaEncode((uint8_t*)outbuf+LZMA_PROPS_SIZE, &out_len, (uint8_t*)inbuf, insize, &props, (uint8_t*)outbuf, &headerSize, 0/*int writeEndMark*/, NULL, &g_Alloc, &g_Alloc); if (res != SZ_OK) return 0; // printf("out_len=%u LZMA_PROPS_SIZE=%d headerSize=%d\n", (int)(out_len + LZMA_PROPS_SIZE), LZMA_PROPS_SIZE, (int)headerSize); return LZMA_PROPS_SIZE + out_len; }
/* * The two functions below are not thread-safe, by design. */ int lzma_init(void **data, int *level, int nthreads, uint64_t chunksize, int file_version, compress_op_t op) { if (!p && op == COMPRESS) { p = (CLzmaEncProps *)slab_alloc(NULL, sizeof (CLzmaEncProps)); LzmaEncProps_Init(p); /* * Set the dictionary size and fast bytes based on level. */ if (*level < 8) { /* * Choose a dict size with a balance between perf and * compression. */ p->dictSize = LZMA_DEFAULT_DICT; } else { /* * Let LZMA determine best dict size. */ p->dictSize = 0; } /* Determine the fast bytes value and also adjust dict size further. */ if (*level < 7) { p->fb = 32; } else if (*level < 10) { p->fb = 64; } else if (*level == 11) { p->fb = 64; p->mc = 128; } else if (*level == 12) { p->fb = 128; p->mc = 256; } else if (*level == 13) { p->fb = 64; p->mc = 128; p->dictSize = (1 << 27); } else if (*level == 14) { p->fb = 128; p->mc = 256; p->dictSize = (1 << 28); } if (*level > 9) *level = 9; p->level = *level; p->numThreads = nthreads; LzmaEncProps_Normalize(p); slab_cache_add(p->litprob_sz); } if (*level > 9) *level = 9; *data = p; return (0); }
static int pylzma_compfile_init(CCompressionFileObject *self, PyObject *args, PyObject *kwargs) { PyObject *inFile; CLzmaEncProps props; Byte header[LZMA_PROPS_SIZE]; size_t headerSize = LZMA_PROPS_SIZE; int result = -1; // possible keywords for this function static char *kwlist[] = {"infile", "dictionary", "fastBytes", "literalContextBits", "literalPosBits", "posBits", "algorithm", "eos", "multithreading", "matchfinder", NULL}; int dictionary = 23; // [0,28], default 23 (8MB) int fastBytes = 128; // [5,255], default 128 int literalContextBits = 3; // [0,8], default 3 int literalPosBits = 0; // [0,4], default 0 int posBits = 2; // [0,4], default 2 int eos = 1; // write "end of stream" marker? int multithreading = 1; // use multithreading if available? char *matchfinder = NULL; // matchfinder algorithm int algorithm = 2; int res; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|iiiiiiiis", kwlist, &inFile, &dictionary, &fastBytes, &literalContextBits, &literalPosBits, &posBits, &algorithm, &eos, &multithreading, &matchfinder)) return -1; CHECK_RANGE(dictionary, 0, 28, "dictionary must be between 0 and 28"); CHECK_RANGE(fastBytes, 5, 255, "fastBytes must be between 5 and 255"); CHECK_RANGE(literalContextBits, 0, 8, "literalContextBits must be between 0 and 8"); CHECK_RANGE(literalPosBits, 0, 4, "literalPosBits must be between 0 and 4"); CHECK_RANGE(posBits, 0, 4, "posBits must be between 0 and 4"); CHECK_RANGE(algorithm, 0, 2, "algorithm must be between 0 and 2"); if (matchfinder != NULL) { #if (PY_VERSION_HEX >= 0x02050000) PyErr_WarnEx(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored", 1); #else PyErr_Warn(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored"); #endif } if (PyString_Check(inFile)) { // create new cStringIO object from string inFile = PycStringIO->NewInput(inFile); if (inFile == NULL) { PyErr_NoMemory(); return -1; } } else if (!PyObject_HasAttrString(inFile, "read")) { PyErr_SetString(PyExc_ValueError, "first parameter must be a file-like object"); return -1; } else { // protect object from being refcounted out... Py_INCREF(inFile); } self->encoder = LzmaEnc_Create(&allocator); if (self->encoder == NULL) { Py_DECREF(inFile); PyErr_NoMemory(); return -1; } LzmaEncProps_Init(&props); props.dictSize = 1 << dictionary; props.lc = literalContextBits; props.lp = literalPosBits; props.pb = posBits; props.algo = algorithm; props.fb = fastBytes; // props.btMode = 1; // props.numHashBytes = 4; // props.mc = 32; props.writeEndMark = eos ? 1 : 0; props.numThreads = multithreading ? 2 : 1; LzmaEncProps_Normalize(&props); res = LzmaEnc_SetProps(self->encoder, &props); if (res != SZ_OK) { Py_DECREF(inFile); PyErr_Format(PyExc_TypeError, "could not set encoder properties: %d", res); return -1; } self->inFile = inFile; CreatePythonInStream(&self->inStream, inFile); CreateMemoryOutStream(&self->outStream); LzmaEnc_WriteProperties(self->encoder, header, &headerSize); if (self->outStream.s.Write(&self->outStream, header, headerSize) != headerSize) { PyErr_SetString(PyExc_TypeError, "could not generate stream header"); goto exit; } LzmaEnc_Prepare(self->encoder, &self->inStream.s, &self->outStream.s, &allocator, &allocator); result = 0; exit: return result; }
static SRes Encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize) { SRes res; size_t inSize = (size_t)fileSize; Byte *inBuffer = 0; Byte *outBuffer = 0; Byte *filteredStream = 0; size_t outSize; CLzmaEncProps props; LzmaEncProps_Init(&props); LzmaEncProps_Normalize(&props); if (inSize != 0) { inBuffer = (Byte *)MyAlloc(inSize); if (inBuffer == 0) return SZ_ERROR_MEM; } else { return SZ_ERROR_INPUT_EOF; } if (SeqInStream_Read(inStream, inBuffer, inSize) != SZ_OK) { res = SZ_ERROR_READ; goto Done; } // we allocate 105% of original size + 64KB for output buffer outSize = (size_t)fileSize / 20 * 21 + (1 << 16); outBuffer = (Byte *)MyAlloc(outSize); if (outBuffer == 0) { res = SZ_ERROR_MEM; goto Done; } { int i; for (i = 0; i < 8; i++) outBuffer[i + LZMA_PROPS_SIZE] = (Byte)(fileSize >> (8 * i)); } if (mConType != NoConverter) { filteredStream = (Byte *)MyAlloc(inSize); if (filteredStream == 0) { res = SZ_ERROR_MEM; goto Done; } memcpy(filteredStream, inBuffer, inSize); if (mConType == X86Converter) { { UInt32 x86State; x86_Convert_Init(x86State); x86_Convert(filteredStream, (SizeT) inSize, 0, &x86State, 1); } } } { size_t outSizeProcessed = outSize - LZMA_HEADER_SIZE; size_t outPropsSize = LZMA_PROPS_SIZE; res = LzmaEncode(outBuffer + LZMA_HEADER_SIZE, &outSizeProcessed, mConType != NoConverter ? filteredStream : inBuffer, inSize, &props, outBuffer, &outPropsSize, 0, NULL, &g_Alloc, &g_Alloc); if (res != SZ_OK) goto Done; outSize = LZMA_HEADER_SIZE + outSizeProcessed; } if (outStream->Write(outStream, outBuffer, outSize) != outSize) res = SZ_ERROR_WRITE; Done: MyFree(outBuffer); MyFree(inBuffer); MyFree(filteredStream); return res; }
PyObject * pylzma_compress(PyObject *self, PyObject *args, PyObject *kwargs) { PyObject *result = NULL; CLzmaEncProps props; CLzmaEncHandle encoder=NULL; CMemoryOutStream outStream; CMemoryInStream inStream; Byte header[LZMA_PROPS_SIZE]; size_t headerSize = LZMA_PROPS_SIZE; int res; // possible keywords for this function static char *kwlist[] = {"data", "dictionary", "fastBytes", "literalContextBits", "literalPosBits", "posBits", "algorithm", "eos", "multithreading", "matchfinder", NULL}; int dictionary = 23; // [0,27], default 23 (8MB) int fastBytes = 128; // [5,273], default 128 int literalContextBits = 3; // [0,8], default 3 int literalPosBits = 0; // [0,4], default 0 int posBits = 2; // [0,4], default 2 int eos = 1; // write "end of stream" marker? int multithreading = 1; // use multithreading if available? char *matchfinder = NULL; // matchfinder algorithm int algorithm = 2; char *data; int length; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|iiiiiiiis", kwlist, &data, &length, &dictionary, &fastBytes, &literalContextBits, &literalPosBits, &posBits, &algorithm, &eos, &multithreading, &matchfinder)) return NULL; outStream.data = NULL; CHECK_RANGE(dictionary, 0, 27, "dictionary must be between 0 and 27"); CHECK_RANGE(fastBytes, 5, 273, "fastBytes must be between 5 and 273"); CHECK_RANGE(literalContextBits, 0, 8, "literalContextBits must be between 0 and 8"); CHECK_RANGE(literalPosBits, 0, 4, "literalPosBits must be between 0 and 4"); CHECK_RANGE(posBits, 0, 4, "posBits must be between 0 and 4"); CHECK_RANGE(algorithm, 0, 2, "algorithm must be between 0 and 2"); if (matchfinder != NULL) { #if (PY_VERSION_HEX >= 0x02050000) PyErr_WarnEx(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored", 1); #else PyErr_Warn(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored"); #endif } encoder = LzmaEnc_Create(&allocator); if (encoder == NULL) return PyErr_NoMemory(); CreateMemoryInStream(&inStream, (Byte *) data, length); CreateMemoryOutStream(&outStream); LzmaEncProps_Init(&props); props.dictSize = 1 << dictionary; props.lc = literalContextBits; props.lp = literalPosBits; props.pb = posBits; props.algo = algorithm; props.fb = fastBytes; // props.btMode = 1; // props.numHashBytes = 4; // props.mc = 32; props.writeEndMark = eos ? 1 : 0; props.numThreads = multithreading ? 2 : 1; LzmaEncProps_Normalize(&props); res = LzmaEnc_SetProps(encoder, &props); if (res != SZ_OK) { PyErr_Format(PyExc_TypeError, "could not set encoder properties: %d", res); goto exit; } Py_BEGIN_ALLOW_THREADS LzmaEnc_WriteProperties(encoder, header, &headerSize); if (outStream.s.Write(&outStream, header, headerSize) != headerSize) { res = SZ_ERROR_WRITE; } else { res = LzmaEnc_Encode(encoder, &outStream.s, &inStream.s, NULL, &allocator, &allocator); } Py_END_ALLOW_THREADS if (res != SZ_OK) { PyErr_Format(PyExc_TypeError, "Error during compressing: %d", res); goto exit; } result = PyBytes_FromStringAndSize((const char *) outStream.data, outStream.size); exit: if (encoder != NULL) { LzmaEnc_Destroy(encoder, &allocator, &allocator); } if (outStream.data != NULL) { free(outStream.data); } return result; }
Vector<uint8_t> Image::serialize() const { Vector<uint8_t> result; #define A(...) appendToVector(result, __VA_ARGS__); //imageinfo A(info.architecture); A(info.baseAddress); A(info.entryPoint); A(info.flag); A(info.platformData); A(info.platformData1); A(info.size); A(fileName); A(static_cast<uint32_t>(exports.size())); for(auto &i : exports) { A(i.address); A(i.forward); A(i.name); A(i.nameHash); A(i.ordinal); } A(static_cast<uint32_t>(sections.size())); for(auto &i : sections) { A(i.name); A(i.baseAddress); A(i.size); A(i.flag); } A(static_cast<uint32_t>(imports.size())); for(auto &i : imports) { A(i.libraryName); A(static_cast<uint32_t>(i.functions.size())); for(auto &j : i.functions) { A(j.iat); A(j.name); A(j.nameHash); A(j.ordinal); } } A(static_cast<uint32_t>(relocations.size())); for(auto &i : relocations) A(i); for(auto &i : sections) { A(i.data->get(), i.data->size()); if((i.flag & SectionFlagCode) && i.data->size() > 5) { uint8_t *codeStart = result.end() - i.data->size(); for(size_t j = 0; j < i.data->size() - 5; j ++) { if(codeStart[j] == 0xE8 || codeStart[j] == 0xE9) //call rel32, jmp rel32 { *reinterpret_cast<int32_t *>(codeStart + j + 1) += (j + 5); j += 4; } else if(codeStart[j] == 0x0f && (codeStart[j + 1] >= 0x80 && codeStart[j + 1] <= 0x8f)) //conditional jmp rel32 { *reinterpret_cast<int32_t *>(codeStart + j + 2) += (j + 6); j += 5; } } } } A(header->get(), header->size()); #undef A CLzmaEncProps props; LzmaEncProps_Init(&props); props.numThreads = 1; LzmaEncProps_Normalize(&props); uint32_t sizeSize = sizeof(uint32_t) * 2; uint32_t propsSize = LZMA_PROPS_SIZE; uint32_t outSize = result.size() + result.size() / 40 + (1 << 12); //igor recommends (http://sourceforge.net/p/sevenzip/discussion/45798/thread/dd3b392c/) Vector<uint8_t> compressed(outSize + propsSize + sizeSize); LzmaEncode(&compressed[propsSize + sizeSize], &outSize, &result[0], result.size(), &props, &compressed[sizeSize], &propsSize, 0, nullptr, &g_Alloc, &g_Alloc); *reinterpret_cast<uint32_t *>(&compressed[0]) = result.size(); *reinterpret_cast<uint32_t *>(&compressed[sizeof(uint32_t)]) = outSize; compressed.resize(outSize + propsSize + sizeSize); return compressed; }