static SRes Encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize, ICompressProgress *pg, int lvl) { CLzmaEncHandle enc; SRes res; CLzmaEncProps props; enc = LzmaEnc_Create(&g_Alloc); if (enc == 0) return SZ_ERROR_MEM; LzmaEncProps_Init(&props); props.level = lvl; res = LzmaEnc_SetProps(enc, &props); if(res == SZ_OK) { Byte header[LZMA_PROPS_SIZE + 8]; size_t headerSize = LZMA_PROPS_SIZE; int i; res = LzmaEnc_WriteProperties(enc, header, &headerSize); for(i = 0; i < 8; i++) header[headerSize++] = (Byte)(fileSize >> (8 * i)); if(outStream->Write(outStream, header, headerSize) != headerSize) res = SZ_ERROR_WRITE; else { if(res == SZ_OK) res = LzmaEnc_Encode(enc, outStream, inStream, pg, &g_Alloc, &g_Alloc); } }
STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream) { Byte props[LZMA_PROPS_SIZE]; size_t size = LZMA_PROPS_SIZE; RINOK(LzmaEnc_WriteProperties(_encoder, props, &size)); return WriteStream(outStream, props, size); }
void CompressInc(std::vector<unsigned char> &outBuf, const std::vector<unsigned char> &inBuf) { CLzmaEncHandle enc = LzmaEnc_Create(&g_Alloc); assert(enc); CLzmaEncProps props; LzmaEncProps_Init(&props); //props.writeEndMark = 1; // 0 or 1 SRes res = LzmaEnc_SetProps(enc, &props); assert(res == SZ_OK); unsigned char tempprops[5]; unsigned propsSize = LZMA_PROPS_SIZE; //outBuf.resize(propsSize); //fill with zero, which cost me a week to check the mistake. res = LzmaEnc_WriteProperties(enc, tempprops, &propsSize); assert(res == SZ_OK && propsSize == LZMA_PROPS_SIZE); VectorInStream inStream = { &VectorInStream_Read, &inBuf, 0 }; VectorOutStream outStream = { &VectorOutStream_Write, &outBuf }; res = LzmaEnc_Encode(enc, (ISeqOutStream*)&outStream, (ISeqInStream*)&inStream, 0, &g_Alloc, &g_Alloc); assert(res == SZ_OK); UInt64 resLen = inBuf.size(); //printf("uint64 resLen : %" PRIu64 "\n", resLen); //print uint64 Byte header[8]; for (int i = 0; i < 8; i++) header[i] = (Byte)(resLen >> (8 * i)); FILE *file = fopen("data3.dat", "wb+"); fwrite(tempprops, 1, propsSize, file); fwrite(&header[0], 1, 8, file); fwrite(&outBuf[0], 1, outBuf.size(), file); fclose(file); LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc); }
SRes MyLzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig) { CLzmaEncHandle *p = (CLzmaEncHandle *)LzmaEnc_Create(alloc); SRes res; if (p == 0) return SZ_ERROR_MEM; res = LzmaEnc_SetProps(p, props); if (res == SZ_OK) { res = LzmaEnc_WriteProperties(p, propsEncoded, propsSize); if (res == SZ_OK) res = LzmaEnc_MemEncode(p, dest, destLen, src, srcLen, writeEndMark, progress, alloc, allocBig); } LzmaEnc_Destroy(p, alloc, allocBig); return res; }
STATIC int INIT lzma_alloc_workspace(CLzmaEncProps *props) { if ((p = (CLzmaEncHandle *)LzmaEnc_Create(&lzma_alloc)) == NULL) { PRINT_ERROR("Failed to allocate lzma deflate workspace\n"); return -ENOMEM; } if (LzmaEnc_SetProps(p, props) != SZ_OK) { lzma_free_workspace(); return -1; } if (LzmaEnc_WriteProperties(p, propsEncoded, &propsSize) != SZ_OK) { lzma_free_workspace(); return -1; } return 0; }
static SRes Encode(FILE *inFile, FILE *outFile, char *rs) { CLzmaEncHandle enc; SRes res; CFileSeqInStream inStream; CFileSeqOutStream outStream; CLzmaEncProps props; enc = LzmaEnc_Create(&g_Alloc); if (enc == 0) return SZ_ERROR_MEM; inStream.funcTable.Read = MyRead; inStream.file = inFile; outStream.funcTable.Write = MyWrite; outStream.file = outFile; LzmaEncProps_Init(&props); res = LzmaEnc_SetProps(enc, &props); if (res == SZ_OK) { Byte header[LZMA_PROPS_SIZE + 8]; size_t headerSize = LZMA_PROPS_SIZE; UInt64 fileSize; int i; res = LzmaEnc_WriteProperties(enc, header, &headerSize); fileSize = MyGetFileLength(inFile); for (i = 0; i < 8; i++) header[headerSize++] = (Byte)(fileSize >> (8 * i)); if (!MyWriteFileAndCheck(outFile, header, headerSize)) return PrintError(rs, "writing error"); if (res == SZ_OK) res = LzmaEnc_Encode(enc, &outStream.funcTable, &inStream.funcTable, NULL, &g_Alloc, &g_Alloc); }
inline TLzmaCompressBase(size_t level) : H_(LzmaEnc_Create(Alloc())) { if (!H_) { ythrow yexception() << "can not init lzma engine"; } LzmaEncProps_Init(&Props_); Props_.level = level; Props_.dictSize = 0; Props_.lc = -1; Props_.lp = -1; Props_.pb = -1; Props_.fb = -1; Props_.numThreads = -1; Props_.writeEndMark = 1; Check(LzmaEnc_SetProps(H_, &Props_)); size_t bufLen = sizeof(PropsBuf_); Zero(PropsBuf_); Check(LzmaEnc_WriteProperties(H_, PropsBuf_, &bufLen)); }
size32_t compress(const void* input, size32_t inlength, void* output) { // returns (size32_t)-1 if cannot compress if (!enc) { enc = LzmaEnc_Create(&g_Alloc); if (enc == 0) throw MakeStringException(-1,"LzmaEnc_Create failed"); LzmaEncProps_Init(&props); if (LzmaEnc_SetProps(enc, &props)!=SZ_OK) throw MakeStringException(-1,"LzmaEnc_SetProps failed"); } if (inlength+LZMA_PROPS_SIZE+sizeof(size32_t)<1024) return (size32_t)-1; // don't compress less than 1K SizeT propsize = LZMA_PROPS_SIZE; LzmaEnc_WriteProperties(enc, (byte *)output+sizeof(size32_t), &propsize); *(size32_t *)output = (size32_t)propsize; SizeT reslen = inlength-1-propsize-sizeof(size32_t); SRes res = LzmaEnc_MemEncode(enc, (byte *)output+propsize+sizeof(size32_t), &reslen, (const byte *)input, inlength, true, NULL, &g_Alloc, &g_Alloc); if (res==SZ_ERROR_OUTPUT_EOF) return (size32_t)-1; if (res!=SZ_OK) throw MakeStringException(-1,"LzmaEnc_MemEncode failed(%d)",(int)res); return reslen+propsize+sizeof(size32_t); }
static int pylzma_compfile_init(CCompressionFileObject *self, PyObject *args, PyObject *kwargs) { PyObject *inFile; CLzmaEncProps props; Byte header[LZMA_PROPS_SIZE]; size_t headerSize = LZMA_PROPS_SIZE; int result = -1; // possible keywords for this function static char *kwlist[] = {"infile", "dictionary", "fastBytes", "literalContextBits", "literalPosBits", "posBits", "algorithm", "eos", "multithreading", "matchfinder", NULL}; int dictionary = 23; // [0,28], default 23 (8MB) int fastBytes = 128; // [5,255], default 128 int literalContextBits = 3; // [0,8], default 3 int literalPosBits = 0; // [0,4], default 0 int posBits = 2; // [0,4], default 2 int eos = 1; // write "end of stream" marker? int multithreading = 1; // use multithreading if available? char *matchfinder = NULL; // matchfinder algorithm int algorithm = 2; int res; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|iiiiiiiis", kwlist, &inFile, &dictionary, &fastBytes, &literalContextBits, &literalPosBits, &posBits, &algorithm, &eos, &multithreading, &matchfinder)) return -1; CHECK_RANGE(dictionary, 0, 28, "dictionary must be between 0 and 28"); CHECK_RANGE(fastBytes, 5, 255, "fastBytes must be between 5 and 255"); CHECK_RANGE(literalContextBits, 0, 8, "literalContextBits must be between 0 and 8"); CHECK_RANGE(literalPosBits, 0, 4, "literalPosBits must be between 0 and 4"); CHECK_RANGE(posBits, 0, 4, "posBits must be between 0 and 4"); CHECK_RANGE(algorithm, 0, 2, "algorithm must be between 0 and 2"); if (matchfinder != NULL) { #if (PY_VERSION_HEX >= 0x02050000) PyErr_WarnEx(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored", 1); #else PyErr_Warn(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored"); #endif } if (PyString_Check(inFile)) { // create new cStringIO object from string inFile = PycStringIO->NewInput(inFile); if (inFile == NULL) { PyErr_NoMemory(); return -1; } } else if (!PyObject_HasAttrString(inFile, "read")) { PyErr_SetString(PyExc_ValueError, "first parameter must be a file-like object"); return -1; } else { // protect object from being refcounted out... Py_INCREF(inFile); } self->encoder = LzmaEnc_Create(&allocator); if (self->encoder == NULL) { Py_DECREF(inFile); PyErr_NoMemory(); return -1; } LzmaEncProps_Init(&props); props.dictSize = 1 << dictionary; props.lc = literalContextBits; props.lp = literalPosBits; props.pb = posBits; props.algo = algorithm; props.fb = fastBytes; // props.btMode = 1; // props.numHashBytes = 4; // props.mc = 32; props.writeEndMark = eos ? 1 : 0; props.numThreads = multithreading ? 2 : 1; LzmaEncProps_Normalize(&props); res = LzmaEnc_SetProps(self->encoder, &props); if (res != SZ_OK) { Py_DECREF(inFile); PyErr_Format(PyExc_TypeError, "could not set encoder properties: %d", res); return -1; } self->inFile = inFile; CreatePythonInStream(&self->inStream, inFile); CreateMemoryOutStream(&self->outStream); LzmaEnc_WriteProperties(self->encoder, header, &headerSize); if (self->outStream.s.Write(&self->outStream, header, headerSize) != headerSize) { PyErr_SetString(PyExc_TypeError, "could not generate stream header"); goto exit; } LzmaEnc_Prepare(self->encoder, &self->inStream.s, &self->outStream.s, &allocator, &allocator); result = 0; exit: return result; }
PyObject * pylzma_compress(PyObject *self, PyObject *args, PyObject *kwargs) { PyObject *result = NULL; CLzmaEncProps props; CLzmaEncHandle encoder=NULL; CMemoryOutStream outStream; CMemoryInStream inStream; Byte header[LZMA_PROPS_SIZE]; size_t headerSize = LZMA_PROPS_SIZE; int res; // possible keywords for this function static char *kwlist[] = {"data", "dictionary", "fastBytes", "literalContextBits", "literalPosBits", "posBits", "algorithm", "eos", "multithreading", "matchfinder", NULL}; int dictionary = 23; // [0,27], default 23 (8MB) int fastBytes = 128; // [5,273], default 128 int literalContextBits = 3; // [0,8], default 3 int literalPosBits = 0; // [0,4], default 0 int posBits = 2; // [0,4], default 2 int eos = 1; // write "end of stream" marker? int multithreading = 1; // use multithreading if available? char *matchfinder = NULL; // matchfinder algorithm int algorithm = 2; char *data; int length; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|iiiiiiiis", kwlist, &data, &length, &dictionary, &fastBytes, &literalContextBits, &literalPosBits, &posBits, &algorithm, &eos, &multithreading, &matchfinder)) return NULL; outStream.data = NULL; CHECK_RANGE(dictionary, 0, 27, "dictionary must be between 0 and 27"); CHECK_RANGE(fastBytes, 5, 273, "fastBytes must be between 5 and 273"); CHECK_RANGE(literalContextBits, 0, 8, "literalContextBits must be between 0 and 8"); CHECK_RANGE(literalPosBits, 0, 4, "literalPosBits must be between 0 and 4"); CHECK_RANGE(posBits, 0, 4, "posBits must be between 0 and 4"); CHECK_RANGE(algorithm, 0, 2, "algorithm must be between 0 and 2"); if (matchfinder != NULL) { #if (PY_VERSION_HEX >= 0x02050000) PyErr_WarnEx(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored", 1); #else PyErr_Warn(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored"); #endif } encoder = LzmaEnc_Create(&allocator); if (encoder == NULL) return PyErr_NoMemory(); CreateMemoryInStream(&inStream, (Byte *) data, length); CreateMemoryOutStream(&outStream); LzmaEncProps_Init(&props); props.dictSize = 1 << dictionary; props.lc = literalContextBits; props.lp = literalPosBits; props.pb = posBits; props.algo = algorithm; props.fb = fastBytes; // props.btMode = 1; // props.numHashBytes = 4; // props.mc = 32; props.writeEndMark = eos ? 1 : 0; props.numThreads = multithreading ? 2 : 1; LzmaEncProps_Normalize(&props); res = LzmaEnc_SetProps(encoder, &props); if (res != SZ_OK) { PyErr_Format(PyExc_TypeError, "could not set encoder properties: %d", res); goto exit; } Py_BEGIN_ALLOW_THREADS LzmaEnc_WriteProperties(encoder, header, &headerSize); if (outStream.s.Write(&outStream, header, headerSize) != headerSize) { res = SZ_ERROR_WRITE; } else { res = LzmaEnc_Encode(encoder, &outStream.s, &inStream.s, NULL, &allocator, &allocator); } Py_END_ALLOW_THREADS if (res != SZ_OK) { PyErr_Format(PyExc_TypeError, "Error during compressing: %d", res); goto exit; } result = PyBytes_FromStringAndSize((const char *) outStream.data, outStream.size); exit: if (encoder != NULL) { LzmaEnc_Destroy(encoder, &allocator, &allocator); } if (outStream.data != NULL) { free(outStream.data); } return result; }