int lzma_compress_size( const unsigned char* data, unsigned size ){ //Initialize LZMA lzma_stream strm = LZMA_STREAM_INIT; if( lzma_easy_encoder( &strm, 9 | LZMA_PRESET_EXTREME, LZMA_CHECK_CRC64 ) != LZMA_OK ) return {}; std::vector<uint8_t> out( size*2 ); //TODO: Find value properly //Setup in/out buffers strm.next_in = (uint8_t*)data; strm.avail_in = size; strm.next_out = out.data(); strm.avail_out = out.size(); //Compress lzma_ret ret = lzma_code( &strm, LZMA_FINISH ); if( ret != LZMA_STREAM_END ){ lzma_end( &strm ); //TODO: Use RAII return {}; } lzma_end( &strm ); //Return only the range actually needed //return out.left( out.size() - strm.avail_out ); return out.size() - strm.avail_out; }
struct ostream *o_stream_create_lzma(struct ostream *output, int level) { struct lzma_ostream *zstream; lzma_ret ret; i_assert(level >= 1 && level <= 9); zstream = i_new(struct lzma_ostream, 1); zstream->ostream.sendv = o_stream_lzma_sendv; zstream->ostream.flush = o_stream_lzma_flush; zstream->ostream.iostream.close = o_stream_lzma_close; ret = lzma_easy_encoder(&zstream->strm, level, LZMA_CHECK_CRC64); switch (ret) { case LZMA_OK: break; case LZMA_MEM_ERROR: i_fatal_status(FATAL_OUTOFMEM, "lzma: Out of memory"); case LZMA_OPTIONS_ERROR: i_fatal("lzma: Invalid level"); default: i_fatal("lzma_easy_encoder() failed with %d", ret); } zstream->strm.next_out = zstream->outbuf; zstream->strm.avail_out = sizeof(zstream->outbuf); return o_stream_create(&zstream->ostream, output, o_stream_get_fd(output)); }
static int xz_open(struct zstream *z, int fd, char *mode) { lzma_ret ret; z->xz.eof = 0; z->xz.fd = fd; if(*mode == 'w') { ret = lzma_easy_encoder(&z->xz.stream, 5, LZMA_CHECK_CRC64); if (ret == LZMA_OK) return 0; return -1; } if(*mode == 'r') { z->xz.stream.next_in = NULL; z->xz.stream.avail_in = 0; z->xz.stream.next_out = z->xz.outbuf; z->xz.stream.avail_out = z->xz.bufsize; ret = lzma_stream_decoder(&z->xz.stream, UINT64_MAX, LZMA_CONCATENATED); if (ret == LZMA_OK) return 0; return -1; } return -1; }
/*@null@*/ static XZFILE *xzopen_internal(const char *path, const char *mode, int fdno, int xz) /*@globals fileSystem @*/ /*@modifies fileSystem @*/ { int level = LZMA_PRESET_DEFAULT; int encoding = 0; FILE *fp; XZFILE *xzfile; lzma_stream tmp; lzma_ret ret; for (; *mode != '\0'; mode++) { if (*mode == 'w') encoding = 1; else if (*mode == 'r') encoding = 0; else if (*mode >= '0' && *mode <= '9') level = (int)(*mode - '0'); } if (fdno != -1) fp = fdopen(fdno, encoding ? "w" : "r"); else fp = fopen(path, encoding ? "w" : "r"); if (!fp) return NULL; xzfile = calloc(1, sizeof(*xzfile)); if (!xzfile) { (void) fclose(fp); return NULL; } xzfile->fp = fp; xzfile->encoding = encoding; xzfile->eof = 0; tmp = (lzma_stream)LZMA_STREAM_INIT; xzfile->strm = tmp; if (encoding) { if (xz) { ret = lzma_easy_encoder(&xzfile->strm, level, LZMA_CHECK_CRC32); } else { lzma_options_lzma options; (void) lzma_lzma_preset(&options, level); ret = lzma_alone_encoder(&xzfile->strm, &options); } } else { /* We set the memlimit for decompression to 100MiB which should be * more than enough to be sufficient for level 9 which requires 65 MiB. */ ret = lzma_auto_decoder(&xzfile->strm, 100<<20, 0); } if (ret != LZMA_OK) { (void) fclose(fp); memset(xzfile, 0, sizeof(*xzfile)); free(xzfile); return NULL; } return xzfile; }
struct lzma_filter *lzma_encoder(void) { lzma_stream strm = LZMA_STREAM_INIT; uint32_t preset = 6; // TODO select maybe? see 'man xz' for -0..-9 if (lzma_easy_encoder(&strm, preset, LZMA_CHECK_CRC64) != LZMA_OK) return NULL; return _lzma_filter_new(&strm); }
void lzma_base::do_init( const lzma_params &p, bool compress, lzma::alloc_func alloc, lzma::free_func free, void *derived ) { lzma_stream *s = static_cast<lzma_stream *>( stream_ ); memset( s, 0, sizeof( *s ) ); lzma_error::check( compress ? lzma_easy_encoder( s, p.level, LZMA_CHECK_CRC32 ) : lzma_stream_decoder( s, 100 * 1024 * 1024, 0 ) ); }
static LZFILE *lzopen(const char *path, const char *mode, int fd, int isxz) { int level = 7; int encoding = 0; FILE *fp; LZFILE *lzfile; lzma_ret ret; if (!path && fd < 0) return 0; for (; *mode; mode++) { if (*mode == 'w') encoding = 1; else if (*mode == 'r') encoding = 0; else if (*mode >= '1' && *mode <= '9') level = *mode - '0'; } if (fd != -1) fp = fdopen(fd, encoding ? "w" : "r"); else fp = fopen(path, encoding ? "w" : "r"); if (!fp) return 0; lzfile = calloc(1, sizeof(*lzfile)); if (!lzfile) { fclose(fp); return 0; } lzfile->file = fp; lzfile->encoding = encoding; lzfile->eof = 0; lzfile->strm = stream_init; if (encoding) { if (isxz) ret = lzma_easy_encoder(&lzfile->strm, level, LZMA_CHECK_SHA256); else ret = setup_alone_encoder(&lzfile->strm, level); } else ret = lzma_auto_decoder(&lzfile->strm, 100 << 20, 0); if (ret != LZMA_OK) { fclose(fp); free(lzfile); return 0; } return lzfile; }
static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int xz) { int level = 7; /* Use XZ's default compression level if unspecified */ int encoding = 0; FILE *fp; LZFILE *lzfile; lzma_ret ret; lzma_stream init_strm = LZMA_STREAM_INIT; for (; *mode; mode++) { if (*mode == 'w') encoding = 1; else if (*mode == 'r') encoding = 0; else if (*mode >= '1' && *mode <= '9') level = *mode - '0'; } if (fd != -1) fp = fdopen(fd, encoding ? "w" : "r"); else fp = fopen(path, encoding ? "w" : "r"); if (!fp) return 0; lzfile = calloc(1, sizeof(*lzfile)); if (!lzfile) { fclose(fp); return 0; } lzfile->file = fp; lzfile->encoding = encoding; lzfile->eof = 0; lzfile->strm = init_strm; if (encoding) { if (xz) { ret = lzma_easy_encoder(&lzfile->strm, level, LZMA_CHECK_SHA256); } else { lzma_options_lzma options; lzma_lzma_preset(&options, level); ret = lzma_alone_encoder(&lzfile->strm, &options); } } else { /* lzma_easy_decoder_memusage(level) is not ready yet, use hardcoded limit for now */ ret = lzma_auto_decoder(&lzfile->strm, 100<<20, 0); } if (ret != LZMA_OK) { fclose(fp); free(lzfile); return 0; } return lzfile; }
static void filter_xz_init(struct io_lzma *io, lzma_stream *s) { uint32_t preset; lzma_ret ret; io->status |= DPKG_STREAM_COMPRESS; preset = io->params->level; if (io->params->strategy == COMPRESSOR_STRATEGY_EXTREME) preset |= LZMA_PRESET_EXTREME; ret = lzma_easy_encoder(s, preset, LZMA_CHECK_CRC32); if (ret != LZMA_OK) filter_lzma_error(io, ret); }
LzmaConnectorWriter::LzmaConnectorWriter (const CompressionSetting * const pCompressionSetting, unsigned long ulOutBufSize) : ConnectorWriter (pCompressionSetting) { if (ulOutBufSize == 0) { // Throw C++ exception here } _ulOutBufSize = ulOutBufSize; if ((_pOutputBuffer = new unsigned char [_ulOutBufSize]) == NULL) { // throw C++ exception here } resetCompStream(); if (LZMA_OK != lzma_easy_encoder (&_lzmaCompStream, getCompressionLevel(), COMPRESSION_CHECK)) { // Throw C++ exception here } }
static bool init_encoder(lzma_stream *strm, uint32_t preset) { // Initialize the encoder using a preset. Set the integrity to check // to CRC64, which is the default in the xz command line tool. If // the .xz file needs to be decompressed with XZ Embedded, use // LZMA_CHECK_CRC32 instead. lzma_ret ret = lzma_easy_encoder(strm, preset, LZMA_CHECK_CRC64); // Return successfully if the initialization went fine. if (ret == LZMA_OK) return true; // Something went wrong. The possible errors are documented in // lzma/container.h (src/liblzma/api/lzma/container.h in the source // package or e.g. /usr/include/lzma/container.h depending on the // install prefix). const char *msg; switch (ret) { case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break; case LZMA_OPTIONS_ERROR: msg = "Specified preset is not supported"; break; case LZMA_UNSUPPORTED_CHECK: msg = "Specified integrity check is not supported"; break; default: // This is most likely LZMA_PROG_ERROR indicating a bug in // this program or in liblzma. It is inconvenient to have a // separate error message for errors that should be impossible // to occur, but knowing the error code is important for // debugging. That's why it is good to print the error code // at least when there is no good error message to show. msg = "Unknown error, possibly a bug"; break; } fprintf(stderr, "Error initializing the encoder: %s (error code %u)\n", msg, ret); return false; }
int import_compress_init(ImportCompress *c, ImportCompressType t) { int r; assert(c); switch (t) { case IMPORT_COMPRESS_XZ: { lzma_ret xzr; xzr = lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64); if (xzr != LZMA_OK) return -EIO; c->type = IMPORT_COMPRESS_XZ; break; } case IMPORT_COMPRESS_GZIP: r = deflateInit2(&c->gzip, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY); if (r != Z_OK) return -EIO; c->type = IMPORT_COMPRESS_GZIP; break; case IMPORT_COMPRESS_BZIP2: r = BZ2_bzCompressInit(&c->bzip2, 9, 0, 0); if (r != BZ_OK) return -EIO; c->type = IMPORT_COMPRESS_BZIP2; break; case IMPORT_COMPRESS_UNCOMPRESSED: c->type = IMPORT_COMPRESS_UNCOMPRESSED; break; default: return -EOPNOTSUPP; } c->encoding = true; return 0; }
static void filter_xz_init(struct io_lzma *io, lzma_stream *s) { uint32_t preset; lzma_check check = LZMA_CHECK_CRC64; #ifdef HAVE_LZMA_MT lzma_mt mt_options = { .flags = 0, .threads = sysconf(_SC_NPROCESSORS_ONLN), .block_size = 0, .timeout = 0, .filters = NULL, .check = check, }; #endif lzma_ret ret; io->status |= DPKG_STREAM_COMPRESS; preset = io->params->level; if (io->params->strategy == COMPRESSOR_STRATEGY_EXTREME) preset |= LZMA_PRESET_EXTREME; #ifdef HAVE_LZMA_MT mt_options.preset = preset; ret = lzma_stream_encoder_mt(s, &mt_options); #else ret = lzma_easy_encoder(s, preset, check); #endif if (ret != LZMA_OK) filter_lzma_error(io, ret); } static int filter_lzma_code(struct io_lzma *io, lzma_stream *s) { lzma_ret ret; ret = lzma_code(s, io->action); if (ret != LZMA_OK && ret != LZMA_STREAM_END) filter_lzma_error(io, ret); return ret; }
static void write_xz(int f, int fd, const char *arg) { uint8_t inbuf[BUFFER_SIZE], outbuf[BUFFER_SIZE]; lzma_stream xz = LZMA_STREAM_INIT; if (lzma_easy_encoder(&xz, 6, LZMA_CHECK_CRC64) != LZMA_OK) goto xz_write_end; xz.avail_in = 0; while (xz.avail_in || (xz.avail_in = read(fd, (uint8_t*)(xz.next_in = inbuf), BUFFER_SIZE)) > 0) { xz.next_out = outbuf; xz.avail_out = sizeof(outbuf); if (lzma_code(&xz, LZMA_RUN) != LZMA_OK) goto xz_write_lzma_end; if (write(f, outbuf, xz.next_out - outbuf) != xz.next_out - outbuf) goto xz_write_lzma_end; } // Flush the stream lzma_ret ret; do { xz.next_out = outbuf; xz.avail_out = sizeof(outbuf); ret = lzma_code(&xz, LZMA_FINISH); if (write(f, outbuf, xz.next_out - outbuf) != xz.next_out - outbuf) goto xz_write_lzma_end; } while (ret == LZMA_OK); xz_write_lzma_end: lzma_end(&xz); xz_write_end: close(f); close(fd); }
static int Compressor_init_xz(lzma_stream *lzs, int check, uint32_t preset, PyObject *filterspecs) { lzma_ret lzret; if (filterspecs == Py_None) { lzret = lzma_easy_encoder(lzs, preset, check); } else { lzma_filter filters[LZMA_FILTERS_MAX + 1]; if (parse_filter_chain_spec(filters, filterspecs) == -1) return -1; lzret = lzma_stream_encoder(lzs, filters, check); free_filter_chain(filters); } if (catch_lzma_error(lzret)) return -1; else return 0; }
int xz_enc_init(lzma_stream *str) { lzma_ret ret_xz; ret_xz = lzma_easy_encoder(str, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64); return ret_xz; }
int main(int argc, char *argv[]) { QString workDir; #ifdef Q_OS_MAC if (QDir(QString()).absolutePath() == "/") { QString first = argc ? QString::fromLocal8Bit(argv[0]) : QString(); if (!first.isEmpty()) { QFileInfo info(first); if (info.exists()) { QDir result(info.absolutePath() + "/../../.."); workDir = result.absolutePath() + '/'; } } } #endif QString remove; int version = 0; QFileInfoList files; for (int i = 0; i < argc; ++i) { if (string("-path") == argv[i] && i + 1 < argc) { QString path = workDir + QString(argv[i + 1]); QFileInfo info(path); files.push_back(info); if (remove.isEmpty()) remove = info.canonicalPath() + "/"; } else if (string("-version") == argv[i] && i + 1 < argc) { version = QString(argv[i + 1]).toInt(); } else if (string("-dev") == argv[i]) { DevChannel = true; } else if (string("-beta") == argv[i] && i + 1 < argc) { BetaVersion = QString(argv[i + 1]).toULongLong(); if (BetaVersion > version * 1000ULL && BetaVersion < (version + 1) * 1000ULL) { DevChannel = false; BetaSignature = countBetaVersionSignature(BetaVersion); if (BetaSignature.isEmpty()) { return -1; } } else { cout << "Bad -beta param value passed, should be for the same version: " << version << ", beta: " << BetaVersion << "\n"; return -1; } } } if (files.isEmpty() || remove.isEmpty() || version <= 1016 || version > 999999999) { #ifdef Q_OS_WIN cout << "Usage: Packer.exe -path {file} -version {version} OR Packer.exe -path {dir} -version {version}\n"; #elif defined Q_OS_MAC cout << "Usage: Packer.app -path {file} -version {version} OR Packer.app -path {dir} -version {version}\n"; #else cout << "Usage: Packer -path {file} -version {version} OR Packer -path {dir} -version {version}\n"; #endif return -1; } bool hasDirs = true; while (hasDirs) { hasDirs = false; for (QFileInfoList::iterator i = files.begin(); i != files.end(); ++i) { QFileInfo info(*i); QString fullPath = info.canonicalFilePath(); if (info.isDir()) { hasDirs = true; files.erase(i); QDir d = QDir(info.absoluteFilePath()); QString fullDir = d.canonicalPath(); QStringList entries = d.entryList(QDir::Files | QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot); files.append(d.entryInfoList(QDir::Files | QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot)); break; } else if (!info.isReadable()) { cout << "Can't read: " << info.absoluteFilePath().toUtf8().constData() << "\n"; return -1; } else if (info.isHidden()) { hasDirs = true; files.erase(i); break; } } } for (QFileInfoList::iterator i = files.begin(); i != files.end(); ++i) { QFileInfo info(*i); if (!info.canonicalFilePath().startsWith(remove)) { cout << "Can't find '" << remove.toUtf8().constData() << "' in file '" << info.canonicalFilePath().toUtf8().constData() << "' :(\n"; return -1; } } QByteArray result; { QBuffer buffer(&result); buffer.open(QIODevice::WriteOnly); QDataStream stream(&buffer); stream.setVersion(QDataStream::Qt_5_1); if (BetaVersion) { stream << quint32(0x7FFFFFFF); stream << quint64(BetaVersion); } else { stream << quint32(version); } stream << quint32(files.size()); cout << "Found " << files.size() << " file" << (files.size() == 1 ? "" : "s") << "..\n"; for (QFileInfoList::iterator i = files.begin(); i != files.end(); ++i) { QFileInfo info(*i); QString fullName = info.canonicalFilePath(); QString name = fullName.mid(remove.length()); cout << name.toUtf8().constData() << " (" << info.size() << ")\n"; QFile f(fullName); if (!f.open(QIODevice::ReadOnly)) { cout << "Can't open '" << fullName.toUtf8().constData() << "' for read..\n"; return -1; } QByteArray inner = f.readAll(); stream << name << quint32(inner.size()) << inner; #if defined Q_OS_MAC || defined Q_OS_LINUX stream << (QFileInfo(fullName).isExecutable() ? true : false); #endif } if (stream.status() != QDataStream::Ok) { cout << "Stream status is bad: " << stream.status() << "\n"; return -1; } } int32 resultSize = result.size(); cout << "Compression start, size: " << resultSize << "\n"; QByteArray compressed, resultCheck; #ifdef Q_OS_WIN // use Lzma SDK for win const int32 hSigLen = 128, hShaLen = 20, hPropsLen = LZMA_PROPS_SIZE, hOriginalSizeLen = sizeof(int32), hSize = hSigLen + hShaLen + hPropsLen + hOriginalSizeLen; // header compressed.resize(hSize + resultSize + 1024 * 1024); // rsa signature + sha1 + lzma props + max compressed size size_t compressedLen = compressed.size() - hSize; size_t outPropsSize = LZMA_PROPS_SIZE; uchar *_dest = (uchar*)(compressed.data() + hSize); size_t *_destLen = &compressedLen; const uchar *_src = (const uchar*)(result.constData()); size_t _srcLen = result.size(); uchar *_outProps = (uchar*)(compressed.data() + hSigLen + hShaLen); int res = LzmaCompress(_dest, _destLen, _src, _srcLen, _outProps, &outPropsSize, 9, 64 * 1024 * 1024, 4, 0, 2, 273, 2); if (res != SZ_OK) { cout << "Error in compression: " << res << "\n"; return -1; } compressed.resize(int(hSize + compressedLen)); memcpy(compressed.data() + hSigLen + hShaLen + hPropsLen, &resultSize, hOriginalSizeLen); cout << "Compressed to size: " << compressedLen << "\n"; cout << "Checking uncompressed..\n"; int32 resultCheckLen; memcpy(&resultCheckLen, compressed.constData() + hSigLen + hShaLen + hPropsLen, hOriginalSizeLen); if (resultCheckLen <= 0 || resultCheckLen > 1024 * 1024 * 1024) { cout << "Bad result len: " << resultCheckLen << "\n"; return -1; } resultCheck.resize(resultCheckLen); size_t resultLen = resultCheck.size(); SizeT srcLen = compressedLen; int uncompressRes = LzmaUncompress((uchar*)resultCheck.data(), &resultLen, (const uchar*)(compressed.constData() + hSize), &srcLen, (const uchar*)(compressed.constData() + hSigLen + hShaLen), LZMA_PROPS_SIZE); if (uncompressRes != SZ_OK) { cout << "Uncompress failed: " << uncompressRes << "\n"; return -1; } if (resultLen != size_t(result.size())) { cout << "Uncompress bad size: " << resultLen << ", was: " << result.size() << "\n"; return -1; } #else // use liblzma for others const int32 hSigLen = 128, hShaLen = 20, hPropsLen = 0, hOriginalSizeLen = sizeof(int32), hSize = hSigLen + hShaLen + hOriginalSizeLen; // header compressed.resize(hSize + resultSize + 1024 * 1024); // rsa signature + sha1 + lzma props + max compressed size size_t compressedLen = compressed.size() - hSize; lzma_stream stream = LZMA_STREAM_INIT; int preset = 9 | LZMA_PRESET_EXTREME; lzma_ret ret = lzma_easy_encoder(&stream, preset, LZMA_CHECK_CRC64); if (ret != LZMA_OK) { const char *msg; switch (ret) { case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break; case LZMA_OPTIONS_ERROR: msg = "Specified preset is not supported"; break; case LZMA_UNSUPPORTED_CHECK: msg = "Specified integrity check is not supported"; break; default: msg = "Unknown error, possibly a bug"; break; } cout << "Error initializing the encoder: " << msg << " (error code " << ret << ")\n"; return -1; } stream.avail_in = resultSize; stream.next_in = (uint8_t*)result.constData(); stream.avail_out = compressedLen; stream.next_out = (uint8_t*)(compressed.data() + hSize); lzma_ret res = lzma_code(&stream, LZMA_FINISH); compressedLen -= stream.avail_out; lzma_end(&stream); if (res != LZMA_OK && res != LZMA_STREAM_END) { const char *msg; switch (res) { case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break; case LZMA_DATA_ERROR: msg = "File size limits exceeded"; break; default: msg = "Unknown error, possibly a bug"; break; } cout << "Error in compression: " << msg << " (error code " << res << ")\n"; return -1; } compressed.resize(int(hSize + compressedLen)); memcpy(compressed.data() + hSigLen + hShaLen, &resultSize, hOriginalSizeLen); cout << "Compressed to size: " << compressedLen << "\n"; cout << "Checking uncompressed..\n"; int32 resultCheckLen; memcpy(&resultCheckLen, compressed.constData() + hSigLen + hShaLen, hOriginalSizeLen); if (resultCheckLen <= 0 || resultCheckLen > 1024 * 1024 * 1024) { cout << "Bad result len: " << resultCheckLen << "\n"; return -1; } resultCheck.resize(resultCheckLen); size_t resultLen = resultCheck.size(); stream = LZMA_STREAM_INIT; ret = lzma_stream_decoder(&stream, UINT64_MAX, LZMA_CONCATENATED); if (ret != LZMA_OK) { const char *msg; switch (ret) { case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break; case LZMA_OPTIONS_ERROR: msg = "Specified preset is not supported"; break; case LZMA_UNSUPPORTED_CHECK: msg = "Specified integrity check is not supported"; break; default: msg = "Unknown error, possibly a bug"; break; } cout << "Error initializing the decoder: " << msg << " (error code " << ret << ")\n"; return -1; } stream.avail_in = compressedLen; stream.next_in = (uint8_t*)(compressed.constData() + hSize); stream.avail_out = resultLen; stream.next_out = (uint8_t*)resultCheck.data(); res = lzma_code(&stream, LZMA_FINISH); if (stream.avail_in) { cout << "Error in decompression, " << stream.avail_in << " bytes left in _in of " << compressedLen << " whole.\n"; return -1; } else if (stream.avail_out) { cout << "Error in decompression, " << stream.avail_out << " bytes free left in _out of " << resultLen << " whole.\n"; return -1; } lzma_end(&stream); if (res != LZMA_OK && res != LZMA_STREAM_END) { const char *msg; switch (res) { case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break; case LZMA_FORMAT_ERROR: msg = "The input data is not in the .xz format"; break; case LZMA_OPTIONS_ERROR: msg = "Unsupported compression options"; break; case LZMA_DATA_ERROR: msg = "Compressed file is corrupt"; break; case LZMA_BUF_ERROR: msg = "Compressed data is truncated or otherwise corrupt"; break; default: msg = "Unknown error, possibly a bug"; break; } cout << "Error in decompression: " << msg << " (error code " << res << ")\n"; return -1; } #endif if (memcmp(result.constData(), resultCheck.constData(), resultLen)) { cout << "Data differ :(\n"; return -1; } /**/ result = resultCheck = QByteArray(); cout << "Counting SHA1 hash..\n"; uchar sha1Buffer[20]; memcpy(compressed.data() + hSigLen, hashSha1(compressed.constData() + hSigLen + hShaLen, uint32(compressedLen + hPropsLen + hOriginalSizeLen), sha1Buffer), hShaLen); // count sha1 uint32 siglen = 0; cout << "Signing..\n"; RSA *prKey = PEM_read_bio_RSAPrivateKey(BIO_new_mem_buf(const_cast<char*>((DevChannel || BetaVersion) ? PrivateDevKey : PrivateKey), -1), 0, 0, 0); if (!prKey) { cout << "Could not read RSA private key!\n"; return -1; } if (RSA_size(prKey) != hSigLen) { cout << "Bad private key, size: " << RSA_size(prKey) << "\n"; RSA_free(prKey); return -1; } if (RSA_sign(NID_sha1, (const uchar*)(compressed.constData() + hSigLen), hShaLen, (uchar*)(compressed.data()), &siglen, prKey) != 1) { // count signature cout << "Signing failed!\n"; RSA_free(prKey); return -1; } RSA_free(prKey); if (siglen != hSigLen) { cout << "Bad signature length: " << siglen << "\n"; return -1; } cout << "Checking signature..\n"; RSA *pbKey = PEM_read_bio_RSAPublicKey(BIO_new_mem_buf(const_cast<char*>((DevChannel || BetaVersion) ? PublicDevKey : PublicKey), -1), 0, 0, 0); if (!pbKey) { cout << "Could not read RSA public key!\n"; return -1; } if (RSA_verify(NID_sha1, (const uchar*)(compressed.constData() + hSigLen), hShaLen, (const uchar*)(compressed.constData()), siglen, pbKey) != 1) { // verify signature RSA_free(pbKey); cout << "Signature verification failed!\n"; return -1; } cout << "Signature verified!\n"; RSA_free(pbKey); #ifdef Q_OS_WIN QString outName(QString("tupdate%1").arg(BetaVersion ? BetaVersion : version)); #elif defined Q_OS_MAC QString outName(QString("tmacupd%1").arg(BetaVersion ? BetaVersion : version)); #elif defined Q_OS_LINUX32 QString outName(QString("tlinux32upd%1").arg(BetaVersion ? BetaVersion : version)); #elif defined Q_OS_LINUX64 QString outName(QString("tlinuxupd%1").arg(BetaVersion ? BetaVersion : version)); #else #error Unknown platform! #endif if (BetaVersion) { outName += "_" + BetaSignature; } QFile out(outName); if (!out.open(QIODevice::WriteOnly)) { cout << "Can't open '" << outName.toUtf8().constData() << "' for write..\n"; return -1; } out.write(compressed); out.close(); if (BetaVersion) { QString keyName(QString("tbeta_%1_key").arg(BetaVersion)); QFile key(keyName); if (!key.open(QIODevice::WriteOnly)) { cout << "Can't open '" << keyName.toUtf8().constData() << "' for write..\n"; return -1; } key.write(BetaSignature.toUtf8()); key.close(); } cout << "Update file '" << outName.toUtf8().constData() << "' written successfully!\n"; return 0; }
int main( int argc, char **argv ) { int i; uint64_t p, threads, chunk_size; uint8_t *m; struct stat s; ssize_t rd, ts = 0; size_t page_size; struct sigaction new_action, old_action; struct utimbuf u; xzcmd_max = sysconf(_SC_ARG_MAX); page_size = sysconf(_SC_PAGE_SIZE); xzcmd = malloc(xzcmd_max); snprintf(xzcmd, xzcmd_max, XZ_BINARY); parse_args(argc, argv); for (i=0; i<files; i++) { int std_in = file[i][0] == '-' && file[i][1] == '\0'; #ifdef _OPENMP threads = omp_get_max_threads(); #else threads = 1; #endif if ( (rd=strlen(file[i])) >= 3 && !strncmp(&file[i][rd-3], ".xz", 3) ) { if (opt_verbose) { error(EXIT_FAILURE, 0, "ignoring '%s', it seems to be already compressed", file[i]); } continue; } if ( !std_in ) { if ( stat(file[i], &s)) { error(EXIT_FAILURE, errno, "can't stat '%s'", file[i]); } } chunk_size = opt_context_size*(1<<(opt_complevel <= 1 ? 16 : opt_complevel + 17)); chunk_size = (chunk_size + page_size)&~(page_size-1); if ( opt_verbose ) { fprintf(stderr, "context size per thread: %ld B\n", chunk_size); } if ( opt_threads && (threads > opt_threads || opt_force) ) { threads = opt_threads; } fo = stdout; if ( std_in ) { fi = stdin; } else { if ( !(fi=fopen(file[i], "rb")) ) { error(EXIT_FAILURE, errno, "can't open '%s' for reading", file[i]); } if ( !opt_stdout ) { snprintf(str, sizeof(str), "%s.xz", file[i]); if ( !(fo=fopen(str, "wb")) ) { error(EXIT_FAILURE, errno, "error creating target archive '%s'", str); } } } if ( opt_verbose ) { if ( fo != stdout ) { fprintf(stderr, "%s -> %ld/%ld thread%c: [", file[i], threads, (s.st_size+chunk_size-1)/chunk_size, threads != 1 ? 's' : ' '); } else { fprintf(stderr, "%ld thread%c: [", threads, threads != 1 ? 's' : ' '); } fflush(stderr); } m = malloc(threads*chunk_size); new_action.sa_handler = term_handler; sigemptyset (&new_action.sa_mask); new_action.sa_flags = 0; sigaction(SIGINT, NULL, &old_action); if (old_action.sa_handler != SIG_IGN) sigaction(SIGINT, &new_action, NULL); sigaction(SIGHUP, NULL, &old_action); if (old_action.sa_handler != SIG_IGN) sigaction(SIGHUP, &new_action, NULL); sigaction(SIGTERM, NULL, &old_action); if (old_action.sa_handler != SIG_IGN) sigaction(SIGTERM, &new_action, NULL); ftemp = malloc(threads*sizeof(ftemp[0])); while ( !feof(fi) ) { size_t actrd; for (p=0; p<threads; p++) { ftemp[p] = tmpfile(); } for ( actrd=rd=0; !feof(fi) && !ferror(fi) && (uint64_t)rd < threads*chunk_size; rd += actrd) { actrd = fread(&m[rd], 1, threads*chunk_size-actrd, fi); } if (ferror(fi)) { error(EXIT_FAILURE, errno, "error in reading input"); } #pragma omp parallel for private(p) num_threads(threads) for ( p=0; p<(rd+chunk_size-1)/chunk_size; p++ ) { off_t pt, len = rd-p*chunk_size >= chunk_size ? chunk_size : rd-p*chunk_size; uint8_t *mo; lzma_stream strm = LZMA_STREAM_INIT; lzma_ret ret; mo = malloc(BUFFSIZE); if ( lzma_easy_encoder(&strm, opt_complevel, LZMA_CHECK_CRC64) != LZMA_OK ) { error(EXIT_FAILURE, errno, "unable to initialize LZMA encoder"); } for (pt=0; pt<len; pt+=BUFFSIZE) { strm.next_in = &m[p*chunk_size+pt]; strm.avail_in = len-pt >= BUFFSIZE ? BUFFSIZE : len-pt; strm.next_out = mo; strm.avail_out = BUFFSIZE; do { ret = lzma_code(&strm, LZMA_RUN); if ( ret != LZMA_OK ) { error(EXIT_FAILURE, 0, "error in LZMA_RUN"); } if ( BUFFSIZE - strm.avail_out > 0 ) { if ( !fwrite(mo, 1, BUFFSIZE - strm.avail_out, ftemp[p]) ) { error(EXIT_FAILURE, errno, "writing to temp file failed"); } strm.next_out = mo; strm.avail_out = BUFFSIZE; } } while ( strm.avail_in ); } strm.next_out = mo; strm.avail_out = BUFFSIZE; do { ret = lzma_code(&strm, LZMA_FINISH); if ( ret != LZMA_OK && ret != LZMA_STREAM_END ) { error(EXIT_FAILURE, 0, "error in LZMA_FINISH"); } if ( BUFFSIZE - strm.avail_out > 0 ) { if ( !fwrite(mo, 1, BUFFSIZE - strm.avail_out, ftemp[p]) ) { error(EXIT_FAILURE, errno, "writing to temp file failed"); } strm.next_out = mo; strm.avail_out = BUFFSIZE; } } while ( ret == LZMA_OK ); lzma_end(&strm); free(mo); if ( opt_verbose ) { fprintf(stderr, "%ld ", p); fflush(stderr); } } for ( p=0; p<threads; p++ ) { rewind(ftemp[p]); while ( (rd=fread(buf, 1, sizeof(buf), ftemp[p])) > 0 ) { if ( fwrite(buf, 1, rd, fo) != (size_t)rd ) { error(0, errno, "writing to archive failed"); if ( fo != stdout && unlink(str) ) { error(0, errno, "error deleting corrupted target archive %s", str); } exit(EXIT_FAILURE); } else ts += rd; } if (rd < 0) { error(0, errno, "reading from temporary file failed"); if ( fo != stdout && unlink(str) ) { error(0, errno, "error deleting corrupted target archive %s", str); } exit(EXIT_FAILURE); } if ( close_stream(ftemp[p]) ) { error(0, errno, "I/O error in temp file"); } } } if ( fi != stdin && close_stream(fi) ) { error(0, errno, "I/O error in input file"); } if ( opt_verbose ) { fprintf(stderr, "] "); } free(ftemp); if ( fo != stdout ) { if ( close_stream(fo) ) { error(0, errno, "I/O error in target archive"); } } else return 0; if ( chmod(str, s.st_mode) ) { error(0, errno, "warning: unable to change archive permissions"); } u.actime = s.st_atime; u.modtime = s.st_mtime; if ( utime(str, &u) ) { error(0, errno, "warning: unable to change archive timestamp"); } sigaction(SIGINT, &old_action, NULL); sigaction(SIGHUP, &old_action, NULL); sigaction(SIGTERM, &old_action, NULL); if ( opt_verbose ) { fprintf(stderr, "%ld -> %ld %3.3f%%\n", s.st_size, ts, ts*100./s.st_size); } if ( !opt_keep && unlink(file[i]) ) { error(0, errno, "error deleting input file %s", file[i]); } } return 0; }
int LzmaConnectorWriter::writeDataAndResetWriter (const unsigned char *pSrc, unsigned int uiSrcLen, unsigned char **pDest, unsigned int &uiDestLen) { int rc; unsigned int uiOldAvail_out = 0; bool bDone = false; *pDest = NULL; uiDestLen = 0; if (!pSrc || (uiSrcLen == 0)) { lzma_end (&_lzmaCompStream); resetCompStream(); if (LZMA_OK != lzma_easy_encoder (&_lzmaCompStream, getCompressionLevel(), COMPRESSION_CHECK)) { return -1; } return 0; } _lzmaCompStream.next_in = pSrc; _lzmaCompStream.avail_in = uiSrcLen; while (!bDone) { uiOldAvail_out = _lzmaCompStream.avail_out; if (0 != (rc = lzma_code ( &_lzmaCompStream, LZMA_FINISH))) { if (rc == LZMA_STREAM_END) { bDone = true; } else if (rc != LZMA_OK) { checkAndLogMsg ("LzmaConnectorWriter::flush", Logger::L_MildError, "deflate with flag Z_FINISH returned with error code %d\n", rc); uiDestLen = 0; *pDest = NULL; return -2; } } if (_lzmaCompStream.avail_out < _ulOutBufSize) { uiDestLen += (uiOldAvail_out - _lzmaCompStream.avail_out); if (_lzmaCompStream.avail_out == 0) { _ulOutBufSize *= 2; _pOutputBuffer = (unsigned char*) realloc (_pOutputBuffer, _ulOutBufSize); if (!_pOutputBuffer) { checkAndLogMsg ("LzmaConnectorWriter::flush", Logger::L_MildError, "error trying to realloc %u (previously %u) bytes\n", _ulOutBufSize, _ulOutBufSize/2); uiDestLen = 0; *pDest = NULL; return -3; } } _lzmaCompStream.avail_out = _ulOutBufSize - uiDestLen; _lzmaCompStream.next_out = _pOutputBuffer + uiDestLen; } else if (!bDone) { // deflate was not done but did not put anything into the output buffer checkAndLogMsg ("LzmaConnectorWriter::flush", Logger::L_MildError, "lzma_code() with flag LZMA_FINISH didn't produce new output but returned code is not LZMA_STREAM_END (code: %d)\n", rc); uiDestLen = 0; *pDest = NULL; return -4; } } if (uiDestLen > 0) { *pDest = _pOutputBuffer; } else { *pDest = NULL; } _lzmaCompStream.avail_out = _ulOutBufSize; _lzmaCompStream.next_out = _pOutputBuffer; _bFlushed = true; lzma_end (&_lzmaCompStream); resetCompStream(); if (LZMA_OK != lzma_easy_encoder (&_lzmaCompStream, getCompressionLevel(), COMPRESSION_CHECK)) { return -5; } return 0; }
LZMACompressor::LZMACompressor(Writer* pWriter, uint32_t preset, lzma_check check_type) : Compressor(pWriter) { bzero(&_stream, sizeof(_stream)); if (lzma_easy_encoder(&_stream, preset, check_type) != LZMA_OK) throw std::runtime_error("lzma_easy_encoder returned non-LZMA_OK"); };
std::unique_ptr<IOBuf> LZMA2Codec::doCompress(const IOBuf* data) { lzma_ret rc; lzma_stream stream = LZMA_STREAM_INIT; rc = lzma_easy_encoder(&stream, level_, LZMA_CHECK_NONE); if (rc != LZMA_OK) { throw std::runtime_error(folly::to<std::string>( "LZMA2Codec: lzma_easy_encoder error: ", rc)); } SCOPE_EXIT { lzma_end(&stream); }; uint64_t uncompressedLength = data->computeChainDataLength(); uint64_t maxCompressedLength = lzma_stream_buffer_bound(uncompressedLength); // Max 64MiB in one go constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB auto out = addOutputBuffer( &stream, (maxCompressedLength <= maxSingleStepLength ? maxCompressedLength : defaultBufferLength)); if (encodeSize()) { auto size = IOBuf::createCombined(kMaxVarintLength64); encodeVarintToIOBuf(uncompressedLength, size.get()); size->appendChain(std::move(out)); out = std::move(size); } for (auto& range : *data) { if (range.empty()) { continue; } stream.next_in = const_cast<uint8_t*>(range.data()); stream.avail_in = range.size(); while (stream.avail_in != 0) { if (stream.avail_out == 0) { out->prependChain(addOutputBuffer(&stream, defaultBufferLength)); } rc = lzma_code(&stream, LZMA_RUN); if (rc != LZMA_OK) { throw std::runtime_error(folly::to<std::string>( "LZMA2Codec: lzma_code error: ", rc)); } } } do { if (stream.avail_out == 0) { out->prependChain(addOutputBuffer(&stream, defaultBufferLength)); } rc = lzma_code(&stream, LZMA_FINISH); } while (rc == LZMA_OK); if (rc != LZMA_STREAM_END) { throw std::runtime_error(folly::to<std::string>( "LZMA2Codec: lzma_code ended with error: ", rc)); } out->prev()->trimEnd(stream.avail_out); return out; }
int compress_stream_xz(int fdf, int fdt, off_t max_bytes) { #ifdef HAVE_XZ _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT; lzma_ret ret; uint8_t buf[BUFSIZ], out[BUFSIZ]; lzma_action action = LZMA_RUN; assert(fdf >= 0); assert(fdt >= 0); ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64); if (ret != LZMA_OK) { log_error("Failed to initialize XZ encoder: code %u", ret); return -EINVAL; } for (;;) { if (s.avail_in == 0 && action == LZMA_RUN) { size_t m = sizeof(buf); ssize_t n; if (max_bytes != -1 && m > (size_t) max_bytes) m = max_bytes; n = read(fdf, buf, m); if (n < 0) return -errno; if (n == 0) action = LZMA_FINISH; else { s.next_in = buf; s.avail_in = n; if (max_bytes != -1) { assert(max_bytes >= n); max_bytes -= n; } } } if (s.avail_out == 0) { s.next_out = out; s.avail_out = sizeof(out); } ret = lzma_code(&s, action); if (ret != LZMA_OK && ret != LZMA_STREAM_END) { log_error("Compression failed: code %u", ret); return -EBADMSG; } if (s.avail_out == 0 || ret == LZMA_STREAM_END) { ssize_t n, k; n = sizeof(out) - s.avail_out; k = loop_write(fdt, out, n, false); if (k < 0) return k; if (ret == LZMA_STREAM_END) { log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)", s.total_in, s.total_out, (double) s.total_out / s.total_in * 100); return 0; } } } #else return -EPROTONOSUPPORT; #endif }