int lzma_compress_size( const unsigned char* data, unsigned size ){
//Initialize LZMA
lzma_stream strm = LZMA_STREAM_INIT;
if( lzma_easy_encoder( &strm, 9 | LZMA_PRESET_EXTREME, LZMA_CHECK_CRC64 ) != LZMA_OK )
return {};
std::vector<uint8_t> out( size*2 ); //TODO: Find value properly
//Setup in/out buffers
strm.next_in = (uint8_t*)data;
strm.avail_in = size;
strm.next_out = out.data();
strm.avail_out = out.size();
//Compress
lzma_ret ret = lzma_code( &strm, LZMA_FINISH );
if( ret != LZMA_STREAM_END ){
lzma_end( &strm ); //TODO: Use RAII
return {};
}
lzma_end( &strm );
//Return only the range actually needed
//return out.left( out.size() - strm.avail_out );
return out.size() - strm.avail_out;
}
struct ostream *o_stream_create_lzma(struct ostream *output, int level)
{
struct lzma_ostream *zstream;
lzma_ret ret;
i_assert(level >= 1 && level <= 9);
zstream = i_new(struct lzma_ostream, 1);
zstream->ostream.sendv = o_stream_lzma_sendv;
zstream->ostream.flush = o_stream_lzma_flush;
zstream->ostream.iostream.close = o_stream_lzma_close;
ret = lzma_easy_encoder(&zstream->strm, level, LZMA_CHECK_CRC64);
switch (ret) {
case LZMA_OK:
break;
case LZMA_MEM_ERROR:
i_fatal_status(FATAL_OUTOFMEM, "lzma: Out of memory");
case LZMA_OPTIONS_ERROR:
i_fatal("lzma: Invalid level");
default:
i_fatal("lzma_easy_encoder() failed with %d", ret);
}
zstream->strm.next_out = zstream->outbuf;
zstream->strm.avail_out = sizeof(zstream->outbuf);
return o_stream_create(&zstream->ostream, output,
o_stream_get_fd(output));
}
static int xz_open(struct zstream *z, int fd, char *mode)
{
lzma_ret ret;
z->xz.eof = 0;
z->xz.fd = fd;
if(*mode == 'w') {
ret = lzma_easy_encoder(&z->xz.stream, 5, LZMA_CHECK_CRC64);
if (ret == LZMA_OK)
return 0;
return -1;
}
if(*mode == 'r') {
z->xz.stream.next_in = NULL;
z->xz.stream.avail_in = 0;
z->xz.stream.next_out = z->xz.outbuf;
z->xz.stream.avail_out = z->xz.bufsize;
ret = lzma_stream_decoder(&z->xz.stream, UINT64_MAX, LZMA_CONCATENATED);
if (ret == LZMA_OK)
return 0;
return -1;
}
return -1;
}
/*@null@*/
static XZFILE *xzopen_internal(const char *path, const char *mode, int fdno, int xz)
/*@globals fileSystem @*/
/*@modifies fileSystem @*/
{
int level = LZMA_PRESET_DEFAULT;
int encoding = 0;
FILE *fp;
XZFILE *xzfile;
lzma_stream tmp;
lzma_ret ret;
for (; *mode != '\0'; mode++) {
if (*mode == 'w')
encoding = 1;
else if (*mode == 'r')
encoding = 0;
else if (*mode >= '0' && *mode <= '9')
level = (int)(*mode - '0');
}
if (fdno != -1)
fp = fdopen(fdno, encoding ? "w" : "r");
else
fp = fopen(path, encoding ? "w" : "r");
if (!fp)
return NULL;
xzfile = calloc(1, sizeof(*xzfile));
if (!xzfile) {
(void) fclose(fp);
return NULL;
}
xzfile->fp = fp;
xzfile->encoding = encoding;
xzfile->eof = 0;
tmp = (lzma_stream)LZMA_STREAM_INIT;
xzfile->strm = tmp;
if (encoding) {
if (xz) {
ret = lzma_easy_encoder(&xzfile->strm, level, LZMA_CHECK_CRC32);
} else {
lzma_options_lzma options;
(void) lzma_lzma_preset(&options, level);
ret = lzma_alone_encoder(&xzfile->strm, &options);
}
} else {
/* We set the memlimit for decompression to 100MiB which should be
* more than enough to be sufficient for level 9 which requires 65 MiB.
*/
ret = lzma_auto_decoder(&xzfile->strm, 100<<20, 0);
}
if (ret != LZMA_OK) {
(void) fclose(fp);
memset(xzfile, 0, sizeof(*xzfile));
free(xzfile);
return NULL;
}
return xzfile;
}
struct lzma_filter *lzma_encoder(void)
{
lzma_stream strm = LZMA_STREAM_INIT;
uint32_t preset = 6; // TODO select maybe? see 'man xz' for -0..-9
if (lzma_easy_encoder(&strm, preset, LZMA_CHECK_CRC64) != LZMA_OK)
return NULL;
return _lzma_filter_new(&strm);
}
void lzma_base::do_init( const lzma_params &p, bool compress, lzma::alloc_func alloc, lzma::free_func free, void *derived )
{
lzma_stream *s = static_cast<lzma_stream *>( stream_ );
memset( s, 0, sizeof( *s ) );
lzma_error::check(
compress ?
lzma_easy_encoder( s, p.level, LZMA_CHECK_CRC32 ) :
lzma_stream_decoder( s, 100 * 1024 * 1024, 0 )
);
}
static LZFILE *lzopen(const char *path, const char *mode, int fd, int isxz)
{
int level = 7;
int encoding = 0;
FILE *fp;
LZFILE *lzfile;
lzma_ret ret;
if (!path && fd < 0)
return 0;
for (; *mode; mode++)
{
if (*mode == 'w')
encoding = 1;
else if (*mode == 'r')
encoding = 0;
else if (*mode >= '1' && *mode <= '9')
level = *mode - '0';
}
if (fd != -1)
fp = fdopen(fd, encoding ? "w" : "r");
else
fp = fopen(path, encoding ? "w" : "r");
if (!fp)
return 0;
lzfile = calloc(1, sizeof(*lzfile));
if (!lzfile)
{
fclose(fp);
return 0;
}
lzfile->file = fp;
lzfile->encoding = encoding;
lzfile->eof = 0;
lzfile->strm = stream_init;
if (encoding)
{
if (isxz)
ret = lzma_easy_encoder(&lzfile->strm, level, LZMA_CHECK_SHA256);
else
ret = setup_alone_encoder(&lzfile->strm, level);
}
else
ret = lzma_auto_decoder(&lzfile->strm, 100 << 20, 0);
if (ret != LZMA_OK)
{
fclose(fp);
free(lzfile);
return 0;
}
return lzfile;
}
static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int xz)
{
int level = 7; /* Use XZ's default compression level if unspecified */
int encoding = 0;
FILE *fp;
LZFILE *lzfile;
lzma_ret ret;
lzma_stream init_strm = LZMA_STREAM_INIT;
for (; *mode; mode++) {
if (*mode == 'w')
encoding = 1;
else if (*mode == 'r')
encoding = 0;
else if (*mode >= '1' && *mode <= '9')
level = *mode - '0';
}
if (fd != -1)
fp = fdopen(fd, encoding ? "w" : "r");
else
fp = fopen(path, encoding ? "w" : "r");
if (!fp)
return 0;
lzfile = calloc(1, sizeof(*lzfile));
if (!lzfile) {
fclose(fp);
return 0;
}
lzfile->file = fp;
lzfile->encoding = encoding;
lzfile->eof = 0;
lzfile->strm = init_strm;
if (encoding) {
if (xz) {
ret = lzma_easy_encoder(&lzfile->strm, level, LZMA_CHECK_SHA256);
} else {
lzma_options_lzma options;
lzma_lzma_preset(&options, level);
ret = lzma_alone_encoder(&lzfile->strm, &options);
}
} else { /* lzma_easy_decoder_memusage(level) is not ready yet, use hardcoded limit for now */
ret = lzma_auto_decoder(&lzfile->strm, 100<<20, 0);
}
if (ret != LZMA_OK) {
fclose(fp);
free(lzfile);
return 0;
}
return lzfile;
}
static void
filter_xz_init(struct io_lzma *io, lzma_stream *s)
{
uint32_t preset;
lzma_ret ret;
io->status |= DPKG_STREAM_COMPRESS;
preset = io->params->level;
if (io->params->strategy == COMPRESSOR_STRATEGY_EXTREME)
preset |= LZMA_PRESET_EXTREME;
ret = lzma_easy_encoder(s, preset, LZMA_CHECK_CRC32);
if (ret != LZMA_OK)
filter_lzma_error(io, ret);
}
LzmaConnectorWriter::LzmaConnectorWriter (const CompressionSetting * const pCompressionSetting, unsigned long ulOutBufSize) : ConnectorWriter (pCompressionSetting)
{
if (ulOutBufSize == 0) {
// Throw C++ exception here
}
_ulOutBufSize = ulOutBufSize;
if ((_pOutputBuffer = new unsigned char [_ulOutBufSize]) == NULL) {
// throw C++ exception here
}
resetCompStream();
if (LZMA_OK != lzma_easy_encoder (&_lzmaCompStream, getCompressionLevel(), COMPRESSION_CHECK)) {
// Throw C++ exception here
}
}
static bool
init_encoder(lzma_stream *strm, uint32_t preset)
{
// Initialize the encoder using a preset. Set the integrity to check
// to CRC64, which is the default in the xz command line tool. If
// the .xz file needs to be decompressed with XZ Embedded, use
// LZMA_CHECK_CRC32 instead.
lzma_ret ret = lzma_easy_encoder(strm, preset, LZMA_CHECK_CRC64);
// Return successfully if the initialization went fine.
if (ret == LZMA_OK)
return true;
// Something went wrong. The possible errors are documented in
// lzma/container.h (src/liblzma/api/lzma/container.h in the source
// package or e.g. /usr/include/lzma/container.h depending on the
// install prefix).
const char *msg;
switch (ret) {
case LZMA_MEM_ERROR:
msg = "Memory allocation failed";
break;
case LZMA_OPTIONS_ERROR:
msg = "Specified preset is not supported";
break;
case LZMA_UNSUPPORTED_CHECK:
msg = "Specified integrity check is not supported";
break;
default:
// This is most likely LZMA_PROG_ERROR indicating a bug in
// this program or in liblzma. It is inconvenient to have a
// separate error message for errors that should be impossible
// to occur, but knowing the error code is important for
// debugging. That's why it is good to print the error code
// at least when there is no good error message to show.
msg = "Unknown error, possibly a bug";
break;
}
fprintf(stderr, "Error initializing the encoder: %s (error code %u)\n",
msg, ret);
return false;
}
int import_compress_init(ImportCompress *c, ImportCompressType t) {
int r;
assert(c);
switch (t) {
case IMPORT_COMPRESS_XZ: {
lzma_ret xzr;
xzr = lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
if (xzr != LZMA_OK)
return -EIO;
c->type = IMPORT_COMPRESS_XZ;
break;
}
case IMPORT_COMPRESS_GZIP:
r = deflateInit2(&c->gzip, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY);
if (r != Z_OK)
return -EIO;
c->type = IMPORT_COMPRESS_GZIP;
break;
case IMPORT_COMPRESS_BZIP2:
r = BZ2_bzCompressInit(&c->bzip2, 9, 0, 0);
if (r != BZ_OK)
return -EIO;
c->type = IMPORT_COMPRESS_BZIP2;
break;
case IMPORT_COMPRESS_UNCOMPRESSED:
c->type = IMPORT_COMPRESS_UNCOMPRESSED;
break;
default:
return -EOPNOTSUPP;
}
c->encoding = true;
return 0;
}
static void
filter_xz_init(struct io_lzma *io, lzma_stream *s)
{
uint32_t preset;
lzma_check check = LZMA_CHECK_CRC64;
#ifdef HAVE_LZMA_MT
lzma_mt mt_options = {
.flags = 0,
.threads = sysconf(_SC_NPROCESSORS_ONLN),
.block_size = 0,
.timeout = 0,
.filters = NULL,
.check = check,
};
#endif
lzma_ret ret;
io->status |= DPKG_STREAM_COMPRESS;
preset = io->params->level;
if (io->params->strategy == COMPRESSOR_STRATEGY_EXTREME)
preset |= LZMA_PRESET_EXTREME;
#ifdef HAVE_LZMA_MT
mt_options.preset = preset;
ret = lzma_stream_encoder_mt(s, &mt_options);
#else
ret = lzma_easy_encoder(s, preset, check);
#endif
if (ret != LZMA_OK)
filter_lzma_error(io, ret);
}
static int
filter_lzma_code(struct io_lzma *io, lzma_stream *s)
{
lzma_ret ret;
ret = lzma_code(s, io->action);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
filter_lzma_error(io, ret);
return ret;
}
static void write_xz(int f, int fd, const char *arg)
{
uint8_t inbuf[BUFFER_SIZE], outbuf[BUFFER_SIZE];
lzma_stream xz = LZMA_STREAM_INIT;
if (lzma_easy_encoder(&xz, 6, LZMA_CHECK_CRC64) != LZMA_OK)
goto xz_write_end;
xz.avail_in = 0;
while (xz.avail_in
|| (xz.avail_in = read(fd, (uint8_t*)(xz.next_in = inbuf), BUFFER_SIZE)) > 0)
{
xz.next_out = outbuf;
xz.avail_out = sizeof(outbuf);
if (lzma_code(&xz, LZMA_RUN) != LZMA_OK)
goto xz_write_lzma_end;
if (write(f, outbuf, xz.next_out - outbuf) != xz.next_out - outbuf)
goto xz_write_lzma_end;
}
// Flush the stream
lzma_ret ret;
do
{
xz.next_out = outbuf;
xz.avail_out = sizeof(outbuf);
ret = lzma_code(&xz, LZMA_FINISH);
if (write(f, outbuf, xz.next_out - outbuf) != xz.next_out - outbuf)
goto xz_write_lzma_end;
}
while (ret == LZMA_OK);
xz_write_lzma_end:
lzma_end(&xz);
xz_write_end:
close(f);
close(fd);
}
static int
Compressor_init_xz(lzma_stream *lzs, int check, uint32_t preset,
PyObject *filterspecs)
{
lzma_ret lzret;
if (filterspecs == Py_None) {
lzret = lzma_easy_encoder(lzs, preset, check);
} else {
lzma_filter filters[LZMA_FILTERS_MAX + 1];
if (parse_filter_chain_spec(filters, filterspecs) == -1)
return -1;
lzret = lzma_stream_encoder(lzs, filters, check);
free_filter_chain(filters);
}
if (catch_lzma_error(lzret))
return -1;
else
return 0;
}
int xz_enc_init(lzma_stream *str)
{
lzma_ret ret_xz;
ret_xz = lzma_easy_encoder(str, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
return ret_xz;
}
int main(int argc, char *argv[])
{
QString workDir;
#ifdef Q_OS_MAC
if (QDir(QString()).absolutePath() == "/") {
QString first = argc ? QString::fromLocal8Bit(argv[0]) : QString();
if (!first.isEmpty()) {
QFileInfo info(first);
if (info.exists()) {
QDir result(info.absolutePath() + "/../../..");
workDir = result.absolutePath() + '/';
}
}
}
#endif
QString remove;
int version = 0;
QFileInfoList files;
for (int i = 0; i < argc; ++i) {
if (string("-path") == argv[i] && i + 1 < argc) {
QString path = workDir + QString(argv[i + 1]);
QFileInfo info(path);
files.push_back(info);
if (remove.isEmpty()) remove = info.canonicalPath() + "/";
} else if (string("-version") == argv[i] && i + 1 < argc) {
version = QString(argv[i + 1]).toInt();
} else if (string("-dev") == argv[i]) {
DevChannel = true;
} else if (string("-beta") == argv[i] && i + 1 < argc) {
BetaVersion = QString(argv[i + 1]).toULongLong();
if (BetaVersion > version * 1000ULL && BetaVersion < (version + 1) * 1000ULL) {
DevChannel = false;
BetaSignature = countBetaVersionSignature(BetaVersion);
if (BetaSignature.isEmpty()) {
return -1;
}
} else {
cout << "Bad -beta param value passed, should be for the same version: " << version << ", beta: " << BetaVersion << "\n";
return -1;
}
}
}
if (files.isEmpty() || remove.isEmpty() || version <= 1016 || version > 999999999) {
#ifdef Q_OS_WIN
cout << "Usage: Packer.exe -path {file} -version {version} OR Packer.exe -path {dir} -version {version}\n";
#elif defined Q_OS_MAC
cout << "Usage: Packer.app -path {file} -version {version} OR Packer.app -path {dir} -version {version}\n";
#else
cout << "Usage: Packer -path {file} -version {version} OR Packer -path {dir} -version {version}\n";
#endif
return -1;
}
bool hasDirs = true;
while (hasDirs) {
hasDirs = false;
for (QFileInfoList::iterator i = files.begin(); i != files.end(); ++i) {
QFileInfo info(*i);
QString fullPath = info.canonicalFilePath();
if (info.isDir()) {
hasDirs = true;
files.erase(i);
QDir d = QDir(info.absoluteFilePath());
QString fullDir = d.canonicalPath();
QStringList entries = d.entryList(QDir::Files | QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot);
files.append(d.entryInfoList(QDir::Files | QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot));
break;
} else if (!info.isReadable()) {
cout << "Can't read: " << info.absoluteFilePath().toUtf8().constData() << "\n";
return -1;
} else if (info.isHidden()) {
hasDirs = true;
files.erase(i);
break;
}
}
}
for (QFileInfoList::iterator i = files.begin(); i != files.end(); ++i) {
QFileInfo info(*i);
if (!info.canonicalFilePath().startsWith(remove)) {
cout << "Can't find '" << remove.toUtf8().constData() << "' in file '" << info.canonicalFilePath().toUtf8().constData() << "' :(\n";
return -1;
}
}
QByteArray result;
{
QBuffer buffer(&result);
buffer.open(QIODevice::WriteOnly);
QDataStream stream(&buffer);
stream.setVersion(QDataStream::Qt_5_1);
if (BetaVersion) {
stream << quint32(0x7FFFFFFF);
stream << quint64(BetaVersion);
} else {
stream << quint32(version);
}
stream << quint32(files.size());
cout << "Found " << files.size() << " file" << (files.size() == 1 ? "" : "s") << "..\n";
for (QFileInfoList::iterator i = files.begin(); i != files.end(); ++i) {
QFileInfo info(*i);
QString fullName = info.canonicalFilePath();
QString name = fullName.mid(remove.length());
cout << name.toUtf8().constData() << " (" << info.size() << ")\n";
QFile f(fullName);
if (!f.open(QIODevice::ReadOnly)) {
cout << "Can't open '" << fullName.toUtf8().constData() << "' for read..\n";
return -1;
}
QByteArray inner = f.readAll();
stream << name << quint32(inner.size()) << inner;
#if defined Q_OS_MAC || defined Q_OS_LINUX
stream << (QFileInfo(fullName).isExecutable() ? true : false);
#endif
}
if (stream.status() != QDataStream::Ok) {
cout << "Stream status is bad: " << stream.status() << "\n";
return -1;
}
}
int32 resultSize = result.size();
cout << "Compression start, size: " << resultSize << "\n";
QByteArray compressed, resultCheck;
#ifdef Q_OS_WIN // use Lzma SDK for win
const int32 hSigLen = 128, hShaLen = 20, hPropsLen = LZMA_PROPS_SIZE, hOriginalSizeLen = sizeof(int32), hSize = hSigLen + hShaLen + hPropsLen + hOriginalSizeLen; // header
compressed.resize(hSize + resultSize + 1024 * 1024); // rsa signature + sha1 + lzma props + max compressed size
size_t compressedLen = compressed.size() - hSize;
size_t outPropsSize = LZMA_PROPS_SIZE;
uchar *_dest = (uchar*)(compressed.data() + hSize);
size_t *_destLen = &compressedLen;
const uchar *_src = (const uchar*)(result.constData());
size_t _srcLen = result.size();
uchar *_outProps = (uchar*)(compressed.data() + hSigLen + hShaLen);
int res = LzmaCompress(_dest, _destLen, _src, _srcLen, _outProps, &outPropsSize, 9, 64 * 1024 * 1024, 4, 0, 2, 273, 2);
if (res != SZ_OK) {
cout << "Error in compression: " << res << "\n";
return -1;
}
compressed.resize(int(hSize + compressedLen));
memcpy(compressed.data() + hSigLen + hShaLen + hPropsLen, &resultSize, hOriginalSizeLen);
cout << "Compressed to size: " << compressedLen << "\n";
cout << "Checking uncompressed..\n";
int32 resultCheckLen;
memcpy(&resultCheckLen, compressed.constData() + hSigLen + hShaLen + hPropsLen, hOriginalSizeLen);
if (resultCheckLen <= 0 || resultCheckLen > 1024 * 1024 * 1024) {
cout << "Bad result len: " << resultCheckLen << "\n";
return -1;
}
resultCheck.resize(resultCheckLen);
size_t resultLen = resultCheck.size();
SizeT srcLen = compressedLen;
int uncompressRes = LzmaUncompress((uchar*)resultCheck.data(), &resultLen, (const uchar*)(compressed.constData() + hSize), &srcLen, (const uchar*)(compressed.constData() + hSigLen + hShaLen), LZMA_PROPS_SIZE);
if (uncompressRes != SZ_OK) {
cout << "Uncompress failed: " << uncompressRes << "\n";
return -1;
}
if (resultLen != size_t(result.size())) {
cout << "Uncompress bad size: " << resultLen << ", was: " << result.size() << "\n";
return -1;
}
#else // use liblzma for others
const int32 hSigLen = 128, hShaLen = 20, hPropsLen = 0, hOriginalSizeLen = sizeof(int32), hSize = hSigLen + hShaLen + hOriginalSizeLen; // header
compressed.resize(hSize + resultSize + 1024 * 1024); // rsa signature + sha1 + lzma props + max compressed size
size_t compressedLen = compressed.size() - hSize;
lzma_stream stream = LZMA_STREAM_INIT;
int preset = 9 | LZMA_PRESET_EXTREME;
lzma_ret ret = lzma_easy_encoder(&stream, preset, LZMA_CHECK_CRC64);
if (ret != LZMA_OK) {
const char *msg;
switch (ret) {
case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break;
case LZMA_OPTIONS_ERROR: msg = "Specified preset is not supported"; break;
case LZMA_UNSUPPORTED_CHECK: msg = "Specified integrity check is not supported"; break;
default: msg = "Unknown error, possibly a bug"; break;
}
cout << "Error initializing the encoder: " << msg << " (error code " << ret << ")\n";
return -1;
}
stream.avail_in = resultSize;
stream.next_in = (uint8_t*)result.constData();
stream.avail_out = compressedLen;
stream.next_out = (uint8_t*)(compressed.data() + hSize);
lzma_ret res = lzma_code(&stream, LZMA_FINISH);
compressedLen -= stream.avail_out;
lzma_end(&stream);
if (res != LZMA_OK && res != LZMA_STREAM_END) {
const char *msg;
switch (res) {
case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break;
case LZMA_DATA_ERROR: msg = "File size limits exceeded"; break;
default: msg = "Unknown error, possibly a bug"; break;
}
cout << "Error in compression: " << msg << " (error code " << res << ")\n";
return -1;
}
compressed.resize(int(hSize + compressedLen));
memcpy(compressed.data() + hSigLen + hShaLen, &resultSize, hOriginalSizeLen);
cout << "Compressed to size: " << compressedLen << "\n";
cout << "Checking uncompressed..\n";
int32 resultCheckLen;
memcpy(&resultCheckLen, compressed.constData() + hSigLen + hShaLen, hOriginalSizeLen);
if (resultCheckLen <= 0 || resultCheckLen > 1024 * 1024 * 1024) {
cout << "Bad result len: " << resultCheckLen << "\n";
return -1;
}
resultCheck.resize(resultCheckLen);
size_t resultLen = resultCheck.size();
stream = LZMA_STREAM_INIT;
ret = lzma_stream_decoder(&stream, UINT64_MAX, LZMA_CONCATENATED);
if (ret != LZMA_OK) {
const char *msg;
switch (ret) {
case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break;
case LZMA_OPTIONS_ERROR: msg = "Specified preset is not supported"; break;
case LZMA_UNSUPPORTED_CHECK: msg = "Specified integrity check is not supported"; break;
default: msg = "Unknown error, possibly a bug"; break;
}
cout << "Error initializing the decoder: " << msg << " (error code " << ret << ")\n";
return -1;
}
stream.avail_in = compressedLen;
stream.next_in = (uint8_t*)(compressed.constData() + hSize);
stream.avail_out = resultLen;
stream.next_out = (uint8_t*)resultCheck.data();
res = lzma_code(&stream, LZMA_FINISH);
if (stream.avail_in) {
cout << "Error in decompression, " << stream.avail_in << " bytes left in _in of " << compressedLen << " whole.\n";
return -1;
} else if (stream.avail_out) {
cout << "Error in decompression, " << stream.avail_out << " bytes free left in _out of " << resultLen << " whole.\n";
return -1;
}
lzma_end(&stream);
if (res != LZMA_OK && res != LZMA_STREAM_END) {
const char *msg;
switch (res) {
case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break;
case LZMA_FORMAT_ERROR: msg = "The input data is not in the .xz format"; break;
case LZMA_OPTIONS_ERROR: msg = "Unsupported compression options"; break;
case LZMA_DATA_ERROR: msg = "Compressed file is corrupt"; break;
case LZMA_BUF_ERROR: msg = "Compressed data is truncated or otherwise corrupt"; break;
default: msg = "Unknown error, possibly a bug"; break;
}
cout << "Error in decompression: " << msg << " (error code " << res << ")\n";
return -1;
}
#endif
if (memcmp(result.constData(), resultCheck.constData(), resultLen)) {
cout << "Data differ :(\n";
return -1;
}
/**/
result = resultCheck = QByteArray();
cout << "Counting SHA1 hash..\n";
uchar sha1Buffer[20];
memcpy(compressed.data() + hSigLen, hashSha1(compressed.constData() + hSigLen + hShaLen, uint32(compressedLen + hPropsLen + hOriginalSizeLen), sha1Buffer), hShaLen); // count sha1
uint32 siglen = 0;
cout << "Signing..\n";
RSA *prKey = PEM_read_bio_RSAPrivateKey(BIO_new_mem_buf(const_cast<char*>((DevChannel || BetaVersion) ? PrivateDevKey : PrivateKey), -1), 0, 0, 0);
if (!prKey) {
cout << "Could not read RSA private key!\n";
return -1;
}
if (RSA_size(prKey) != hSigLen) {
cout << "Bad private key, size: " << RSA_size(prKey) << "\n";
RSA_free(prKey);
return -1;
}
if (RSA_sign(NID_sha1, (const uchar*)(compressed.constData() + hSigLen), hShaLen, (uchar*)(compressed.data()), &siglen, prKey) != 1) { // count signature
cout << "Signing failed!\n";
RSA_free(prKey);
return -1;
}
RSA_free(prKey);
if (siglen != hSigLen) {
cout << "Bad signature length: " << siglen << "\n";
return -1;
}
cout << "Checking signature..\n";
RSA *pbKey = PEM_read_bio_RSAPublicKey(BIO_new_mem_buf(const_cast<char*>((DevChannel || BetaVersion) ? PublicDevKey : PublicKey), -1), 0, 0, 0);
if (!pbKey) {
cout << "Could not read RSA public key!\n";
return -1;
}
if (RSA_verify(NID_sha1, (const uchar*)(compressed.constData() + hSigLen), hShaLen, (const uchar*)(compressed.constData()), siglen, pbKey) != 1) { // verify signature
RSA_free(pbKey);
cout << "Signature verification failed!\n";
return -1;
}
cout << "Signature verified!\n";
RSA_free(pbKey);
#ifdef Q_OS_WIN
QString outName(QString("tupdate%1").arg(BetaVersion ? BetaVersion : version));
#elif defined Q_OS_MAC
QString outName(QString("tmacupd%1").arg(BetaVersion ? BetaVersion : version));
#elif defined Q_OS_LINUX32
QString outName(QString("tlinux32upd%1").arg(BetaVersion ? BetaVersion : version));
#elif defined Q_OS_LINUX64
QString outName(QString("tlinuxupd%1").arg(BetaVersion ? BetaVersion : version));
#else
#error Unknown platform!
#endif
if (BetaVersion) {
outName += "_" + BetaSignature;
}
QFile out(outName);
if (!out.open(QIODevice::WriteOnly)) {
cout << "Can't open '" << outName.toUtf8().constData() << "' for write..\n";
return -1;
}
out.write(compressed);
out.close();
if (BetaVersion) {
QString keyName(QString("tbeta_%1_key").arg(BetaVersion));
QFile key(keyName);
if (!key.open(QIODevice::WriteOnly)) {
cout << "Can't open '" << keyName.toUtf8().constData() << "' for write..\n";
return -1;
}
key.write(BetaSignature.toUtf8());
key.close();
}
cout << "Update file '" << outName.toUtf8().constData() << "' written successfully!\n";
return 0;
}
int main( int argc, char **argv ) {
int i;
uint64_t p, threads, chunk_size;
uint8_t *m;
struct stat s;
ssize_t rd, ts = 0;
size_t page_size;
struct sigaction new_action, old_action;
struct utimbuf u;
xzcmd_max = sysconf(_SC_ARG_MAX);
page_size = sysconf(_SC_PAGE_SIZE);
xzcmd = malloc(xzcmd_max);
snprintf(xzcmd, xzcmd_max, XZ_BINARY);
parse_args(argc, argv);
for (i=0; i<files; i++) {
int std_in = file[i][0] == '-' && file[i][1] == '\0';
#ifdef _OPENMP
threads = omp_get_max_threads();
#else
threads = 1;
#endif
if ( (rd=strlen(file[i])) >= 3 && !strncmp(&file[i][rd-3], ".xz", 3) ) {
if (opt_verbose) {
error(EXIT_FAILURE, 0, "ignoring '%s', it seems to be already compressed", file[i]);
}
continue;
}
if ( !std_in ) {
if ( stat(file[i], &s)) {
error(EXIT_FAILURE, errno, "can't stat '%s'", file[i]);
}
}
chunk_size = opt_context_size*(1<<(opt_complevel <= 1 ? 16 : opt_complevel + 17));
chunk_size = (chunk_size + page_size)&~(page_size-1);
if ( opt_verbose ) {
fprintf(stderr, "context size per thread: %ld B\n", chunk_size);
}
if ( opt_threads && (threads > opt_threads || opt_force) ) {
threads = opt_threads;
}
fo = stdout;
if ( std_in ) {
fi = stdin;
} else {
if ( !(fi=fopen(file[i], "rb")) ) {
error(EXIT_FAILURE, errno, "can't open '%s' for reading", file[i]);
}
if ( !opt_stdout ) {
snprintf(str, sizeof(str), "%s.xz", file[i]);
if ( !(fo=fopen(str, "wb")) ) {
error(EXIT_FAILURE, errno, "error creating target archive '%s'", str);
}
}
}
if ( opt_verbose ) {
if ( fo != stdout ) {
fprintf(stderr, "%s -> %ld/%ld thread%c: [", file[i], threads, (s.st_size+chunk_size-1)/chunk_size, threads != 1 ? 's' : ' ');
} else {
fprintf(stderr, "%ld thread%c: [", threads, threads != 1 ? 's' : ' ');
}
fflush(stderr);
}
m = malloc(threads*chunk_size);
new_action.sa_handler = term_handler;
sigemptyset (&new_action.sa_mask);
new_action.sa_flags = 0;
sigaction(SIGINT, NULL, &old_action);
if (old_action.sa_handler != SIG_IGN) sigaction(SIGINT, &new_action, NULL);
sigaction(SIGHUP, NULL, &old_action);
if (old_action.sa_handler != SIG_IGN) sigaction(SIGHUP, &new_action, NULL);
sigaction(SIGTERM, NULL, &old_action);
if (old_action.sa_handler != SIG_IGN) sigaction(SIGTERM, &new_action, NULL);
ftemp = malloc(threads*sizeof(ftemp[0]));
while ( !feof(fi) ) {
size_t actrd;
for (p=0; p<threads; p++) {
ftemp[p] = tmpfile();
}
for ( actrd=rd=0; !feof(fi) && !ferror(fi) && (uint64_t)rd < threads*chunk_size; rd += actrd) {
actrd = fread(&m[rd], 1, threads*chunk_size-actrd, fi);
}
if (ferror(fi)) {
error(EXIT_FAILURE, errno, "error in reading input");
}
#pragma omp parallel for private(p) num_threads(threads)
for ( p=0; p<(rd+chunk_size-1)/chunk_size; p++ ) {
off_t pt, len = rd-p*chunk_size >= chunk_size ? chunk_size : rd-p*chunk_size;
uint8_t *mo;
lzma_stream strm = LZMA_STREAM_INIT;
lzma_ret ret;
mo = malloc(BUFFSIZE);
if ( lzma_easy_encoder(&strm, opt_complevel, LZMA_CHECK_CRC64) != LZMA_OK ) {
error(EXIT_FAILURE, errno, "unable to initialize LZMA encoder");
}
for (pt=0; pt<len; pt+=BUFFSIZE) {
strm.next_in = &m[p*chunk_size+pt];
strm.avail_in = len-pt >= BUFFSIZE ? BUFFSIZE : len-pt;
strm.next_out = mo;
strm.avail_out = BUFFSIZE;
do {
ret = lzma_code(&strm, LZMA_RUN);
if ( ret != LZMA_OK ) {
error(EXIT_FAILURE, 0, "error in LZMA_RUN");
}
if ( BUFFSIZE - strm.avail_out > 0 ) {
if ( !fwrite(mo, 1, BUFFSIZE - strm.avail_out, ftemp[p]) ) {
error(EXIT_FAILURE, errno, "writing to temp file failed");
}
strm.next_out = mo;
strm.avail_out = BUFFSIZE;
}
} while ( strm.avail_in );
}
strm.next_out = mo;
strm.avail_out = BUFFSIZE;
do {
ret = lzma_code(&strm, LZMA_FINISH);
if ( ret != LZMA_OK && ret != LZMA_STREAM_END ) {
error(EXIT_FAILURE, 0, "error in LZMA_FINISH");
}
if ( BUFFSIZE - strm.avail_out > 0 ) {
if ( !fwrite(mo, 1, BUFFSIZE - strm.avail_out, ftemp[p]) ) {
error(EXIT_FAILURE, errno, "writing to temp file failed");
}
strm.next_out = mo;
strm.avail_out = BUFFSIZE;
}
} while ( ret == LZMA_OK );
lzma_end(&strm);
free(mo);
if ( opt_verbose ) {
fprintf(stderr, "%ld ", p);
fflush(stderr);
}
}
for ( p=0; p<threads; p++ ) {
rewind(ftemp[p]);
while ( (rd=fread(buf, 1, sizeof(buf), ftemp[p])) > 0 ) {
if ( fwrite(buf, 1, rd, fo) != (size_t)rd ) {
error(0, errno, "writing to archive failed");
if ( fo != stdout && unlink(str) ) {
error(0, errno, "error deleting corrupted target archive %s", str);
}
exit(EXIT_FAILURE);
} else ts += rd;
}
if (rd < 0) {
error(0, errno, "reading from temporary file failed");
if ( fo != stdout && unlink(str) ) {
error(0, errno, "error deleting corrupted target archive %s", str);
}
exit(EXIT_FAILURE);
}
if ( close_stream(ftemp[p]) ) {
error(0, errno, "I/O error in temp file");
}
}
}
if ( fi != stdin && close_stream(fi) ) {
error(0, errno, "I/O error in input file");
}
if ( opt_verbose ) {
fprintf(stderr, "] ");
}
free(ftemp);
if ( fo != stdout ) {
if ( close_stream(fo) ) {
error(0, errno, "I/O error in target archive");
}
} else return 0;
if ( chmod(str, s.st_mode) ) {
error(0, errno, "warning: unable to change archive permissions");
}
u.actime = s.st_atime;
u.modtime = s.st_mtime;
if ( utime(str, &u) ) {
error(0, errno, "warning: unable to change archive timestamp");
}
sigaction(SIGINT, &old_action, NULL);
sigaction(SIGHUP, &old_action, NULL);
sigaction(SIGTERM, &old_action, NULL);
if ( opt_verbose ) {
fprintf(stderr, "%ld -> %ld %3.3f%%\n", s.st_size, ts, ts*100./s.st_size);
}
if ( !opt_keep && unlink(file[i]) ) {
error(0, errno, "error deleting input file %s", file[i]);
}
}
return 0;
}
int LzmaConnectorWriter::writeDataAndResetWriter (const unsigned char *pSrc, unsigned int uiSrcLen, unsigned char **pDest, unsigned int &uiDestLen)
{
int rc;
unsigned int uiOldAvail_out = 0;
bool bDone = false;
*pDest = NULL;
uiDestLen = 0;
if (!pSrc || (uiSrcLen == 0)) {
lzma_end (&_lzmaCompStream);
resetCompStream();
if (LZMA_OK != lzma_easy_encoder (&_lzmaCompStream, getCompressionLevel(), COMPRESSION_CHECK)) {
return -1;
}
return 0;
}
_lzmaCompStream.next_in = pSrc;
_lzmaCompStream.avail_in = uiSrcLen;
while (!bDone) {
uiOldAvail_out = _lzmaCompStream.avail_out;
if (0 != (rc = lzma_code ( &_lzmaCompStream, LZMA_FINISH))) {
if (rc == LZMA_STREAM_END) {
bDone = true;
}
else if (rc != LZMA_OK) {
checkAndLogMsg ("LzmaConnectorWriter::flush", Logger::L_MildError,
"deflate with flag Z_FINISH returned with error code %d\n", rc);
uiDestLen = 0;
*pDest = NULL;
return -2;
}
}
if (_lzmaCompStream.avail_out < _ulOutBufSize) {
uiDestLen += (uiOldAvail_out - _lzmaCompStream.avail_out);
if (_lzmaCompStream.avail_out == 0) {
_ulOutBufSize *= 2;
_pOutputBuffer = (unsigned char*) realloc (_pOutputBuffer, _ulOutBufSize);
if (!_pOutputBuffer) {
checkAndLogMsg ("LzmaConnectorWriter::flush", Logger::L_MildError,
"error trying to realloc %u (previously %u) bytes\n",
_ulOutBufSize, _ulOutBufSize/2);
uiDestLen = 0;
*pDest = NULL;
return -3;
}
}
_lzmaCompStream.avail_out = _ulOutBufSize - uiDestLen;
_lzmaCompStream.next_out = _pOutputBuffer + uiDestLen;
}
else if (!bDone) {
// deflate was not done but did not put anything into the output buffer
checkAndLogMsg ("LzmaConnectorWriter::flush", Logger::L_MildError,
"lzma_code() with flag LZMA_FINISH didn't produce new output but returned code is not LZMA_STREAM_END (code: %d)\n", rc);
uiDestLen = 0;
*pDest = NULL;
return -4;
}
}
if (uiDestLen > 0) {
*pDest = _pOutputBuffer;
}
else {
*pDest = NULL;
}
_lzmaCompStream.avail_out = _ulOutBufSize;
_lzmaCompStream.next_out = _pOutputBuffer;
_bFlushed = true;
lzma_end (&_lzmaCompStream);
resetCompStream();
if (LZMA_OK != lzma_easy_encoder (&_lzmaCompStream, getCompressionLevel(), COMPRESSION_CHECK)) {
return -5;
}
return 0;
}
LZMACompressor::LZMACompressor(Writer* pWriter, uint32_t preset, lzma_check check_type)
: Compressor(pWriter) {
bzero(&_stream, sizeof(_stream));
if (lzma_easy_encoder(&_stream, preset, check_type) != LZMA_OK)
throw std::runtime_error("lzma_easy_encoder returned non-LZMA_OK");
};
std::unique_ptr<IOBuf> LZMA2Codec::doCompress(const IOBuf* data) {
lzma_ret rc;
lzma_stream stream = LZMA_STREAM_INIT;
rc = lzma_easy_encoder(&stream, level_, LZMA_CHECK_NONE);
if (rc != LZMA_OK) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_easy_encoder error: ", rc));
}
SCOPE_EXIT { lzma_end(&stream); };
uint64_t uncompressedLength = data->computeChainDataLength();
uint64_t maxCompressedLength = lzma_stream_buffer_bound(uncompressedLength);
// Max 64MiB in one go
constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
auto out = addOutputBuffer(
&stream,
(maxCompressedLength <= maxSingleStepLength ?
maxCompressedLength :
defaultBufferLength));
if (encodeSize()) {
auto size = IOBuf::createCombined(kMaxVarintLength64);
encodeVarintToIOBuf(uncompressedLength, size.get());
size->appendChain(std::move(out));
out = std::move(size);
}
for (auto& range : *data) {
if (range.empty()) {
continue;
}
stream.next_in = const_cast<uint8_t*>(range.data());
stream.avail_in = range.size();
while (stream.avail_in != 0) {
if (stream.avail_out == 0) {
out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
}
rc = lzma_code(&stream, LZMA_RUN);
if (rc != LZMA_OK) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_code error: ", rc));
}
}
}
do {
if (stream.avail_out == 0) {
out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
}
rc = lzma_code(&stream, LZMA_FINISH);
} while (rc == LZMA_OK);
if (rc != LZMA_STREAM_END) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_code ended with error: ", rc));
}
out->prev()->trimEnd(stream.avail_out);
return out;
}
int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
#ifdef HAVE_XZ
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret;
uint8_t buf[BUFSIZ], out[BUFSIZ];
lzma_action action = LZMA_RUN;
assert(fdf >= 0);
assert(fdt >= 0);
ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
if (ret != LZMA_OK) {
log_error("Failed to initialize XZ encoder: code %u", ret);
return -EINVAL;
}
for (;;) {
if (s.avail_in == 0 && action == LZMA_RUN) {
size_t m = sizeof(buf);
ssize_t n;
if (max_bytes != -1 && m > (size_t) max_bytes)
m = max_bytes;
n = read(fdf, buf, m);
if (n < 0)
return -errno;
if (n == 0)
action = LZMA_FINISH;
else {
s.next_in = buf;
s.avail_in = n;
if (max_bytes != -1) {
assert(max_bytes >= n);
max_bytes -= n;
}
}
}
if (s.avail_out == 0) {
s.next_out = out;
s.avail_out = sizeof(out);
}
ret = lzma_code(&s, action);
if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
log_error("Compression failed: code %u", ret);
return -EBADMSG;
}
if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
ssize_t n, k;
n = sizeof(out) - s.avail_out;
k = loop_write(fdt, out, n, false);
if (k < 0)
return k;
if (ret == LZMA_STREAM_END) {
log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
s.total_in, s.total_out,
(double) s.total_out / s.total_in * 100);
return 0;
}
}
}
#else
return -EPROTONOSUPPORT;
#endif
}
