Beispiel #1
0
int main(int argc, char *argv[])
{
	QString workDir;
#ifdef Q_OS_MAC
    if (QDir(QString()).absolutePath() == "/") {
		QString first = argc ? QString::fromLocal8Bit(argv[0]) : QString();
		if (!first.isEmpty()) {
			QFileInfo info(first);
			if (info.exists()) {
				QDir result(info.absolutePath() + "/../../..");
				workDir = result.absolutePath() + '/';
			}
		}
	}
#endif

	QString remove;
	int version = 0;
	QFileInfoList files;
	for (int i = 0; i < argc; ++i) {
		if (string("-path") == argv[i] && i + 1 < argc) {
			QString path = workDir + QString(argv[i + 1]);
			QFileInfo info(path);
			files.push_back(info);
			if (remove.isEmpty()) remove = info.canonicalPath() + "/";
		} else if (string("-version") == argv[i] && i + 1 < argc) {
			version = QString(argv[i + 1]).toInt();
		} else if (string("-dev") == argv[i]) {
			DevChannel = true;
		} else if (string("-beta") == argv[i] && i + 1 < argc) {
			BetaVersion = QString(argv[i + 1]).toULongLong();
			if (BetaVersion > version * 1000ULL && BetaVersion < (version + 1) * 1000ULL) {
				DevChannel = false;
				BetaSignature = countBetaVersionSignature(BetaVersion);
				if (BetaSignature.isEmpty()) {
					return -1;
				}
			} else {
				cout << "Bad -beta param value passed, should be for the same version: " << version << ", beta: " << BetaVersion << "\n";
				return -1;
			}
		}
	}

	if (files.isEmpty() || remove.isEmpty() || version <= 1016 || version > 999999999) {
#ifdef Q_OS_WIN
		cout << "Usage: Packer.exe -path {file} -version {version} OR Packer.exe -path {dir} -version {version}\n";
#elif defined Q_OS_MAC
		cout << "Usage: Packer.app -path {file} -version {version} OR Packer.app -path {dir} -version {version}\n";
#else
		cout << "Usage: Packer -path {file} -version {version} OR Packer -path {dir} -version {version}\n";
#endif
		return -1;
	}

	bool hasDirs = true;
	while (hasDirs) {
		hasDirs = false;
		for (QFileInfoList::iterator i = files.begin(); i != files.end(); ++i) {
			QFileInfo info(*i);
			QString fullPath = info.canonicalFilePath();
			if (info.isDir()) {
				hasDirs = true;
				files.erase(i);
				QDir d = QDir(info.absoluteFilePath());
				QString fullDir = d.canonicalPath();
				QStringList entries = d.entryList(QDir::Files | QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot);
				files.append(d.entryInfoList(QDir::Files | QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot));
				break;
			} else if (!info.isReadable()) {
				cout << "Can't read: " << info.absoluteFilePath().toUtf8().constData() << "\n";
				return -1;
			} else if (info.isHidden()) {
				hasDirs = true;
				files.erase(i);
				break;
			}
		}
	}
	for (QFileInfoList::iterator i = files.begin(); i != files.end(); ++i) {
		QFileInfo info(*i);
		if (!info.canonicalFilePath().startsWith(remove)) {
			cout << "Can't find '" << remove.toUtf8().constData() << "' in file '" << info.canonicalFilePath().toUtf8().constData() << "' :(\n";
			return -1;
		}
	}

	QByteArray result;
	{
		QBuffer buffer(&result);
		buffer.open(QIODevice::WriteOnly);
		QDataStream stream(&buffer);
		stream.setVersion(QDataStream::Qt_5_1);

		if (BetaVersion) {
			stream << quint32(0x7FFFFFFF);
			stream << quint64(BetaVersion);
		} else {
			stream << quint32(version);
		}

		stream << quint32(files.size());
		cout << "Found " << files.size() << " file" << (files.size() == 1 ? "" : "s") << "..\n";
		for (QFileInfoList::iterator i = files.begin(); i != files.end(); ++i) {
			QFileInfo info(*i);
			QString fullName = info.canonicalFilePath();
			QString name = fullName.mid(remove.length());
			cout << name.toUtf8().constData() << " (" << info.size() << ")\n";

			QFile f(fullName);
			if (!f.open(QIODevice::ReadOnly)) {
				cout << "Can't open '" << fullName.toUtf8().constData() << "' for read..\n";
				return -1;
			}
			QByteArray inner = f.readAll();
			stream << name << quint32(inner.size()) << inner;
#if defined Q_OS_MAC || defined Q_OS_LINUX
			stream << (QFileInfo(fullName).isExecutable() ? true : false);
#endif
		}
		if (stream.status() != QDataStream::Ok) {
			cout << "Stream status is bad: " << stream.status() << "\n";
			return -1;
		}
	}

	int32 resultSize = result.size();
	cout << "Compression start, size: " << resultSize << "\n";

	QByteArray compressed, resultCheck;
#ifdef Q_OS_WIN // use Lzma SDK for win
	const int32 hSigLen = 128, hShaLen = 20, hPropsLen = LZMA_PROPS_SIZE, hOriginalSizeLen = sizeof(int32), hSize = hSigLen + hShaLen + hPropsLen + hOriginalSizeLen; // header

	compressed.resize(hSize + resultSize + 1024 * 1024); // rsa signature + sha1 + lzma props + max compressed size

	size_t compressedLen = compressed.size() - hSize;
	size_t outPropsSize = LZMA_PROPS_SIZE;
	uchar *_dest = (uchar*)(compressed.data() + hSize);
	size_t *_destLen = &compressedLen;
	const uchar *_src = (const uchar*)(result.constData());
	size_t _srcLen = result.size();
	uchar *_outProps = (uchar*)(compressed.data() + hSigLen + hShaLen);
	int res = LzmaCompress(_dest, _destLen, _src, _srcLen, _outProps, &outPropsSize, 9, 64 * 1024 * 1024, 4, 0, 2, 273, 2);
	if (res != SZ_OK) {
		cout << "Error in compression: " << res << "\n";
		return -1;
	}
	compressed.resize(int(hSize + compressedLen));
	memcpy(compressed.data() + hSigLen + hShaLen + hPropsLen, &resultSize, hOriginalSizeLen);

	cout << "Compressed to size: " << compressedLen << "\n";

	cout << "Checking uncompressed..\n";

	int32 resultCheckLen;
	memcpy(&resultCheckLen, compressed.constData() + hSigLen + hShaLen + hPropsLen, hOriginalSizeLen);
	if (resultCheckLen <= 0 || resultCheckLen > 1024 * 1024 * 1024) {
		cout << "Bad result len: " << resultCheckLen << "\n";
		return -1;
	}
	resultCheck.resize(resultCheckLen);

	size_t resultLen = resultCheck.size();
	SizeT srcLen = compressedLen;
	int uncompressRes = LzmaUncompress((uchar*)resultCheck.data(), &resultLen, (const uchar*)(compressed.constData() + hSize), &srcLen, (const uchar*)(compressed.constData() + hSigLen + hShaLen), LZMA_PROPS_SIZE);
	if (uncompressRes != SZ_OK) {
		cout << "Uncompress failed: " << uncompressRes << "\n";
		return -1;
	}
	if (resultLen != size_t(result.size())) {
		cout << "Uncompress bad size: " << resultLen << ", was: " << result.size() << "\n";
		return -1;
	}
#else // use liblzma for others
	const int32 hSigLen = 128, hShaLen = 20, hPropsLen = 0, hOriginalSizeLen = sizeof(int32), hSize = hSigLen + hShaLen + hOriginalSizeLen; // header

	compressed.resize(hSize + resultSize + 1024 * 1024); // rsa signature + sha1 + lzma props + max compressed size

	size_t compressedLen = compressed.size() - hSize;

	lzma_stream stream = LZMA_STREAM_INIT;

	int preset = 9 | LZMA_PRESET_EXTREME;
	lzma_ret ret = lzma_easy_encoder(&stream, preset, LZMA_CHECK_CRC64);
	if (ret != LZMA_OK) {
		const char *msg;
		switch (ret) {
			case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break;
			case LZMA_OPTIONS_ERROR: msg = "Specified preset is not supported"; break;
			case LZMA_UNSUPPORTED_CHECK: msg = "Specified integrity check is not supported"; break;
			default: msg = "Unknown error, possibly a bug"; break;
		}
		cout << "Error initializing the encoder: " << msg << " (error code " << ret << ")\n";
		return -1;
	}

	stream.avail_in = resultSize;
	stream.next_in = (uint8_t*)result.constData();
	stream.avail_out = compressedLen;
	stream.next_out = (uint8_t*)(compressed.data() + hSize);

	lzma_ret res = lzma_code(&stream, LZMA_FINISH);
	compressedLen -= stream.avail_out;
	lzma_end(&stream);
	if (res != LZMA_OK && res != LZMA_STREAM_END) {
		const char *msg;
		switch (res) {
			case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break;
			case LZMA_DATA_ERROR: msg = "File size limits exceeded"; break;
			default: msg = "Unknown error, possibly a bug"; break;
		}
		cout << "Error in compression: " << msg << " (error code " << res << ")\n";
		return -1;
	}

	compressed.resize(int(hSize + compressedLen));
	memcpy(compressed.data() + hSigLen + hShaLen, &resultSize, hOriginalSizeLen);

	cout << "Compressed to size: " << compressedLen << "\n";

	cout << "Checking uncompressed..\n";

	int32 resultCheckLen;
	memcpy(&resultCheckLen, compressed.constData() + hSigLen + hShaLen, hOriginalSizeLen);
	if (resultCheckLen <= 0 || resultCheckLen > 1024 * 1024 * 1024) {
		cout << "Bad result len: " << resultCheckLen << "\n";
		return -1;
	}
	resultCheck.resize(resultCheckLen);

	size_t resultLen = resultCheck.size();

	stream = LZMA_STREAM_INIT;

	ret = lzma_stream_decoder(&stream, UINT64_MAX, LZMA_CONCATENATED);
	if (ret != LZMA_OK) {
		const char *msg;
		switch (ret) {
			case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break;
			case LZMA_OPTIONS_ERROR: msg = "Specified preset is not supported"; break;
			case LZMA_UNSUPPORTED_CHECK: msg = "Specified integrity check is not supported"; break;
			default: msg = "Unknown error, possibly a bug"; break;
		}
		cout << "Error initializing the decoder: " << msg << " (error code " << ret << ")\n";
		return -1;
	}

	stream.avail_in = compressedLen;
	stream.next_in = (uint8_t*)(compressed.constData() + hSize);
	stream.avail_out = resultLen;
	stream.next_out = (uint8_t*)resultCheck.data();

	res = lzma_code(&stream, LZMA_FINISH);
	if (stream.avail_in) {
		cout << "Error in decompression, " << stream.avail_in << " bytes left in _in of " << compressedLen << " whole.\n";
		return -1;
	} else if (stream.avail_out) {
		cout << "Error in decompression, " << stream.avail_out << " bytes free left in _out of " << resultLen << " whole.\n";
		return -1;
	}
	lzma_end(&stream);
	if (res != LZMA_OK && res != LZMA_STREAM_END) {
		const char *msg;
		switch (res) {
			case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break;
			case LZMA_FORMAT_ERROR: msg = "The input data is not in the .xz format"; break;
			case LZMA_OPTIONS_ERROR: msg = "Unsupported compression options"; break;
			case LZMA_DATA_ERROR: msg = "Compressed file is corrupt"; break;
			case LZMA_BUF_ERROR: msg = "Compressed data is truncated or otherwise corrupt"; break;
			default: msg = "Unknown error, possibly a bug"; break;
		}
		cout << "Error in decompression: " << msg << " (error code " << res << ")\n";
		return -1;
	}
#endif
	if (memcmp(result.constData(), resultCheck.constData(), resultLen)) {
		cout << "Data differ :(\n";
		return -1;
	}
	/**/
	result = resultCheck = QByteArray();

	cout << "Counting SHA1 hash..\n";

	uchar sha1Buffer[20];
	memcpy(compressed.data() + hSigLen, hashSha1(compressed.constData() + hSigLen + hShaLen, uint32(compressedLen + hPropsLen + hOriginalSizeLen), sha1Buffer), hShaLen); // count sha1

	uint32 siglen = 0;

	cout << "Signing..\n";
	RSA *prKey = PEM_read_bio_RSAPrivateKey(BIO_new_mem_buf(const_cast<char*>((DevChannel || BetaVersion) ? PrivateDevKey : PrivateKey), -1), 0, 0, 0);
	if (!prKey) {
		cout << "Could not read RSA private key!\n";
		return -1;
	}
	if (RSA_size(prKey) != hSigLen) {
		cout << "Bad private key, size: " << RSA_size(prKey) << "\n";
		RSA_free(prKey);
		return -1;
	}
	if (RSA_sign(NID_sha1, (const uchar*)(compressed.constData() + hSigLen), hShaLen, (uchar*)(compressed.data()), &siglen, prKey) != 1) { // count signature
		cout << "Signing failed!\n";
		RSA_free(prKey);
		return -1;
	}
	RSA_free(prKey);

	if (siglen != hSigLen) {
		cout << "Bad signature length: " << siglen << "\n";
		return -1;
	}

	cout << "Checking signature..\n";
	RSA *pbKey = PEM_read_bio_RSAPublicKey(BIO_new_mem_buf(const_cast<char*>((DevChannel || BetaVersion) ? PublicDevKey : PublicKey), -1), 0, 0, 0);
	if (!pbKey) {
		cout << "Could not read RSA public key!\n";
		return -1;
	}
	if (RSA_verify(NID_sha1, (const uchar*)(compressed.constData() + hSigLen), hShaLen, (const uchar*)(compressed.constData()), siglen, pbKey) != 1) { // verify signature
		RSA_free(pbKey);
		cout << "Signature verification failed!\n";
		return -1;
	}
	cout << "Signature verified!\n";
	RSA_free(pbKey);
#ifdef Q_OS_WIN
	QString outName(QString("tupdate%1").arg(BetaVersion ? BetaVersion : version));
#elif defined Q_OS_MAC
	QString outName(QString("tmacupd%1").arg(BetaVersion ? BetaVersion : version));
#elif defined Q_OS_LINUX32
    QString outName(QString("tlinux32upd%1").arg(BetaVersion ? BetaVersion : version));
#elif defined Q_OS_LINUX64
    QString outName(QString("tlinuxupd%1").arg(BetaVersion ? BetaVersion : version));
#else
#error Unknown platform!
#endif
	if (BetaVersion) {
		outName += "_" + BetaSignature;
	}
	QFile out(outName);
	if (!out.open(QIODevice::WriteOnly)) {
		cout << "Can't open '" << outName.toUtf8().constData() << "' for write..\n";
		return -1;
	}
	out.write(compressed);
	out.close();

	if (BetaVersion) {
		QString keyName(QString("tbeta_%1_key").arg(BetaVersion));
		QFile key(keyName);
		if (!key.open(QIODevice::WriteOnly)) {
			cout << "Can't open '" << keyName.toUtf8().constData() << "' for write..\n";
			return -1;
		}
		key.write(BetaSignature.toUtf8());
		key.close();
	}

	cout << "Update file '" << outName.toUtf8().constData() << "' written successfully!\n";

	return 0;
}
Beispiel #2
0
bool LocateConverter::QQWryToLocate(const wchar_t *qqwry, const wchar_t *locate, bool compress/* = false */, OnProgress progress/* = NULL */)
{
    QQWryReader QQWry(qqwry);
    if ( !QQWry.IsAvailable() ) return false;

    StringTable string_table1;
    StringTable string_table2;
    RecordTable record_table;

    Buffer buffer;

    uint32_t last_begin_ip = 0;
    uint32_t last_end_ip = 0;

    for(uint32_t i=0; i<QQWry.GetInfo()->count; i++)
    {
        LocateItem *item = QQWry.GetItem(i + 1);

        LOCATE record;
        record.begin_ip = item->begin_ip;
        record.table1 = string_table1.Append(item->region);
        record.table2 = string_table2.Append(item->address);

        if ( i > 0 )
        {
            if ( record.begin_ip - last_end_ip != 1 )
            {
                //printf("逻辑错误,IP段不连续! 行:%d\n", i + 1);
                //getchar();
                return false;
            }

            uint32_t diff = record.begin_ip - last_begin_ip;
            if ( compress && isPowerOf2(diff) )
            {
                record.begin_ip = LogBase2(diff);
            }
        }


        record_table.Append(&record);

        last_begin_ip = item->begin_ip;
        last_end_ip = item->end_ip;
    }

    //合并数据区
    Buffer *record_table_buffer = record_table;
    Buffer *string_table1_buffer = string_table1;
    Buffer *string_table2_buffer = string_table2;

    std::copy(record_table_buffer->begin(), record_table_buffer->end(), std::back_inserter(buffer));
    std::copy(string_table1_buffer->begin(), string_table1_buffer->end(), std::back_inserter(buffer));
    std::copy(string_table2_buffer->begin(), string_table2_buffer->end(), std::back_inserter(buffer));

    //生成文件头
    HEADER header;
    header.magic = LOCATE_MAGIC;
    header.version = LOCATE_VERISON;
    header.compress = compress?1:0;
    header.total = QQWry.GetInfo()->count;
    header.time = QQWry.GetInfo()->time;

    header.table1 = sizeof(header) + record_table_buffer->size(); // 这里不加LZMA_PROPS_SIZE的原因是解压后,抛弃props信息
    header.table2 = header.table1 + string_table1_buffer->size();
    header.size = buffer.size();
    header.crc32 = CRC32_MEM((uint8_t*)&header, sizeof(header) - 4);

    uint32_t lzma_buffer_len = buffer.size();
    uint8_t *lzma_buffer = 0;

    size_t prop_size = LZMA_PROPS_SIZE;
    BYTE outProps[LZMA_PROPS_SIZE];

    //准备压缩
    if(compress)
    {
        lzma_buffer = (uint8_t *)malloc(lzma_buffer_len);
        
        ProgressCallback LzmaCompressProgress;
        LzmaCompressProgress.Progress = LzmaOnProgress;
        LzmaCompressProgress.totalInSize = buffer.size();
        LzmaCompressProgress.progress = progress;
        LzmaCompress(lzma_buffer, &lzma_buffer_len, &buffer[0], buffer.size(), (ICompressProgress*)&LzmaCompressProgress, outProps, &prop_size, 5, 1<<27, 8, 0, 2, 64, 4);
    }

    //保存文件
    FILE * out = _wfopen(locate, L"wb");
    fwrite(&header, 1, sizeof(header), out);

    if(compress)
    {
        fwrite(outProps, 1, sizeof(outProps), out);
        fwrite(lzma_buffer, 1, lzma_buffer_len, out);
    }
    else
    {
        fwrite(&buffer[0], 1, buffer.size(), out);
    }
    fclose(out);

    if(compress)
    {
        free(lzma_buffer);
    }
    return true;
}
Beispiel #3
0
bool LocateConverter::GeneratePatch(const wchar_t *qqwry1_path, const wchar_t *qqwry2_path)
{
    QQWryReader QQWry1(qqwry1_path);
    QQWryReader QQWry2(qqwry2_path);

    if ( !QQWry1.IsAvailable() ) return false;
    if ( !QQWry2.IsAvailable() ) return false;

    if(QQWry1.GetInfo()->time==QQWry2.GetInfo()->time) return false;

    // 根据时间交换,低版本在前
    QQWryReader *qqwry1 = &QQWry1;
    QQWryReader *qqwry2 = &QQWry2;
    if(QQWry1.GetInfo()->time>QQWry2.GetInfo()->time)
    {
        QQWryReader *temp = qqwry1;
        qqwry1 = qqwry2;
        qqwry2 = temp;
    }

    uint32_t i = 0;
    uint32_t j = 0;

    uint8_t flag = 0;

    uint32_t n = 0;
    uint32_t m = 0;
    //int x = 1;

    StringTable string_table1;
    StringTable string_table2;
    Buffer RecordBuffer;         // 记录
    Buffer ResultBuffer;         // 最终结果

    while(i<qqwry1->GetInfo()->count && j<qqwry2->GetInfo()->count)
    {
        LocateItem *Item1 = qqwry1->GetItem(i + 1);
        LocateItem *Item2 = qqwry2->GetItem(j + 1);

        if( Item1->begin_ip != Item2->begin_ip )
        {
            if( Item1->begin_ip < Item2->begin_ip ) i++;
            else j++;

            flag = 1;
            continue;
        }
        if( Item1->end_ip != Item2->end_ip )
        {
            if( Item1->end_ip < Item2->end_ip ) i++;
            else j++;

            flag = 1;
            continue;
        }

        if(flag==1)
        {
            flag = 0;

            //printf("%d-%d,%d-%d\n",n+1,i,m+1,j);
            for(uint32_t k=0; k<i-n; k++)
            {
                DIFFITEM record;
                record.line = n + 1 + k;
                record.method = REMOVE;
                record.begin_ip = 0;
                record.table1 = 0;
                record.table2 = 0;

                AppendBuffer(RecordBuffer, (uint8_t*)&record, sizeof(record));
                //printf("%d: 删除 %d\n",x++,n+1+k);
            }
            for(uint32_t l=0; l<j-m; l++)
            {
                Item2 = qqwry2->GetItem(m + l + 1);

                DIFFITEM record;
                record.line = i;
                record.method = INSERT;
                record.begin_ip = Item2->begin_ip;
                record.table1 = string_table1.Append(Item2->region);
                record.table2 = string_table2.Append(Item2->address);

                AppendBuffer(RecordBuffer, (uint8_t*)&record, sizeof(record));
                //printf("%d: 添加 (%d) -> %d\n", x++, m+1+k, i);
            }
        }

        Item1 = qqwry1->GetItem(i + 1);
        Item2 = qqwry2->GetItem(j + 1);
        if( strcmp(Item1->region, Item2->region) || strcmp(Item1->address, Item2->address) )
        {
            DIFFITEM record;
            record.line = i + 1;
            record.method = MODIFY;
            record.begin_ip = 0;
            record.table1 = string_table1.Append(Item2->region);
            record.table2 = string_table2.Append(Item2->address);

            AppendBuffer(RecordBuffer, (uint8_t*)&record, sizeof(record));
            //printf("%d: 修改 (%d) -> %d\n", x++, j+1, i+1);
        }

        i++;
        j++;
        n = i;
        m = j;
    }

    //合并数据区
    Buffer *string_table1_buffer = string_table1;
    Buffer *string_table2_buffer = string_table2;

    std::copy(RecordBuffer.begin(), RecordBuffer.end(), std::back_inserter(ResultBuffer));
    std::copy(string_table1_buffer->begin(), string_table1_buffer->end(), std::back_inserter(ResultBuffer));
    std::copy(string_table2_buffer->begin(), string_table2_buffer->end(), std::back_inserter(ResultBuffer));

    //生成文件头
    DIFFHEADER header;
    header.magic = DIFF_MAGIC;
    header.total1 = qqwry1->GetInfo()->count;
    header.total2 = qqwry2->GetInfo()->count;
    header.time1 = qqwry1->GetInfo()->time;
    header.time2 = qqwry2->GetInfo()->time;
    header.table1 = sizeof(header) + RecordBuffer.size();;
    header.table2 = header.table1 + string_table1_buffer->size();
    header.size = ResultBuffer.size();
    header.crc32 = CRC32_MEM((uint8_t*)&header, sizeof(header) - 4);

    //准备压缩
    uint32_t lzma_buffer_len = ResultBuffer.size();
    uint8_t *lzma_buffer = (uint8_t *)malloc(lzma_buffer_len);

    size_t prop_size = LZMA_PROPS_SIZE;
    BYTE outProps[LZMA_PROPS_SIZE];
    LzmaCompress(lzma_buffer, &lzma_buffer_len, &ResultBuffer[0], ResultBuffer.size(), NULL, outProps, &prop_size, 5, 1<<27, 8, 0, 2, 64, 4);

    //保存文件
    char temp[1024];
    sprintf(temp,"%d-%d.dif",qqwry1->GetInfo()->time,qqwry2->GetInfo()->time);
    FILE * out = fopen(temp, "wb");
    fwrite(&header, 1, sizeof(header), out);
    fwrite(outProps, 1, sizeof(outProps), out);
    fwrite(lzma_buffer, 1, lzma_buffer_len, out);
    //fwrite(&ResultBuffer[0], 1, ResultBuffer.size(), out);
    fclose(out);

    free(lzma_buffer);
    return true;
}
Beispiel #4
0
bool LocateConverter::LocateUpdate(const wchar_t *locate, const wchar_t *patch, const wchar_t *path, bool compress/* = false */, OnProgress progress/* = NULL */)
{
    LocateReader loc(locate);
    if ( !loc.IsAvailable() ) return false;

    const uint8_t *diff_buffer = 0;
    PDIFFHEADER diff_header = 0;

    HANDLE hfile = CreateFileW(patch, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
    if(hfile!=INVALID_HANDLE_VALUE)
    {
        uint32_t file_length = GetFileSize(hfile, NULL);

        HANDLE hfilemap = CreateFileMapping(hfile, NULL, PAGE_READONLY, 0, 0, NULL);
        CloseHandle(hfile);

        diff_buffer = (const uint8_t*) MapViewOfFile(hfilemap, FILE_MAP_READ, 0, 0, 0);
        CloseHandle(hfilemap);

        diff_header = (PDIFFHEADER)diff_buffer;
        int data_size = file_length - sizeof(DIFFHEADER) - LZMA_PROPS_SIZE;
        if( data_size<0 || diff_header->magic != DIFF_MAGIC || CRC32_MEM(diff_buffer, sizeof(DIFFHEADER) - 4)!=diff_header->crc32 )
        {
            UnmapViewOfFile((void*)diff_buffer);
            return false;
        }
        else
        {
            uint32_t lzma_buffer_len = diff_header->size;
            uint8_t *lzma_buffer = (uint8_t *)malloc(sizeof(DIFFHEADER) + lzma_buffer_len);

            if( LzmaUncompress(lzma_buffer + sizeof(DIFFHEADER), &lzma_buffer_len, (unsigned char*)diff_header->data + LZMA_PROPS_SIZE, (uint32_t*)&data_size, (unsigned char*)diff_header->data, LZMA_PROPS_SIZE)==SZ_OK )
            {
                memcpy(lzma_buffer, diff_buffer, sizeof(DIFFHEADER));
                UnmapViewOfFile((void*)diff_buffer);
                diff_buffer = lzma_buffer;

                diff_header = (PDIFFHEADER)diff_buffer;
                diff_header->table1 += (uint32_t)diff_buffer;
                diff_header->table2 += (uint32_t)diff_buffer;
            }
            else
            {
                free(lzma_buffer);
                UnmapViewOfFile((void*)diff_buffer);
                return false;
            }
        }
    }
    else
    {
        return false;
    }

    if ( loc.GetInfo()->count!=diff_header->total1 || loc.GetInfo()->time!=diff_header->time1 ) return false;

    PDIFFITEM diffitem = diff_header->data;
    LocateItem *Locate = (LocateItem *)malloc( diff_header->total2 * sizeof(LocateItem) );

    uint32_t last_diff = ( diff_header->table1 - sizeof(DIFFHEADER) - (uint32_t)diff_buffer ) / sizeof(DIFFITEM) - 1;
    uint32_t last_line = diff_header->total2 - 1;

    uint32_t i = loc.GetInfo()->count;
    for(; i>0; i--)
    {
        LocateItem *item = loc.GetItem(i);

        if(i!=diffitem[last_diff].line)
        {
            Locate[last_line].begin_ip = item->begin_ip;
            Locate[last_line].region = item->region;
            Locate[last_line].address = item->address;
            last_line--;
        }
        else
        {
            switch(diffitem[last_diff].method)
            {
            case INSERT:
                //printf("INSERT %d %d\n", i, diffitem[last_diff-1].line);
                Locate[last_line].begin_ip = diffitem[last_diff].begin_ip;
                Locate[last_line].region = (const char*)( diffitem[last_diff].table1 + diff_header->table1 );
                Locate[last_line].address = (const char*)( diffitem[last_diff].table2 + diff_header->table2 );
                last_line--;
                i++;
                break;
            case REMOVE:
                //printf("REMOVE %d %d %d\n", i, diffitem[last_diff-1].line, diffitem[last_diff-2].line);
                break;
            case MODIFY:
                Locate[last_line].begin_ip = item->begin_ip;
                Locate[last_line].region = (const char*)( diffitem[last_diff].table1 + diff_header->table1 );
                Locate[last_line].address = (const char*)( diffitem[last_diff].table2 + diff_header->table2 );
                //printf("MODIFY %d %s %s\n", last_line+1, Locate[last_line].region, Locate[last_line].address);
                //getchar();
                last_line--;
                break;
            }
            last_diff--;
        }
    }

    //printf("%d %d\n",last_diff,last_line);
    if ( last_diff!=-1 || last_line!=-1 ) return false;

    StringTable string_table1;
    StringTable string_table2;
    RecordTable record_table;

    Buffer buffer;

    uint32_t last_begin_ip = 0;

    for(i=last_line+1; i<diff_header->total2; i++)
    {
        LocateItem *item = &Locate[i];

        LOCATE record;
        record.begin_ip = item->begin_ip;
        record.table1 = string_table1.Append(item->region);
        record.table2 = string_table2.Append(item->address);

        if ( i > 0 )
        {
            uint32_t diff = record.begin_ip - last_begin_ip;
            if ( compress && isPowerOf2(diff) )
            {
                record.begin_ip = LogBase2(diff);
            }
        }

        record_table.Append(&record);

        last_begin_ip = item->begin_ip;
    }
    free(Locate);

    //合并数据区
    Buffer *record_table_buffer = record_table;
    Buffer *string_table1_buffer = string_table1;
    Buffer *string_table2_buffer = string_table2;

    std::copy(record_table_buffer->begin(), record_table_buffer->end(), std::back_inserter(buffer));
    std::copy(string_table1_buffer->begin(), string_table1_buffer->end(), std::back_inserter(buffer));
    std::copy(string_table2_buffer->begin(), string_table2_buffer->end(), std::back_inserter(buffer));

    //生成文件头
    HEADER header;
    header.magic = LOCATE_MAGIC;
    header.version = LOCATE_VERISON;
    header.compress = compress?1:0;
    header.total = diff_header->total2;
    header.time = diff_header->time2;

    header.table1 = sizeof(header) + record_table_buffer->size(); // 这里不加LZMA_PROPS_SIZE的原因是解压后,抛弃props信息
    header.table2 = header.table1 + string_table1_buffer->size();
    header.size = buffer.size();
    header.crc32 = CRC32_MEM((uint8_t*)&header, sizeof(header) - 4);

    uint32_t lzma_buffer_len = buffer.size();
    uint8_t *lzma_buffer = 0;

    size_t prop_size = LZMA_PROPS_SIZE;
    BYTE outProps[LZMA_PROPS_SIZE];

    //准备压缩
    if(compress)
    {
        lzma_buffer = (uint8_t *)malloc(lzma_buffer_len);
        
        ProgressCallback LzmaCompressProgress;
        LzmaCompressProgress.Progress = LzmaOnProgress;
        LzmaCompressProgress.totalInSize = buffer.size();
        LzmaCompressProgress.progress = progress;
        LzmaCompress(lzma_buffer, &lzma_buffer_len, &buffer[0], buffer.size(), (ICompressProgress*)&LzmaCompressProgress, outProps, &prop_size, 5, 1<<27, 8, 0, 2, 64, 4);
    }

    //保存文件
    FILE * out = _wfopen(path, L"wb");
    fwrite(&header, 1, sizeof(header), out);

    if(compress)
    {
        fwrite(outProps, 1, sizeof(outProps), out);
        fwrite(lzma_buffer, 1, lzma_buffer_len, out);
    }
    else
    {
        fwrite(&buffer[0], 1, buffer.size(), out);
    }
    fclose(out);

    if(compress)
    {
        free(lzma_buffer);
    }

    free((void*)diff_buffer);
    return true;
}
Beispiel #5
0
int main(int argc, char **argv) {
	if(argc < 2) {
		printf("Usage: %s infile\n", argv[0]);
		exit(1);
	}

	MPI_Comm comm = MPI_COMM_WORLD;
	MPI_Info mpi_info = MPI_INFO_NULL;
	MPI_File fh, fw;
	MPI_Offset file_size, frag_size, read_size;
	MPI_Offset offset;
	MPI_Status status;
	int retval;
	double start, end;

	unsigned char *buf, *outbuf, *outProps;
	size_t destlen;
	size_t propsize = 5;

	MPI_Init(&argc, &argv);
	MPI_Comm_rank(comm, &mpi_rank);
	MPI_Comm_size(comm, &mpi_size);

	MPI_Barrier(comm);
	start = MPI_Wtime();
	/*
	 * read
	 */
	MPI_File_open(comm, argv[1], MPI_MODE_RDONLY, mpi_info, &fh);
	MPI_File_get_size(fh, &file_size);
	//printf("file size:%d\n", file_size);

	frag_size = file_size / mpi_size;
	offset = frag_size * mpi_rank;
	read_size = MIN(frag_size, file_size - offset);
	//printf("rank %d offset %d\n", mpi_rank, offset);

	buf = malloc(frag_size + 2);
	assert(buf != NULL);
	MPI_File_open(comm, argv[1], MPI_MODE_RDONLY, mpi_info, &fh);
	MPI_File_read_at(fh, offset, buf, read_size, MPI_CHAR, &status);
	MPI_File_close(&fh);

	/*
	 * compress
	 */
	destlen = 1.2 * frag_size + 1024 * 1024;
	outbuf = (unsigned char *)malloc(destlen);
	assert(outbuf != NULL);
	destlen = destlen - DATA_OFFSET -propsize;
	outProps = outbuf + DATA_OFFSET;
	retval = LzmaCompress(outbuf + DATA_OFFSET + propsize, &destlen, buf, read_size, outProps, &propsize, -1, 0, -1, -1, -1, -1, 1);
	if(retval != SZ_OK) {
		error_print(retval);
		free(buf);
		free(outbuf);
		exit(1);
	}

	/*
	 * write
	 */
	char *fwname;
	unsigned long long *len;
	fwname = get_fwname(argv[1]);
	len = (unsigned long long *)outbuf;
	*len = read_size;
	//printf("%s %d\n", fwname, destlen);
	MPI_File_open(MPI_COMM_SELF, fwname, MPI_MODE_WRONLY | MPI_MODE_CREATE, mpi_info, &fw);
	MPI_File_set_size(fw, destlen);
	MPI_File_write(fw, outbuf, destlen + DATA_OFFSET + propsize, MPI_CHAR, &status);
	MPI_File_close(&fw);

	MPI_Barrier(comm);
	end = MPI_Wtime();

	size_t cmprs_len;
	double cmprs_ratio;
	MPI_Reduce(&destlen, &cmprs_len, 1, MPI_UNSIGNED_LONG, MPI_SUM, 0, comm);
	if(0 == mpi_rank) {
		cmprs_ratio = (double)cmprs_len / file_size;
		printf("file size: %lu\n", file_size);
		printf("after compressed: %lu\n", cmprs_len);
		printf("compress ratio: %f\n", cmprs_ratio);
		printf("number of processes: %d\n", mpi_size);
		printf("time used: %fs\n", end - start);
	}
	MPI_Finalize();
	free(fwname);
	free(buf);
	free(outbuf);
	return 0;
}
Beispiel #6
0
static void lzma_compress_buf(struct stream *s, int *c_type, i64 *c_len)
{
	uchar *c_buf;
	size_t dlen;
	size_t prop_size = 5; /* return value for lzma_properties */
	int lzma_ret;

	if (!lzo_compresses(s))
		goto out;

	dlen = s->buflen;
	c_buf = malloc(dlen);
	if (!c_buf)
		return;

	if (control.flags & FLAG_SHOW_PROGRESS) {
		fprintf(control.msgout, "\tProgress percentage pausing during lzma compression...");
		fflush(control.msgout);
	}
	/* with LZMA SDK 4.63, we pass compression level and threads only
	 * and receive properties in control->lzma_properties */

	lzma_ret = LzmaCompress(c_buf, &dlen, s->buf, (size_t)s->buflen, control.lzma_properties, &prop_size, control.compression_level,
			      0, /* dict size. set default */
			      -1, -1, -1, -1, /* lc, lp, pb, fb */
			      control.threads);
	if (lzma_ret != SZ_OK) {
		switch (lzma_ret) {
			case SZ_ERROR_MEM:
				err_msg("\nLZMA ERROR: %d. Try a smaller compression window.\n", SZ_ERROR_MEM);
				break;
			case SZ_ERROR_PARAM:
				err_msg("\nLZMA Parameter ERROR: %d. This should not happen.\n", SZ_ERROR_PARAM);
				break;
			case SZ_ERROR_OUTPUT_EOF:
				err_msg("\nHarmless LZMA Output Buffer Overflow error: %d. Incompressible block.\n", SZ_ERROR_OUTPUT_EOF);
				break;
			case SZ_ERROR_THREAD:
				err_msg("\nLZMA Multi Thread ERROR: %d. This should not happen.\n", SZ_ERROR_THREAD);
				break;
			default:
				err_msg("Unidentified LZMA ERROR: %d. This should not happen.\n", lzma_ret);
				break;
		}
		/* can pass -1 if not compressible! Thanks Lasse Collin */
		free(c_buf);
		goto out;
	}
	if ((i64)dlen >= *c_len) {
		/* Incompressible, leave as CTYPE_NONE */
		free(c_buf);
		goto out;
	}

	*c_len = dlen;
	free(s->buf);
	s->buf = c_buf;
	*c_type = CTYPE_LZMA;
out:
	if (control.flags & FLAG_VERBOSITY_MAX)
		fprintf(control.msgout, "\n");
	else if ((control.flags & FLAG_SHOW_PROGRESS || control.flags & FLAG_VERBOSITY ))
		fprintf(control.msgout, "\r\t                                                                                      \r");
	fflush(control.msgout);
}