int main(int argc, char *argv[])
{
QString workDir;
#ifdef Q_OS_MAC
if (QDir(QString()).absolutePath() == "/") {
QString first = argc ? QString::fromLocal8Bit(argv[0]) : QString();
if (!first.isEmpty()) {
QFileInfo info(first);
if (info.exists()) {
QDir result(info.absolutePath() + "/../../..");
workDir = result.absolutePath() + '/';
}
}
}
#endif
QString remove;
int version = 0;
QFileInfoList files;
for (int i = 0; i < argc; ++i) {
if (string("-path") == argv[i] && i + 1 < argc) {
QString path = workDir + QString(argv[i + 1]);
QFileInfo info(path);
files.push_back(info);
if (remove.isEmpty()) remove = info.canonicalPath() + "/";
} else if (string("-version") == argv[i] && i + 1 < argc) {
version = QString(argv[i + 1]).toInt();
} else if (string("-dev") == argv[i]) {
DevChannel = true;
} else if (string("-beta") == argv[i] && i + 1 < argc) {
BetaVersion = QString(argv[i + 1]).toULongLong();
if (BetaVersion > version * 1000ULL && BetaVersion < (version + 1) * 1000ULL) {
DevChannel = false;
BetaSignature = countBetaVersionSignature(BetaVersion);
if (BetaSignature.isEmpty()) {
return -1;
}
} else {
cout << "Bad -beta param value passed, should be for the same version: " << version << ", beta: " << BetaVersion << "\n";
return -1;
}
}
}
if (files.isEmpty() || remove.isEmpty() || version <= 1016 || version > 999999999) {
#ifdef Q_OS_WIN
cout << "Usage: Packer.exe -path {file} -version {version} OR Packer.exe -path {dir} -version {version}\n";
#elif defined Q_OS_MAC
cout << "Usage: Packer.app -path {file} -version {version} OR Packer.app -path {dir} -version {version}\n";
#else
cout << "Usage: Packer -path {file} -version {version} OR Packer -path {dir} -version {version}\n";
#endif
return -1;
}
bool hasDirs = true;
while (hasDirs) {
hasDirs = false;
for (QFileInfoList::iterator i = files.begin(); i != files.end(); ++i) {
QFileInfo info(*i);
QString fullPath = info.canonicalFilePath();
if (info.isDir()) {
hasDirs = true;
files.erase(i);
QDir d = QDir(info.absoluteFilePath());
QString fullDir = d.canonicalPath();
QStringList entries = d.entryList(QDir::Files | QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot);
files.append(d.entryInfoList(QDir::Files | QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot));
break;
} else if (!info.isReadable()) {
cout << "Can't read: " << info.absoluteFilePath().toUtf8().constData() << "\n";
return -1;
} else if (info.isHidden()) {
hasDirs = true;
files.erase(i);
break;
}
}
}
for (QFileInfoList::iterator i = files.begin(); i != files.end(); ++i) {
QFileInfo info(*i);
if (!info.canonicalFilePath().startsWith(remove)) {
cout << "Can't find '" << remove.toUtf8().constData() << "' in file '" << info.canonicalFilePath().toUtf8().constData() << "' :(\n";
return -1;
}
}
QByteArray result;
{
QBuffer buffer(&result);
buffer.open(QIODevice::WriteOnly);
QDataStream stream(&buffer);
stream.setVersion(QDataStream::Qt_5_1);
if (BetaVersion) {
stream << quint32(0x7FFFFFFF);
stream << quint64(BetaVersion);
} else {
stream << quint32(version);
}
stream << quint32(files.size());
cout << "Found " << files.size() << " file" << (files.size() == 1 ? "" : "s") << "..\n";
for (QFileInfoList::iterator i = files.begin(); i != files.end(); ++i) {
QFileInfo info(*i);
QString fullName = info.canonicalFilePath();
QString name = fullName.mid(remove.length());
cout << name.toUtf8().constData() << " (" << info.size() << ")\n";
QFile f(fullName);
if (!f.open(QIODevice::ReadOnly)) {
cout << "Can't open '" << fullName.toUtf8().constData() << "' for read..\n";
return -1;
}
QByteArray inner = f.readAll();
stream << name << quint32(inner.size()) << inner;
#if defined Q_OS_MAC || defined Q_OS_LINUX
stream << (QFileInfo(fullName).isExecutable() ? true : false);
#endif
}
if (stream.status() != QDataStream::Ok) {
cout << "Stream status is bad: " << stream.status() << "\n";
return -1;
}
}
int32 resultSize = result.size();
cout << "Compression start, size: " << resultSize << "\n";
QByteArray compressed, resultCheck;
#ifdef Q_OS_WIN // use Lzma SDK for win
const int32 hSigLen = 128, hShaLen = 20, hPropsLen = LZMA_PROPS_SIZE, hOriginalSizeLen = sizeof(int32), hSize = hSigLen + hShaLen + hPropsLen + hOriginalSizeLen; // header
compressed.resize(hSize + resultSize + 1024 * 1024); // rsa signature + sha1 + lzma props + max compressed size
size_t compressedLen = compressed.size() - hSize;
size_t outPropsSize = LZMA_PROPS_SIZE;
uchar *_dest = (uchar*)(compressed.data() + hSize);
size_t *_destLen = &compressedLen;
const uchar *_src = (const uchar*)(result.constData());
size_t _srcLen = result.size();
uchar *_outProps = (uchar*)(compressed.data() + hSigLen + hShaLen);
int res = LzmaCompress(_dest, _destLen, _src, _srcLen, _outProps, &outPropsSize, 9, 64 * 1024 * 1024, 4, 0, 2, 273, 2);
if (res != SZ_OK) {
cout << "Error in compression: " << res << "\n";
return -1;
}
compressed.resize(int(hSize + compressedLen));
memcpy(compressed.data() + hSigLen + hShaLen + hPropsLen, &resultSize, hOriginalSizeLen);
cout << "Compressed to size: " << compressedLen << "\n";
cout << "Checking uncompressed..\n";
int32 resultCheckLen;
memcpy(&resultCheckLen, compressed.constData() + hSigLen + hShaLen + hPropsLen, hOriginalSizeLen);
if (resultCheckLen <= 0 || resultCheckLen > 1024 * 1024 * 1024) {
cout << "Bad result len: " << resultCheckLen << "\n";
return -1;
}
resultCheck.resize(resultCheckLen);
size_t resultLen = resultCheck.size();
SizeT srcLen = compressedLen;
int uncompressRes = LzmaUncompress((uchar*)resultCheck.data(), &resultLen, (const uchar*)(compressed.constData() + hSize), &srcLen, (const uchar*)(compressed.constData() + hSigLen + hShaLen), LZMA_PROPS_SIZE);
if (uncompressRes != SZ_OK) {
cout << "Uncompress failed: " << uncompressRes << "\n";
return -1;
}
if (resultLen != size_t(result.size())) {
cout << "Uncompress bad size: " << resultLen << ", was: " << result.size() << "\n";
return -1;
}
#else // use liblzma for others
const int32 hSigLen = 128, hShaLen = 20, hPropsLen = 0, hOriginalSizeLen = sizeof(int32), hSize = hSigLen + hShaLen + hOriginalSizeLen; // header
compressed.resize(hSize + resultSize + 1024 * 1024); // rsa signature + sha1 + lzma props + max compressed size
size_t compressedLen = compressed.size() - hSize;
lzma_stream stream = LZMA_STREAM_INIT;
int preset = 9 | LZMA_PRESET_EXTREME;
lzma_ret ret = lzma_easy_encoder(&stream, preset, LZMA_CHECK_CRC64);
if (ret != LZMA_OK) {
const char *msg;
switch (ret) {
case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break;
case LZMA_OPTIONS_ERROR: msg = "Specified preset is not supported"; break;
case LZMA_UNSUPPORTED_CHECK: msg = "Specified integrity check is not supported"; break;
default: msg = "Unknown error, possibly a bug"; break;
}
cout << "Error initializing the encoder: " << msg << " (error code " << ret << ")\n";
return -1;
}
stream.avail_in = resultSize;
stream.next_in = (uint8_t*)result.constData();
stream.avail_out = compressedLen;
stream.next_out = (uint8_t*)(compressed.data() + hSize);
lzma_ret res = lzma_code(&stream, LZMA_FINISH);
compressedLen -= stream.avail_out;
lzma_end(&stream);
if (res != LZMA_OK && res != LZMA_STREAM_END) {
const char *msg;
switch (res) {
case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break;
case LZMA_DATA_ERROR: msg = "File size limits exceeded"; break;
default: msg = "Unknown error, possibly a bug"; break;
}
cout << "Error in compression: " << msg << " (error code " << res << ")\n";
return -1;
}
compressed.resize(int(hSize + compressedLen));
memcpy(compressed.data() + hSigLen + hShaLen, &resultSize, hOriginalSizeLen);
cout << "Compressed to size: " << compressedLen << "\n";
cout << "Checking uncompressed..\n";
int32 resultCheckLen;
memcpy(&resultCheckLen, compressed.constData() + hSigLen + hShaLen, hOriginalSizeLen);
if (resultCheckLen <= 0 || resultCheckLen > 1024 * 1024 * 1024) {
cout << "Bad result len: " << resultCheckLen << "\n";
return -1;
}
resultCheck.resize(resultCheckLen);
size_t resultLen = resultCheck.size();
stream = LZMA_STREAM_INIT;
ret = lzma_stream_decoder(&stream, UINT64_MAX, LZMA_CONCATENATED);
if (ret != LZMA_OK) {
const char *msg;
switch (ret) {
case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break;
case LZMA_OPTIONS_ERROR: msg = "Specified preset is not supported"; break;
case LZMA_UNSUPPORTED_CHECK: msg = "Specified integrity check is not supported"; break;
default: msg = "Unknown error, possibly a bug"; break;
}
cout << "Error initializing the decoder: " << msg << " (error code " << ret << ")\n";
return -1;
}
stream.avail_in = compressedLen;
stream.next_in = (uint8_t*)(compressed.constData() + hSize);
stream.avail_out = resultLen;
stream.next_out = (uint8_t*)resultCheck.data();
res = lzma_code(&stream, LZMA_FINISH);
if (stream.avail_in) {
cout << "Error in decompression, " << stream.avail_in << " bytes left in _in of " << compressedLen << " whole.\n";
return -1;
} else if (stream.avail_out) {
cout << "Error in decompression, " << stream.avail_out << " bytes free left in _out of " << resultLen << " whole.\n";
return -1;
}
lzma_end(&stream);
if (res != LZMA_OK && res != LZMA_STREAM_END) {
const char *msg;
switch (res) {
case LZMA_MEM_ERROR: msg = "Memory allocation failed"; break;
case LZMA_FORMAT_ERROR: msg = "The input data is not in the .xz format"; break;
case LZMA_OPTIONS_ERROR: msg = "Unsupported compression options"; break;
case LZMA_DATA_ERROR: msg = "Compressed file is corrupt"; break;
case LZMA_BUF_ERROR: msg = "Compressed data is truncated or otherwise corrupt"; break;
default: msg = "Unknown error, possibly a bug"; break;
}
cout << "Error in decompression: " << msg << " (error code " << res << ")\n";
return -1;
}
#endif
if (memcmp(result.constData(), resultCheck.constData(), resultLen)) {
cout << "Data differ :(\n";
return -1;
}
/**/
result = resultCheck = QByteArray();
cout << "Counting SHA1 hash..\n";
uchar sha1Buffer[20];
memcpy(compressed.data() + hSigLen, hashSha1(compressed.constData() + hSigLen + hShaLen, uint32(compressedLen + hPropsLen + hOriginalSizeLen), sha1Buffer), hShaLen); // count sha1
uint32 siglen = 0;
cout << "Signing..\n";
RSA *prKey = PEM_read_bio_RSAPrivateKey(BIO_new_mem_buf(const_cast<char*>((DevChannel || BetaVersion) ? PrivateDevKey : PrivateKey), -1), 0, 0, 0);
if (!prKey) {
cout << "Could not read RSA private key!\n";
return -1;
}
if (RSA_size(prKey) != hSigLen) {
cout << "Bad private key, size: " << RSA_size(prKey) << "\n";
RSA_free(prKey);
return -1;
}
if (RSA_sign(NID_sha1, (const uchar*)(compressed.constData() + hSigLen), hShaLen, (uchar*)(compressed.data()), &siglen, prKey) != 1) { // count signature
cout << "Signing failed!\n";
RSA_free(prKey);
return -1;
}
RSA_free(prKey);
if (siglen != hSigLen) {
cout << "Bad signature length: " << siglen << "\n";
return -1;
}
cout << "Checking signature..\n";
RSA *pbKey = PEM_read_bio_RSAPublicKey(BIO_new_mem_buf(const_cast<char*>((DevChannel || BetaVersion) ? PublicDevKey : PublicKey), -1), 0, 0, 0);
if (!pbKey) {
cout << "Could not read RSA public key!\n";
return -1;
}
if (RSA_verify(NID_sha1, (const uchar*)(compressed.constData() + hSigLen), hShaLen, (const uchar*)(compressed.constData()), siglen, pbKey) != 1) { // verify signature
RSA_free(pbKey);
cout << "Signature verification failed!\n";
return -1;
}
cout << "Signature verified!\n";
RSA_free(pbKey);
#ifdef Q_OS_WIN
QString outName(QString("tupdate%1").arg(BetaVersion ? BetaVersion : version));
#elif defined Q_OS_MAC
QString outName(QString("tmacupd%1").arg(BetaVersion ? BetaVersion : version));
#elif defined Q_OS_LINUX32
QString outName(QString("tlinux32upd%1").arg(BetaVersion ? BetaVersion : version));
#elif defined Q_OS_LINUX64
QString outName(QString("tlinuxupd%1").arg(BetaVersion ? BetaVersion : version));
#else
#error Unknown platform!
#endif
if (BetaVersion) {
outName += "_" + BetaSignature;
}
QFile out(outName);
if (!out.open(QIODevice::WriteOnly)) {
cout << "Can't open '" << outName.toUtf8().constData() << "' for write..\n";
return -1;
}
out.write(compressed);
out.close();
if (BetaVersion) {
QString keyName(QString("tbeta_%1_key").arg(BetaVersion));
QFile key(keyName);
if (!key.open(QIODevice::WriteOnly)) {
cout << "Can't open '" << keyName.toUtf8().constData() << "' for write..\n";
return -1;
}
key.write(BetaSignature.toUtf8());
key.close();
}
cout << "Update file '" << outName.toUtf8().constData() << "' written successfully!\n";
return 0;
}
bool LocateConverter::QQWryToLocate(const wchar_t *qqwry, const wchar_t *locate, bool compress/* = false */, OnProgress progress/* = NULL */)
{
QQWryReader QQWry(qqwry);
if ( !QQWry.IsAvailable() ) return false;
StringTable string_table1;
StringTable string_table2;
RecordTable record_table;
Buffer buffer;
uint32_t last_begin_ip = 0;
uint32_t last_end_ip = 0;
for(uint32_t i=0; i<QQWry.GetInfo()->count; i++)
{
LocateItem *item = QQWry.GetItem(i + 1);
LOCATE record;
record.begin_ip = item->begin_ip;
record.table1 = string_table1.Append(item->region);
record.table2 = string_table2.Append(item->address);
if ( i > 0 )
{
if ( record.begin_ip - last_end_ip != 1 )
{
//printf("逻辑错误，IP段不连续！ 行:%d\n", i + 1);
//getchar();
return false;
}
uint32_t diff = record.begin_ip - last_begin_ip;
if ( compress && isPowerOf2(diff) )
{
record.begin_ip = LogBase2(diff);
}
}
record_table.Append(&record);
last_begin_ip = item->begin_ip;
last_end_ip = item->end_ip;
}
//合并数据区
Buffer *record_table_buffer = record_table;
Buffer *string_table1_buffer = string_table1;
Buffer *string_table2_buffer = string_table2;
std::copy(record_table_buffer->begin(), record_table_buffer->end(), std::back_inserter(buffer));
std::copy(string_table1_buffer->begin(), string_table1_buffer->end(), std::back_inserter(buffer));
std::copy(string_table2_buffer->begin(), string_table2_buffer->end(), std::back_inserter(buffer));
//生成文件头
HEADER header;
header.magic = LOCATE_MAGIC;
header.version = LOCATE_VERISON;
header.compress = compress?1:0;
header.total = QQWry.GetInfo()->count;
header.time = QQWry.GetInfo()->time;
header.table1 = sizeof(header) + record_table_buffer->size(); // 这里不加LZMA_PROPS_SIZE的原因是解压后，抛弃props信息
header.table2 = header.table1 + string_table1_buffer->size();
header.size = buffer.size();
header.crc32 = CRC32_MEM((uint8_t*)&header, sizeof(header) - 4);
uint32_t lzma_buffer_len = buffer.size();
uint8_t *lzma_buffer = 0;
size_t prop_size = LZMA_PROPS_SIZE;
BYTE outProps[LZMA_PROPS_SIZE];
//准备压缩
if(compress)
{
lzma_buffer = (uint8_t *)malloc(lzma_buffer_len);
ProgressCallback LzmaCompressProgress;
LzmaCompressProgress.Progress = LzmaOnProgress;
LzmaCompressProgress.totalInSize = buffer.size();
LzmaCompressProgress.progress = progress;
LzmaCompress(lzma_buffer, &lzma_buffer_len, &buffer[0], buffer.size(), (ICompressProgress*)&LzmaCompressProgress, outProps, &prop_size, 5, 1<<27, 8, 0, 2, 64, 4);
}
//保存文件
FILE * out = _wfopen(locate, L"wb");
fwrite(&header, 1, sizeof(header), out);
if(compress)
{
fwrite(outProps, 1, sizeof(outProps), out);
fwrite(lzma_buffer, 1, lzma_buffer_len, out);
}
else
{
fwrite(&buffer[0], 1, buffer.size(), out);
}
fclose(out);
if(compress)
{
free(lzma_buffer);
}
return true;
}
bool LocateConverter::GeneratePatch(const wchar_t *qqwry1_path, const wchar_t *qqwry2_path)
{
QQWryReader QQWry1(qqwry1_path);
QQWryReader QQWry2(qqwry2_path);
if ( !QQWry1.IsAvailable() ) return false;
if ( !QQWry2.IsAvailable() ) return false;
if(QQWry1.GetInfo()->time==QQWry2.GetInfo()->time) return false;
// 根据时间交换，低版本在前
QQWryReader *qqwry1 = &QQWry1;
QQWryReader *qqwry2 = &QQWry2;
if(QQWry1.GetInfo()->time>QQWry2.GetInfo()->time)
{
QQWryReader *temp = qqwry1;
qqwry1 = qqwry2;
qqwry2 = temp;
}
uint32_t i = 0;
uint32_t j = 0;
uint8_t flag = 0;
uint32_t n = 0;
uint32_t m = 0;
//int x = 1;
StringTable string_table1;
StringTable string_table2;
Buffer RecordBuffer; // 记录
Buffer ResultBuffer; // 最终结果
while(i<qqwry1->GetInfo()->count && j<qqwry2->GetInfo()->count)
{
LocateItem *Item1 = qqwry1->GetItem(i + 1);
LocateItem *Item2 = qqwry2->GetItem(j + 1);
if( Item1->begin_ip != Item2->begin_ip )
{
if( Item1->begin_ip < Item2->begin_ip ) i++;
else j++;
flag = 1;
continue;
}
if( Item1->end_ip != Item2->end_ip )
{
if( Item1->end_ip < Item2->end_ip ) i++;
else j++;
flag = 1;
continue;
}
if(flag==1)
{
flag = 0;
//printf("%d-%d,%d-%d\n",n+1,i,m+1,j);
for(uint32_t k=0; k<i-n; k++)
{
DIFFITEM record;
record.line = n + 1 + k;
record.method = REMOVE;
record.begin_ip = 0;
record.table1 = 0;
record.table2 = 0;
AppendBuffer(RecordBuffer, (uint8_t*)&record, sizeof(record));
//printf("%d: 删除 %d\n",x++,n+1+k);
}
for(uint32_t l=0; l<j-m; l++)
{
Item2 = qqwry2->GetItem(m + l + 1);
DIFFITEM record;
record.line = i;
record.method = INSERT;
record.begin_ip = Item2->begin_ip;
record.table1 = string_table1.Append(Item2->region);
record.table2 = string_table2.Append(Item2->address);
AppendBuffer(RecordBuffer, (uint8_t*)&record, sizeof(record));
//printf("%d: 添加 (%d) -> %d\n", x++, m+1+k, i);
}
}
Item1 = qqwry1->GetItem(i + 1);
Item2 = qqwry2->GetItem(j + 1);
if( strcmp(Item1->region, Item2->region) || strcmp(Item1->address, Item2->address) )
{
DIFFITEM record;
record.line = i + 1;
record.method = MODIFY;
record.begin_ip = 0;
record.table1 = string_table1.Append(Item2->region);
record.table2 = string_table2.Append(Item2->address);
AppendBuffer(RecordBuffer, (uint8_t*)&record, sizeof(record));
//printf("%d: 修改 (%d) -> %d\n", x++, j+1, i+1);
}
i++;
j++;
n = i;
m = j;
}
//合并数据区
Buffer *string_table1_buffer = string_table1;
Buffer *string_table2_buffer = string_table2;
std::copy(RecordBuffer.begin(), RecordBuffer.end(), std::back_inserter(ResultBuffer));
std::copy(string_table1_buffer->begin(), string_table1_buffer->end(), std::back_inserter(ResultBuffer));
std::copy(string_table2_buffer->begin(), string_table2_buffer->end(), std::back_inserter(ResultBuffer));
//生成文件头
DIFFHEADER header;
header.magic = DIFF_MAGIC;
header.total1 = qqwry1->GetInfo()->count;
header.total2 = qqwry2->GetInfo()->count;
header.time1 = qqwry1->GetInfo()->time;
header.time2 = qqwry2->GetInfo()->time;
header.table1 = sizeof(header) + RecordBuffer.size();;
header.table2 = header.table1 + string_table1_buffer->size();
header.size = ResultBuffer.size();
header.crc32 = CRC32_MEM((uint8_t*)&header, sizeof(header) - 4);
//准备压缩
uint32_t lzma_buffer_len = ResultBuffer.size();
uint8_t *lzma_buffer = (uint8_t *)malloc(lzma_buffer_len);
size_t prop_size = LZMA_PROPS_SIZE;
BYTE outProps[LZMA_PROPS_SIZE];
LzmaCompress(lzma_buffer, &lzma_buffer_len, &ResultBuffer[0], ResultBuffer.size(), NULL, outProps, &prop_size, 5, 1<<27, 8, 0, 2, 64, 4);
//保存文件
char temp[1024];
sprintf(temp,"%d-%d.dif",qqwry1->GetInfo()->time,qqwry2->GetInfo()->time);
FILE * out = fopen(temp, "wb");
fwrite(&header, 1, sizeof(header), out);
fwrite(outProps, 1, sizeof(outProps), out);
fwrite(lzma_buffer, 1, lzma_buffer_len, out);
//fwrite(&ResultBuffer[0], 1, ResultBuffer.size(), out);
fclose(out);
free(lzma_buffer);
return true;
}
bool LocateConverter::LocateUpdate(const wchar_t *locate, const wchar_t *patch, const wchar_t *path, bool compress/* = false */, OnProgress progress/* = NULL */)
{
LocateReader loc(locate);
if ( !loc.IsAvailable() ) return false;
const uint8_t *diff_buffer = 0;
PDIFFHEADER diff_header = 0;
HANDLE hfile = CreateFileW(patch, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if(hfile!=INVALID_HANDLE_VALUE)
{
uint32_t file_length = GetFileSize(hfile, NULL);
HANDLE hfilemap = CreateFileMapping(hfile, NULL, PAGE_READONLY, 0, 0, NULL);
CloseHandle(hfile);
diff_buffer = (const uint8_t*) MapViewOfFile(hfilemap, FILE_MAP_READ, 0, 0, 0);
CloseHandle(hfilemap);
diff_header = (PDIFFHEADER)diff_buffer;
int data_size = file_length - sizeof(DIFFHEADER) - LZMA_PROPS_SIZE;
if( data_size<0 || diff_header->magic != DIFF_MAGIC || CRC32_MEM(diff_buffer, sizeof(DIFFHEADER) - 4)!=diff_header->crc32 )
{
UnmapViewOfFile((void*)diff_buffer);
return false;
}
else
{
uint32_t lzma_buffer_len = diff_header->size;
uint8_t *lzma_buffer = (uint8_t *)malloc(sizeof(DIFFHEADER) + lzma_buffer_len);
if( LzmaUncompress(lzma_buffer + sizeof(DIFFHEADER), &lzma_buffer_len, (unsigned char*)diff_header->data + LZMA_PROPS_SIZE, (uint32_t*)&data_size, (unsigned char*)diff_header->data, LZMA_PROPS_SIZE)==SZ_OK )
{
memcpy(lzma_buffer, diff_buffer, sizeof(DIFFHEADER));
UnmapViewOfFile((void*)diff_buffer);
diff_buffer = lzma_buffer;
diff_header = (PDIFFHEADER)diff_buffer;
diff_header->table1 += (uint32_t)diff_buffer;
diff_header->table2 += (uint32_t)diff_buffer;
}
else
{
free(lzma_buffer);
UnmapViewOfFile((void*)diff_buffer);
return false;
}
}
}
else
{
return false;
}
if ( loc.GetInfo()->count!=diff_header->total1 || loc.GetInfo()->time!=diff_header->time1 ) return false;
PDIFFITEM diffitem = diff_header->data;
LocateItem *Locate = (LocateItem *)malloc( diff_header->total2 * sizeof(LocateItem) );
uint32_t last_diff = ( diff_header->table1 - sizeof(DIFFHEADER) - (uint32_t)diff_buffer ) / sizeof(DIFFITEM) - 1;
uint32_t last_line = diff_header->total2 - 1;
uint32_t i = loc.GetInfo()->count;
for(; i>0; i--)
{
LocateItem *item = loc.GetItem(i);
if(i!=diffitem[last_diff].line)
{
Locate[last_line].begin_ip = item->begin_ip;
Locate[last_line].region = item->region;
Locate[last_line].address = item->address;
last_line--;
}
else
{
switch(diffitem[last_diff].method)
{
case INSERT:
//printf("INSERT %d %d\n", i, diffitem[last_diff-1].line);
Locate[last_line].begin_ip = diffitem[last_diff].begin_ip;
Locate[last_line].region = (const char*)( diffitem[last_diff].table1 + diff_header->table1 );
Locate[last_line].address = (const char*)( diffitem[last_diff].table2 + diff_header->table2 );
last_line--;
i++;
break;
case REMOVE:
//printf("REMOVE %d %d %d\n", i, diffitem[last_diff-1].line, diffitem[last_diff-2].line);
break;
case MODIFY:
Locate[last_line].begin_ip = item->begin_ip;
Locate[last_line].region = (const char*)( diffitem[last_diff].table1 + diff_header->table1 );
Locate[last_line].address = (const char*)( diffitem[last_diff].table2 + diff_header->table2 );
//printf("MODIFY %d %s %s\n", last_line+1, Locate[last_line].region, Locate[last_line].address);
//getchar();
last_line--;
break;
}
last_diff--;
}
}
//printf("%d %d\n",last_diff,last_line);
if ( last_diff!=-1 || last_line!=-1 ) return false;
StringTable string_table1;
StringTable string_table2;
RecordTable record_table;
Buffer buffer;
uint32_t last_begin_ip = 0;
for(i=last_line+1; i<diff_header->total2; i++)
{
LocateItem *item = &Locate[i];
LOCATE record;
record.begin_ip = item->begin_ip;
record.table1 = string_table1.Append(item->region);
record.table2 = string_table2.Append(item->address);
if ( i > 0 )
{
uint32_t diff = record.begin_ip - last_begin_ip;
if ( compress && isPowerOf2(diff) )
{
record.begin_ip = LogBase2(diff);
}
}
record_table.Append(&record);
last_begin_ip = item->begin_ip;
}
free(Locate);
//合并数据区
Buffer *record_table_buffer = record_table;
Buffer *string_table1_buffer = string_table1;
Buffer *string_table2_buffer = string_table2;
std::copy(record_table_buffer->begin(), record_table_buffer->end(), std::back_inserter(buffer));
std::copy(string_table1_buffer->begin(), string_table1_buffer->end(), std::back_inserter(buffer));
std::copy(string_table2_buffer->begin(), string_table2_buffer->end(), std::back_inserter(buffer));
//生成文件头
HEADER header;
header.magic = LOCATE_MAGIC;
header.version = LOCATE_VERISON;
header.compress = compress?1:0;
header.total = diff_header->total2;
header.time = diff_header->time2;
header.table1 = sizeof(header) + record_table_buffer->size(); // 这里不加LZMA_PROPS_SIZE的原因是解压后，抛弃props信息
header.table2 = header.table1 + string_table1_buffer->size();
header.size = buffer.size();
header.crc32 = CRC32_MEM((uint8_t*)&header, sizeof(header) - 4);
uint32_t lzma_buffer_len = buffer.size();
uint8_t *lzma_buffer = 0;
size_t prop_size = LZMA_PROPS_SIZE;
BYTE outProps[LZMA_PROPS_SIZE];
//准备压缩
if(compress)
{
lzma_buffer = (uint8_t *)malloc(lzma_buffer_len);
ProgressCallback LzmaCompressProgress;
LzmaCompressProgress.Progress = LzmaOnProgress;
LzmaCompressProgress.totalInSize = buffer.size();
LzmaCompressProgress.progress = progress;
LzmaCompress(lzma_buffer, &lzma_buffer_len, &buffer[0], buffer.size(), (ICompressProgress*)&LzmaCompressProgress, outProps, &prop_size, 5, 1<<27, 8, 0, 2, 64, 4);
}
//保存文件
FILE * out = _wfopen(path, L"wb");
fwrite(&header, 1, sizeof(header), out);
if(compress)
{
fwrite(outProps, 1, sizeof(outProps), out);
fwrite(lzma_buffer, 1, lzma_buffer_len, out);
}
else
{
fwrite(&buffer[0], 1, buffer.size(), out);
}
fclose(out);
if(compress)
{
free(lzma_buffer);
}
free((void*)diff_buffer);
return true;
}
int main(int argc, char **argv) {
if(argc < 2) {
printf("Usage: %s infile\n", argv[0]);
exit(1);
}
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info mpi_info = MPI_INFO_NULL;
MPI_File fh, fw;
MPI_Offset file_size, frag_size, read_size;
MPI_Offset offset;
MPI_Status status;
int retval;
double start, end;
unsigned char *buf, *outbuf, *outProps;
size_t destlen;
size_t propsize = 5;
MPI_Init(&argc, &argv);
MPI_Comm_rank(comm, &mpi_rank);
MPI_Comm_size(comm, &mpi_size);
MPI_Barrier(comm);
start = MPI_Wtime();
/*
* read
*/
MPI_File_open(comm, argv[1], MPI_MODE_RDONLY, mpi_info, &fh);
MPI_File_get_size(fh, &file_size);
//printf("file size:%d\n", file_size);
frag_size = file_size / mpi_size;
offset = frag_size * mpi_rank;
read_size = MIN(frag_size, file_size - offset);
//printf("rank %d offset %d\n", mpi_rank, offset);
buf = malloc(frag_size + 2);
assert(buf != NULL);
MPI_File_open(comm, argv[1], MPI_MODE_RDONLY, mpi_info, &fh);
MPI_File_read_at(fh, offset, buf, read_size, MPI_CHAR, &status);
MPI_File_close(&fh);
/*
* compress
*/
destlen = 1.2 * frag_size + 1024 * 1024;
outbuf = (unsigned char *)malloc(destlen);
assert(outbuf != NULL);
destlen = destlen - DATA_OFFSET -propsize;
outProps = outbuf + DATA_OFFSET;
retval = LzmaCompress(outbuf + DATA_OFFSET + propsize, &destlen, buf, read_size, outProps, &propsize, -1, 0, -1, -1, -1, -1, 1);
if(retval != SZ_OK) {
error_print(retval);
free(buf);
free(outbuf);
exit(1);
}
/*
* write
*/
char *fwname;
unsigned long long *len;
fwname = get_fwname(argv[1]);
len = (unsigned long long *)outbuf;
*len = read_size;
//printf("%s %d\n", fwname, destlen);
MPI_File_open(MPI_COMM_SELF, fwname, MPI_MODE_WRONLY | MPI_MODE_CREATE, mpi_info, &fw);
MPI_File_set_size(fw, destlen);
MPI_File_write(fw, outbuf, destlen + DATA_OFFSET + propsize, MPI_CHAR, &status);
MPI_File_close(&fw);
MPI_Barrier(comm);
end = MPI_Wtime();
size_t cmprs_len;
double cmprs_ratio;
MPI_Reduce(&destlen, &cmprs_len, 1, MPI_UNSIGNED_LONG, MPI_SUM, 0, comm);
if(0 == mpi_rank) {
cmprs_ratio = (double)cmprs_len / file_size;
printf("file size: %lu\n", file_size);
printf("after compressed: %lu\n", cmprs_len);
printf("compress ratio: %f\n", cmprs_ratio);
printf("number of processes: %d\n", mpi_size);
printf("time used: %fs\n", end - start);
}
MPI_Finalize();
free(fwname);
free(buf);
free(outbuf);
return 0;
}
static void lzma_compress_buf(struct stream *s, int *c_type, i64 *c_len)
{
uchar *c_buf;
size_t dlen;
size_t prop_size = 5; /* return value for lzma_properties */
int lzma_ret;
if (!lzo_compresses(s))
goto out;
dlen = s->buflen;
c_buf = malloc(dlen);
if (!c_buf)
return;
if (control.flags & FLAG_SHOW_PROGRESS) {
fprintf(control.msgout, "\tProgress percentage pausing during lzma compression...");
fflush(control.msgout);
}
/* with LZMA SDK 4.63, we pass compression level and threads only
* and receive properties in control->lzma_properties */
lzma_ret = LzmaCompress(c_buf, &dlen, s->buf, (size_t)s->buflen, control.lzma_properties, &prop_size, control.compression_level,
0, /* dict size. set default */
-1, -1, -1, -1, /* lc, lp, pb, fb */
control.threads);
if (lzma_ret != SZ_OK) {
switch (lzma_ret) {
case SZ_ERROR_MEM:
err_msg("\nLZMA ERROR: %d. Try a smaller compression window.\n", SZ_ERROR_MEM);
break;
case SZ_ERROR_PARAM:
err_msg("\nLZMA Parameter ERROR: %d. This should not happen.\n", SZ_ERROR_PARAM);
break;
case SZ_ERROR_OUTPUT_EOF:
err_msg("\nHarmless LZMA Output Buffer Overflow error: %d. Incompressible block.\n", SZ_ERROR_OUTPUT_EOF);
break;
case SZ_ERROR_THREAD:
err_msg("\nLZMA Multi Thread ERROR: %d. This should not happen.\n", SZ_ERROR_THREAD);
break;
default:
err_msg("Unidentified LZMA ERROR: %d. This should not happen.\n", lzma_ret);
break;
}
/* can pass -1 if not compressible! Thanks Lasse Collin */
free(c_buf);
goto out;
}
if ((i64)dlen >= *c_len) {
/* Incompressible, leave as CTYPE_NONE */
free(c_buf);
goto out;
}
*c_len = dlen;
free(s->buf);
s->buf = c_buf;
*c_type = CTYPE_LZMA;
out:
if (control.flags & FLAG_VERBOSITY_MAX)
fprintf(control.msgout, "\n");
else if ((control.flags & FLAG_SHOW_PROGRESS || control.flags & FLAG_VERBOSITY ))
fprintf(control.msgout, "\r\t \r");
fflush(control.msgout);
}
