// --------------------------------------------------------------------
void rma_receiver::read_chunk_list()
{
for (chunk_struct &c : chunks_)
{
if (c.type_ == serialization::chunk_type_pointer)
{
region_type *get_region =
memory_pool_->allocate_region(c.size_);
LOG_TRACE_MSG(
CRC32_MEM(get_region->get_address(), c.size_,
"(RDMA GET region (new))"));
rma_regions_.push_back(get_region);
get_region->set_message_length(c.size_);
// overwrite the serialization chunk data pointer because the chunk
// info sent contains the pointer to the remote data and when we
// deocde the parcel we want the chunk to point to the local copy of it
const void *remoteAddr = c.data_.cpos_;
c.data_.cpos_ = get_region->get_address();
// call the rma read function for the chunk
read_one_chunk(src_addr_, get_region, remoteAddr, c.rkey_);
}
}
}
// --------------------------------------------------------------------
void rma_receiver::read_one_chunk(
fi_addr_t src_addr, region_type *get_region,
const void *remoteAddr, uint64_t rkey)
{
// post the rdma read/get
LOG_DEBUG_MSG("rma_receiver " << hexpointer(this)
<< "RDMA Get fi_read :"
<< "client " << hexpointer(endpoint_)
<< "fi_addr " << hexpointer(src_addr_)
<< "tag " << hexuint64(header_->tag())
<< "local addr " << hexpointer(get_region->get_address())
<< "local desc " << hexpointer(get_region->get_desc())
<< "size " << hexlength(get_region->get_message_length())
<< "rkey " << hexpointer(rkey)
<< "remote cpos " << hexpointer(remoteAddr));
// count reads
++rma_reads_;
ssize_t ret = 0;
for (std::size_t k = 0; true; ++k)
{
LOG_EXCLUSIVE(
// write a pattern and dump out data for debugging purposes
uint32_t *buffer =
reinterpret_cast<uint32_t*>(get_region->get_address());
std::fill(buffer, buffer + get_region->get_size()/4, 0xDEADC0DE);
LOG_TRACE_MSG(
CRC32_MEM(get_region->get_address(), c.size_,
"(RDMA GET region (pre-fi_read))"));
);
ret = fi_read(endpoint_,
get_region->get_address(),
get_region->get_message_length(),
get_region->get_desc(),
src_addr_,
(uint64_t)(remoteAddr), rkey, this);
if (ret == -FI_EAGAIN)
{
LOG_ERROR_MSG("receiver " << hexpointer(this)
<< "reposting fi_read...\n");
hpx::util::detail::yield_k(k, "libfabric::receiver::async_read");
continue;
}
if (ret) throw fabric_error(ret, "fi_read");
break;
}
bool LocateConverter::GeneratePatch(const wchar_t *qqwry1_path, const wchar_t *qqwry2_path)
{
QQWryReader QQWry1(qqwry1_path);
QQWryReader QQWry2(qqwry2_path);
if ( !QQWry1.IsAvailable() ) return false;
if ( !QQWry2.IsAvailable() ) return false;
if(QQWry1.GetInfo()->time==QQWry2.GetInfo()->time) return false;
// 根据时间交换，低版本在前
QQWryReader *qqwry1 = &QQWry1;
QQWryReader *qqwry2 = &QQWry2;
if(QQWry1.GetInfo()->time>QQWry2.GetInfo()->time)
{
QQWryReader *temp = qqwry1;
qqwry1 = qqwry2;
qqwry2 = temp;
}
uint32_t i = 0;
uint32_t j = 0;
uint8_t flag = 0;
uint32_t n = 0;
uint32_t m = 0;
//int x = 1;
StringTable string_table1;
StringTable string_table2;
Buffer RecordBuffer; // 记录
Buffer ResultBuffer; // 最终结果
while(i<qqwry1->GetInfo()->count && j<qqwry2->GetInfo()->count)
{
LocateItem *Item1 = qqwry1->GetItem(i + 1);
LocateItem *Item2 = qqwry2->GetItem(j + 1);
if( Item1->begin_ip != Item2->begin_ip )
{
if( Item1->begin_ip < Item2->begin_ip ) i++;
else j++;
flag = 1;
continue;
}
if( Item1->end_ip != Item2->end_ip )
{
if( Item1->end_ip < Item2->end_ip ) i++;
else j++;
flag = 1;
continue;
}
if(flag==1)
{
flag = 0;
//printf("%d-%d,%d-%d\n",n+1,i,m+1,j);
for(uint32_t k=0; k<i-n; k++)
{
DIFFITEM record;
record.line = n + 1 + k;
record.method = REMOVE;
record.begin_ip = 0;
record.table1 = 0;
record.table2 = 0;
AppendBuffer(RecordBuffer, (uint8_t*)&record, sizeof(record));
//printf("%d: 删除 %d\n",x++,n+1+k);
}
for(uint32_t l=0; l<j-m; l++)
{
Item2 = qqwry2->GetItem(m + l + 1);
DIFFITEM record;
record.line = i;
record.method = INSERT;
record.begin_ip = Item2->begin_ip;
record.table1 = string_table1.Append(Item2->region);
record.table2 = string_table2.Append(Item2->address);
AppendBuffer(RecordBuffer, (uint8_t*)&record, sizeof(record));
//printf("%d: 添加 (%d) -> %d\n", x++, m+1+k, i);
}
}
Item1 = qqwry1->GetItem(i + 1);
Item2 = qqwry2->GetItem(j + 1);
if( strcmp(Item1->region, Item2->region) || strcmp(Item1->address, Item2->address) )
{
DIFFITEM record;
record.line = i + 1;
record.method = MODIFY;
record.begin_ip = 0;
record.table1 = string_table1.Append(Item2->region);
record.table2 = string_table2.Append(Item2->address);
AppendBuffer(RecordBuffer, (uint8_t*)&record, sizeof(record));
//printf("%d: 修改 (%d) -> %d\n", x++, j+1, i+1);
}
i++;
j++;
n = i;
m = j;
}
//合并数据区
Buffer *string_table1_buffer = string_table1;
Buffer *string_table2_buffer = string_table2;
std::copy(RecordBuffer.begin(), RecordBuffer.end(), std::back_inserter(ResultBuffer));
std::copy(string_table1_buffer->begin(), string_table1_buffer->end(), std::back_inserter(ResultBuffer));
std::copy(string_table2_buffer->begin(), string_table2_buffer->end(), std::back_inserter(ResultBuffer));
//生成文件头
DIFFHEADER header;
header.magic = DIFF_MAGIC;
header.total1 = qqwry1->GetInfo()->count;
header.total2 = qqwry2->GetInfo()->count;
header.time1 = qqwry1->GetInfo()->time;
header.time2 = qqwry2->GetInfo()->time;
header.table1 = sizeof(header) + RecordBuffer.size();;
header.table2 = header.table1 + string_table1_buffer->size();
header.size = ResultBuffer.size();
header.crc32 = CRC32_MEM((uint8_t*)&header, sizeof(header) - 4);
//准备压缩
uint32_t lzma_buffer_len = ResultBuffer.size();
uint8_t *lzma_buffer = (uint8_t *)malloc(lzma_buffer_len);
size_t prop_size = LZMA_PROPS_SIZE;
BYTE outProps[LZMA_PROPS_SIZE];
LzmaCompress(lzma_buffer, &lzma_buffer_len, &ResultBuffer[0], ResultBuffer.size(), NULL, outProps, &prop_size, 5, 1<<27, 8, 0, 2, 64, 4);
//保存文件
char temp[1024];
sprintf(temp,"%d-%d.dif",qqwry1->GetInfo()->time,qqwry2->GetInfo()->time);
FILE * out = fopen(temp, "wb");
fwrite(&header, 1, sizeof(header), out);
fwrite(outProps, 1, sizeof(outProps), out);
fwrite(lzma_buffer, 1, lzma_buffer_len, out);
//fwrite(&ResultBuffer[0], 1, ResultBuffer.size(), out);
fclose(out);
free(lzma_buffer);
return true;
}
bool LocateConverter::QQWryToLocate(const wchar_t *qqwry, const wchar_t *locate, bool compress/* = false */, OnProgress progress/* = NULL */)
{
QQWryReader QQWry(qqwry);
if ( !QQWry.IsAvailable() ) return false;
StringTable string_table1;
StringTable string_table2;
RecordTable record_table;
Buffer buffer;
uint32_t last_begin_ip = 0;
uint32_t last_end_ip = 0;
for(uint32_t i=0; i<QQWry.GetInfo()->count; i++)
{
LocateItem *item = QQWry.GetItem(i + 1);
LOCATE record;
record.begin_ip = item->begin_ip;
record.table1 = string_table1.Append(item->region);
record.table2 = string_table2.Append(item->address);
if ( i > 0 )
{
if ( record.begin_ip - last_end_ip != 1 )
{
//printf("逻辑错误，IP段不连续！ 行:%d\n", i + 1);
//getchar();
return false;
}
uint32_t diff = record.begin_ip - last_begin_ip;
if ( compress && isPowerOf2(diff) )
{
record.begin_ip = LogBase2(diff);
}
}
record_table.Append(&record);
last_begin_ip = item->begin_ip;
last_end_ip = item->end_ip;
}
//合并数据区
Buffer *record_table_buffer = record_table;
Buffer *string_table1_buffer = string_table1;
Buffer *string_table2_buffer = string_table2;
std::copy(record_table_buffer->begin(), record_table_buffer->end(), std::back_inserter(buffer));
std::copy(string_table1_buffer->begin(), string_table1_buffer->end(), std::back_inserter(buffer));
std::copy(string_table2_buffer->begin(), string_table2_buffer->end(), std::back_inserter(buffer));
//生成文件头
HEADER header;
header.magic = LOCATE_MAGIC;
header.version = LOCATE_VERISON;
header.compress = compress?1:0;
header.total = QQWry.GetInfo()->count;
header.time = QQWry.GetInfo()->time;
header.table1 = sizeof(header) + record_table_buffer->size(); // 这里不加LZMA_PROPS_SIZE的原因是解压后，抛弃props信息
header.table2 = header.table1 + string_table1_buffer->size();
header.size = buffer.size();
header.crc32 = CRC32_MEM((uint8_t*)&header, sizeof(header) - 4);
uint32_t lzma_buffer_len = buffer.size();
uint8_t *lzma_buffer = 0;
size_t prop_size = LZMA_PROPS_SIZE;
BYTE outProps[LZMA_PROPS_SIZE];
//准备压缩
if(compress)
{
lzma_buffer = (uint8_t *)malloc(lzma_buffer_len);
ProgressCallback LzmaCompressProgress;
LzmaCompressProgress.Progress = LzmaOnProgress;
LzmaCompressProgress.totalInSize = buffer.size();
LzmaCompressProgress.progress = progress;
LzmaCompress(lzma_buffer, &lzma_buffer_len, &buffer[0], buffer.size(), (ICompressProgress*)&LzmaCompressProgress, outProps, &prop_size, 5, 1<<27, 8, 0, 2, 64, 4);
}
//保存文件
FILE * out = _wfopen(locate, L"wb");
fwrite(&header, 1, sizeof(header), out);
if(compress)
{
fwrite(outProps, 1, sizeof(outProps), out);
fwrite(lzma_buffer, 1, lzma_buffer_len, out);
}
else
{
fwrite(&buffer[0], 1, buffer.size(), out);
}
fclose(out);
if(compress)
{
free(lzma_buffer);
}
return true;
}
bool LocateConverter::LocateUpdate(const wchar_t *locate, const wchar_t *patch, const wchar_t *path, bool compress/* = false */, OnProgress progress/* = NULL */)
{
LocateReader loc(locate);
if ( !loc.IsAvailable() ) return false;
const uint8_t *diff_buffer = 0;
PDIFFHEADER diff_header = 0;
HANDLE hfile = CreateFileW(patch, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if(hfile!=INVALID_HANDLE_VALUE)
{
uint32_t file_length = GetFileSize(hfile, NULL);
HANDLE hfilemap = CreateFileMapping(hfile, NULL, PAGE_READONLY, 0, 0, NULL);
CloseHandle(hfile);
diff_buffer = (const uint8_t*) MapViewOfFile(hfilemap, FILE_MAP_READ, 0, 0, 0);
CloseHandle(hfilemap);
diff_header = (PDIFFHEADER)diff_buffer;
int data_size = file_length - sizeof(DIFFHEADER) - LZMA_PROPS_SIZE;
if( data_size<0 || diff_header->magic != DIFF_MAGIC || CRC32_MEM(diff_buffer, sizeof(DIFFHEADER) - 4)!=diff_header->crc32 )
{
UnmapViewOfFile((void*)diff_buffer);
return false;
}
else
{
uint32_t lzma_buffer_len = diff_header->size;
uint8_t *lzma_buffer = (uint8_t *)malloc(sizeof(DIFFHEADER) + lzma_buffer_len);
if( LzmaUncompress(lzma_buffer + sizeof(DIFFHEADER), &lzma_buffer_len, (unsigned char*)diff_header->data + LZMA_PROPS_SIZE, (uint32_t*)&data_size, (unsigned char*)diff_header->data, LZMA_PROPS_SIZE)==SZ_OK )
{
memcpy(lzma_buffer, diff_buffer, sizeof(DIFFHEADER));
UnmapViewOfFile((void*)diff_buffer);
diff_buffer = lzma_buffer;
diff_header = (PDIFFHEADER)diff_buffer;
diff_header->table1 += (uint32_t)diff_buffer;
diff_header->table2 += (uint32_t)diff_buffer;
}
else
{
free(lzma_buffer);
UnmapViewOfFile((void*)diff_buffer);
return false;
}
}
}
else
{
return false;
}
if ( loc.GetInfo()->count!=diff_header->total1 || loc.GetInfo()->time!=diff_header->time1 ) return false;
PDIFFITEM diffitem = diff_header->data;
LocateItem *Locate = (LocateItem *)malloc( diff_header->total2 * sizeof(LocateItem) );
uint32_t last_diff = ( diff_header->table1 - sizeof(DIFFHEADER) - (uint32_t)diff_buffer ) / sizeof(DIFFITEM) - 1;
uint32_t last_line = diff_header->total2 - 1;
uint32_t i = loc.GetInfo()->count;
for(; i>0; i--)
{
LocateItem *item = loc.GetItem(i);
if(i!=diffitem[last_diff].line)
{
Locate[last_line].begin_ip = item->begin_ip;
Locate[last_line].region = item->region;
Locate[last_line].address = item->address;
last_line--;
}
else
{
switch(diffitem[last_diff].method)
{
case INSERT:
//printf("INSERT %d %d\n", i, diffitem[last_diff-1].line);
Locate[last_line].begin_ip = diffitem[last_diff].begin_ip;
Locate[last_line].region = (const char*)( diffitem[last_diff].table1 + diff_header->table1 );
Locate[last_line].address = (const char*)( diffitem[last_diff].table2 + diff_header->table2 );
last_line--;
i++;
break;
case REMOVE:
//printf("REMOVE %d %d %d\n", i, diffitem[last_diff-1].line, diffitem[last_diff-2].line);
break;
case MODIFY:
Locate[last_line].begin_ip = item->begin_ip;
Locate[last_line].region = (const char*)( diffitem[last_diff].table1 + diff_header->table1 );
Locate[last_line].address = (const char*)( diffitem[last_diff].table2 + diff_header->table2 );
//printf("MODIFY %d %s %s\n", last_line+1, Locate[last_line].region, Locate[last_line].address);
//getchar();
last_line--;
break;
}
last_diff--;
}
}
//printf("%d %d\n",last_diff,last_line);
if ( last_diff!=-1 || last_line!=-1 ) return false;
StringTable string_table1;
StringTable string_table2;
RecordTable record_table;
Buffer buffer;
uint32_t last_begin_ip = 0;
for(i=last_line+1; i<diff_header->total2; i++)
{
LocateItem *item = &Locate[i];
LOCATE record;
record.begin_ip = item->begin_ip;
record.table1 = string_table1.Append(item->region);
record.table2 = string_table2.Append(item->address);
if ( i > 0 )
{
uint32_t diff = record.begin_ip - last_begin_ip;
if ( compress && isPowerOf2(diff) )
{
record.begin_ip = LogBase2(diff);
}
}
record_table.Append(&record);
last_begin_ip = item->begin_ip;
}
free(Locate);
//合并数据区
Buffer *record_table_buffer = record_table;
Buffer *string_table1_buffer = string_table1;
Buffer *string_table2_buffer = string_table2;
std::copy(record_table_buffer->begin(), record_table_buffer->end(), std::back_inserter(buffer));
std::copy(string_table1_buffer->begin(), string_table1_buffer->end(), std::back_inserter(buffer));
std::copy(string_table2_buffer->begin(), string_table2_buffer->end(), std::back_inserter(buffer));
//生成文件头
HEADER header;
header.magic = LOCATE_MAGIC;
header.version = LOCATE_VERISON;
header.compress = compress?1:0;
header.total = diff_header->total2;
header.time = diff_header->time2;
header.table1 = sizeof(header) + record_table_buffer->size(); // 这里不加LZMA_PROPS_SIZE的原因是解压后，抛弃props信息
header.table2 = header.table1 + string_table1_buffer->size();
header.size = buffer.size();
header.crc32 = CRC32_MEM((uint8_t*)&header, sizeof(header) - 4);
uint32_t lzma_buffer_len = buffer.size();
uint8_t *lzma_buffer = 0;
size_t prop_size = LZMA_PROPS_SIZE;
BYTE outProps[LZMA_PROPS_SIZE];
//准备压缩
if(compress)
{
lzma_buffer = (uint8_t *)malloc(lzma_buffer_len);
ProgressCallback LzmaCompressProgress;
LzmaCompressProgress.Progress = LzmaOnProgress;
LzmaCompressProgress.totalInSize = buffer.size();
LzmaCompressProgress.progress = progress;
LzmaCompress(lzma_buffer, &lzma_buffer_len, &buffer[0], buffer.size(), (ICompressProgress*)&LzmaCompressProgress, outProps, &prop_size, 5, 1<<27, 8, 0, 2, 64, 4);
}
//保存文件
FILE * out = _wfopen(path, L"wb");
fwrite(&header, 1, sizeof(header), out);
if(compress)
{
fwrite(outProps, 1, sizeof(outProps), out);
fwrite(lzma_buffer, 1, lzma_buffer_len, out);
}
else
{
fwrite(&buffer[0], 1, buffer.size(), out);
}
fclose(out);
if(compress)
{
free(lzma_buffer);
}
free((void*)diff_buffer);
return true;
}
