void SQLiteObjectStorage::forward (RoutableMessageHeader&hdr, Protocol::Response&resp) {
String databuf;
resp.SerializeToString(&databuf);
MemoryReference membuf(databuf);
for (std::vector<MessageService*>::iterator i=mInterestedParties.begin(),ie=mInterestedParties.end();
i!=ie;
++i) {
(*i)->processMessage(hdr,membuf);
}
}
void split_and_sort(
const mem_chunk &available_mem_,
const file_id_t &src_file,
const file_id_t &dest_file,
std::deque<file_id_t> &transient_files)
{
mem_chunk input_mem;
mem_chunk available_mem;
available_mem_.split_at(4 * MiB, input_mem, available_mem);
parser<record_header2, record_header> input(input_mem, src_file);
input_file input2(src_file);
file_size_t threshold = input2.is_seekable() ? 1 * MiB : -1;
int segment_no = 0;
do {
mem_chunk output_mem;
mem_chunk membuf_mem;
available_mem.split_at(25 * MiB, output_mem, membuf_mem);
render_buf membuf(membuf_mem);
sort_element *vb, *ve;
vb = ve = reinterpret_cast<sort_element *>(membuf.get_free_mem().end());
/*
* Memory layout:
*
* DATA DATA DATA .... DATA -> FREE FREE FREE .... FREE <- P P P .... P
*/
while (input.is_header_valid()) {
size_t available_sz = membuf.get_free_mem().size();
size_t reserved_sz = (ve - vb + 1)*(sizeof *vb);
size_t body_sz;
record_header2 hd = input.get_header();
if (hd.body_size >= threshold) {
hd.is_body_present = 0;
body_sz = 0;
} else {
body_sz = hd.body_size;
}
if (available_sz < alignof(hd) + sizeof(hd) + body_sz + reserved_sz) {
break;
}
membuf.align(alignof(hd));
sort_element::init(*(--vb), membuf.put(hd));
if (hd.is_body_present) {
mem_chunk buf = membuf.get_free_mem();
input.read_body(buf);
membuf.write(buf);
}
input.parse_next();
}
std::sort(vb, ve);
bool is_final = (segment_no==0 && !input.is_header_valid());
file_id_t output_file_id;
if (is_final) {
output_file_id = dest_file;
} else {
output_file_id = file_id::create_temporary("yndx-xxlsort");
transient_files.push_back(output_file_id);
}
render_buf output(output_mem, output_file_id);
for (sort_element *i = vb; i != ve; i++) {
if (is_final) {
/* export public format (record_header) */
export_record(i->get_header(), output, input2);
} else {
/* write private extended format (record_header2) */
output.put(i->get_header());
}
output.write(i->get_body());
}
output.flush();
segment_no ++;
}
while (input.is_header_valid());
}
