int
compress_buf2buf (BUFFER * in, BUFFER * out, long offset)
{
z_stream s;
int err;
char outbuf[4096];
if (compressed_buf (in, offset))
{
/* is already compressed */
add_to_buffer (out, in->message, in->length);
return 0;
}
add_to_buffer (out, in->message, offset);
s.zalloc = (alloc_func) 0;
s.zfree = (free_func) 0;
s.opaque = (voidpf) 0;
s.next_in = NULL;
sprintf (outbuf, "%c%c%c%c%c%c%c%c%c%c", gz_magic[0], gz_magic[1],
Z_DEFLATED, 0 /*flags */ , 0, 0, 0, 0 /*time */ , 0 /*xflags */ ,
3 /* Unix */ );
add_to_buffer (out, outbuf, 10);
if (deflateInit2 (&s, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -MAX_WBITS,
8, 0) != Z_OK)
return 0;
s.next_in = in->message + offset;
s.avail_in = in->length - offset;
s.next_out = outbuf;
s.avail_out = sizeof (outbuf);
while ((err = deflate (&s, Z_FINISH)) == Z_OK)
{
add_to_buffer (out, outbuf, sizeof (outbuf) - s.avail_out);
s.next_out = outbuf;
s.avail_out = sizeof (outbuf);
}
add_to_buffer (out, outbuf, sizeof (outbuf) - s.avail_out);
if (deflateEnd (&s) != Z_OK || err != Z_STREAM_END)
return 0;
return 1;
}
int
uncompress_buf2buf (BUFFER * in, BUFFER * out, long offset)
{
z_stream s;
char outbuf[4096];
byte *p;
int err;
if (!compressed_buf (in, offset))
return 0; /* not compressed */
add_to_buffer (out, in->message, offset);
s.zalloc = (alloc_func) 0;
s.zfree = (free_func) 0;
s.opaque = (voidpf) 0;
p = gzheader (in->message + offset, in->length);
if (p == NULL)
return -1;
s.next_in = p;
s.avail_in = in->length - (p - in->message);
s.next_out = NULL;
if (inflateInit2 (&s, -MAX_WBITS) != Z_OK)
return (-1);
s.next_out = outbuf;
s.avail_out = sizeof (outbuf);
while ((err = inflate (&s, Z_PARTIAL_FLUSH)) == Z_OK)
{
add_to_buffer (out, outbuf, sizeof (outbuf) - s.avail_out);
s.next_out = outbuf;
s.avail_out = sizeof (outbuf);
}
add_to_buffer (out, outbuf, sizeof (outbuf) - s.avail_out);
if ((err=inflateEnd (&s) != Z_OK))
{
fprintf (errlog, "Decompression error. %d\n",err);
return (-1);
}
return 1; /* compressed */
}
Block MergeSortingBlockInputStream::readImpl()
{
/** Algorithm:
* - read to memory blocks from source stream;
* - if too much of them and if external sorting is enabled,
* - merge all blocks to sorted stream and write it to temporary file;
* - at the end, merge all sorted streams from temporary files and also from rest of blocks in memory.
*/
/// If has not read source blocks.
if (!impl)
{
while (Block block = children.back()->read())
{
if (!sample_block)
{
sample_block = block.cloneEmpty();
removeConstantsFromSortDescription(sample_block, description);
}
/// If there were only const columns in sort description, then there is no need to sort.
/// Return the blocks as is.
if (description.empty())
return block;
removeConstantsFromBlock(block);
blocks.push_back(block);
sum_bytes_in_blocks += block.bytes();
/** If too much of them and if external sorting is enabled,
* will merge blocks that we have in memory at this moment and write merged stream to temporary (compressed) file.
* NOTE. It's possible to check free space in filesystem.
*/
if (max_bytes_before_external_sort && sum_bytes_in_blocks > max_bytes_before_external_sort)
{
temporary_files.emplace_back(new Poco::TemporaryFile(tmp_path));
const std::string & path = temporary_files.back()->path();
WriteBufferFromFile file_buf(path);
CompressedWriteBuffer compressed_buf(file_buf);
NativeBlockOutputStream block_out(compressed_buf);
MergeSortingBlocksBlockInputStream block_in(blocks, description, max_merged_block_size, limit);
LOG_INFO(log, "Sorting and writing part of data into temporary file " + path);
ProfileEvents::increment(ProfileEvents::ExternalSortWritePart);
copyData(block_in, block_out, &is_cancelled); /// NOTE. Possibly limit disk usage.
LOG_INFO(log, "Done writing part of data into temporary file " + path);
blocks.clear();
sum_bytes_in_blocks = 0;
}
}
if ((blocks.empty() && temporary_files.empty()) || isCancelled())
return Block();
if (temporary_files.empty())
{
impl = std::make_unique<MergeSortingBlocksBlockInputStream>(blocks, description, max_merged_block_size, limit);
}
else
{
/// If there was temporary files.
ProfileEvents::increment(ProfileEvents::ExternalSortMerge);
LOG_INFO(log, "There are " << temporary_files.size() << " temporary sorted parts to merge.");
/// Create sorted streams to merge.
for (const auto & file : temporary_files)
{
temporary_inputs.emplace_back(std::make_unique<TemporaryFileStream>(file->path()));
inputs_to_merge.emplace_back(temporary_inputs.back()->block_in);
}
/// Rest of blocks in memory.
if (!blocks.empty())
inputs_to_merge.emplace_back(std::make_shared<MergeSortingBlocksBlockInputStream>(blocks, description, max_merged_block_size, limit));
/// Will merge that sorted streams.
impl = std::make_unique<MergingSortedBlockInputStream>(inputs_to_merge, description, max_merged_block_size, limit);
}
}
Block res = impl->read();
if (res)
enrichBlockWithConstants(res, sample_block);
return res;
}
