void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_num)
{
auto & view = views[view_num];
try
{
BlockInputStreamPtr from = std::make_shared<OneBlockInputStream>(block);
InterpreterSelectQuery select(view.query, *views_context, from);
BlockInputStreamPtr in = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
/// Squashing is needed here because the materialized view query can generate a lot of blocks
/// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
/// and two-level aggregation is triggered).
in = std::make_shared<SquashingBlockInputStream>(
in, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes);
in->readPrefix();
while (Block result_block = in->read())
{
Nested::validateArraySizes(result_block);
view.out->write(result_block);
}
in->readSuffix();
}
catch (Exception & ex)
{
ex.addMessage("while pushing to view " + backQuoteIfNeed(view.database) + "." + backQuoteIfNeed(view.table));
throw;
}
}
NamesAndTypesList getStructureOfRemoteTable(
const Cluster & cluster,
const std::string & database,
const std::string & table,
const Context & context)
{
/// Запрос на описание таблицы
String query = "DESC TABLE " + backQuoteIfNeed(database) + "." + backQuoteIfNeed(table);
Settings settings = context.getSettings();
NamesAndTypesList res;
/// Отправляем на первый попавшийся удалённый шард.
const auto & shard_info = cluster.getAnyShardInfo();
if (shard_info.isLocal())
return context.getTable(database, table)->getColumnsList();
ConnectionPoolPtr pool = shard_info.pool;
BlockInputStreamPtr input =
std::make_shared<RemoteBlockInputStream>(
pool.get(), query, &settings, nullptr,
Tables(), QueryProcessingStage::Complete, context);
input->readPrefix();
const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
while (Block current = input->read())
{
ColumnPtr name = current.getByName("name").column;
ColumnPtr type = current.getByName("type").column;
size_t size = name->size();
for (size_t i = 0; i < size; ++i)
{
String column_name = (*name)[i].get<const String &>();
String data_type_name = (*type)[i].get<const String &>();
res.emplace_back(column_name, data_type_factory.get(data_type_name));
}
}
return res;
}
void inputThread(BlockInputStreamPtr in, BlockOutputStreamPtr out, WriteBuffer & wb, std::mutex & mutex)
{
while (Block block = in->read())
{
std::lock_guard<std::mutex> lock(mutex);
out->write(block);
wb.next();
}
}
void StorageMerge::convertingSourceStream(const Block & header, const Context & context, ASTPtr & query,
BlockInputStreamPtr & source_stream, QueryProcessingStage::Enum processed_stage)
{
Block before_block_header = source_stream->getHeader();
source_stream = std::make_shared<ConvertingBlockInputStream>(context, source_stream, header, ConvertingBlockInputStream::MatchColumnsMode::Name);
ASTPtr where_expression = typeid_cast<ASTSelectQuery *>(query.get())->where_expression;
if (!where_expression)
return;
for (size_t column_index : ext::range(0, header.columns()))
{
ColumnWithTypeAndName header_column = header.getByPosition(column_index);
ColumnWithTypeAndName before_column = before_block_header.getByName(header_column.name);
/// If the processed_stage greater than FetchColumns and the block structure between streams is different.
/// the where expression maybe invalid because of convertingBlockInputStream.
/// So we need to throw exception.
if (!header_column.type->equals(*before_column.type.get()) && processed_stage > QueryProcessingStage::FetchColumns)
{
NamesAndTypesList source_columns = getSampleBlock().getNamesAndTypesList();
NameAndTypePair virtual_column = getColumn("_table");
source_columns.insert(source_columns.end(), virtual_column);
auto syntax_result = SyntaxAnalyzer(context).analyze(where_expression, source_columns);
ExpressionActionsPtr actions = ExpressionAnalyzer{where_expression, syntax_result, context}.getActions(false, false);
Names required_columns = actions->getRequiredColumns();
for (const auto & required_column : required_columns)
{
if (required_column == header_column.name)
throw Exception("Block structure mismatch in Merge Storage: different types:\n" + before_block_header.dumpStructure()
+ "\n" + header.dumpStructure(), ErrorCodes::BLOCKS_HAVE_DIFFERENT_STRUCTURE);
}
}
}
}
SquashingBlockInputStream::SquashingBlockInputStream(
const BlockInputStreamPtr & src, size_t min_block_size_rows, size_t min_block_size_bytes)
: header(src->getHeader()), transform(min_block_size_rows, min_block_size_bytes)
{
children.emplace_back(src);
}
int main(int argc, char ** argv)
{
using namespace DB;
try
{
size_t n = argc == 2 ? atoi(argv[1]) : 10;
Block block;
ColumnWithTypeAndName column_x;
column_x.name = "x";
column_x.type = std::make_shared<DataTypeInt16>();
auto x = std::make_shared<ColumnInt16>();
column_x.column = x;
auto & vec_x = x->getData();
vec_x.resize(n);
for (size_t i = 0; i < n; ++i)
vec_x[i] = i % 9;
block.insert(column_x);
const char * strings[] = {"abc", "def", "abcd", "defg", "ac"};
ColumnWithTypeAndName column_s1;
column_s1.name = "s1";
column_s1.type = std::make_shared<DataTypeString>();
column_s1.column = std::make_shared<ColumnString>();
for (size_t i = 0; i < n; ++i)
column_s1.column->insert(std::string(strings[i % 5]));
block.insert(column_s1);
ColumnWithTypeAndName column_s2;
column_s2.name = "s2";
column_s2.type = std::make_shared<DataTypeString>();
column_s2.column = std::make_shared<ColumnString>();
for (size_t i = 0; i < n; ++i)
column_s2.column->insert(std::string(strings[i % 3]));
block.insert(column_s2);
Names key_column_names;
key_column_names.emplace_back("x");
AggregateFunctionFactory factory;
AggregateDescriptions aggregate_descriptions(1);
DataTypes empty_list_of_types;
aggregate_descriptions[0].function = factory.get("count", empty_list_of_types);
DataTypes result_types
{
std::make_shared<DataTypeInt16>(),
// std::make_shared<DataTypeString>(),
std::make_shared<DataTypeUInt64>(),
};
Block sample;
for (DataTypes::const_iterator it = result_types.begin(); it != result_types.end(); ++it)
{
ColumnWithTypeAndName col;
col.type = *it;
sample.insert(std::move(col));
}
Aggregator::Params params(key_column_names, aggregate_descriptions, false);
BlockInputStreamPtr stream = std::make_shared<OneBlockInputStream>(block);
stream = std::make_shared<AggregatingBlockInputStream>(stream, params, true);
WriteBufferFromOStream ob(std::cout);
RowOutputStreamPtr row_out = std::make_shared<TabSeparatedRowOutputStream>(ob, sample);
BlockOutputStreamPtr out = std::make_shared<BlockOutputStreamFromRowOutputStream>(row_out);
{
Stopwatch stopwatch;
stopwatch.start();
copyData(*stream, *out);
stopwatch.stop();
std::cout << std::fixed << std::setprecision(2)
<< "Elapsed " << stopwatch.elapsedSeconds() << " sec."
<< ", " << n / stopwatch.elapsedSeconds() << " rows/sec."
<< std::endl;
}
std::cout << std::endl;
stream->dumpTree(std::cout);
std::cout << std::endl;
}
catch (const Exception & e)
{
std::cerr << e.displayText() << std::endl;
}
return 0;
}