コード例 #1
0
void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_num)
{
    auto & view = views[view_num];

    try
    {
        BlockInputStreamPtr from = std::make_shared<OneBlockInputStream>(block);
        InterpreterSelectQuery select(view.query, *views_context, from);
        BlockInputStreamPtr in = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
        /// Squashing is needed here because the materialized view query can generate a lot of blocks
        /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
        /// and two-level aggregation is triggered).
        in = std::make_shared<SquashingBlockInputStream>(
            in, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes);

        in->readPrefix();

        while (Block result_block = in->read())
        {
            Nested::validateArraySizes(result_block);
            view.out->write(result_block);
        }

        in->readSuffix();
    }
    catch (Exception & ex)
    {
        ex.addMessage("while pushing to view " + backQuoteIfNeed(view.database) + "." + backQuoteIfNeed(view.table));
        throw;
    }
}
コード例 #2
0
NamesAndTypesList getStructureOfRemoteTable(
    const Cluster & cluster,
    const std::string & database,
    const std::string & table,
    const Context & context)
{
    /// Запрос на описание таблицы
    String query = "DESC TABLE " + backQuoteIfNeed(database) + "." + backQuoteIfNeed(table);
    Settings settings = context.getSettings();
    NamesAndTypesList res;

    /// Отправляем на первый попавшийся удалённый шард.
    const auto & shard_info = cluster.getAnyShardInfo();

    if (shard_info.isLocal())
        return context.getTable(database, table)->getColumnsList();

    ConnectionPoolPtr pool = shard_info.pool;

    BlockInputStreamPtr input =
        std::make_shared<RemoteBlockInputStream>(
            pool.get(), query, &settings, nullptr,
            Tables(), QueryProcessingStage::Complete, context);
    input->readPrefix();

    const DataTypeFactory & data_type_factory = DataTypeFactory::instance();

    while (Block current = input->read())
    {
        ColumnPtr name = current.getByName("name").column;
        ColumnPtr type = current.getByName("type").column;
        size_t size = name->size();

        for (size_t i = 0; i < size; ++i)
        {
            String column_name = (*name)[i].get<const String &>();
            String data_type_name = (*type)[i].get<const String &>();

            res.emplace_back(column_name, data_type_factory.get(data_type_name));
        }
    }

    return res;
}
コード例 #3
0
void inputThread(BlockInputStreamPtr in, BlockOutputStreamPtr out, WriteBuffer & wb, std::mutex & mutex)
{
	while (Block block = in->read())
	{
		std::lock_guard<std::mutex> lock(mutex);

		out->write(block);
		wb.next();
	}
}
コード例 #4
0
void StorageMerge::convertingSourceStream(const Block & header, const Context & context, ASTPtr & query,
                                          BlockInputStreamPtr & source_stream, QueryProcessingStage::Enum processed_stage)
{
    Block before_block_header = source_stream->getHeader();
    source_stream = std::make_shared<ConvertingBlockInputStream>(context, source_stream, header, ConvertingBlockInputStream::MatchColumnsMode::Name);

    ASTPtr where_expression = typeid_cast<ASTSelectQuery *>(query.get())->where_expression;

    if (!where_expression)
        return;

    for (size_t column_index : ext::range(0, header.columns()))
    {
        ColumnWithTypeAndName header_column = header.getByPosition(column_index);
        ColumnWithTypeAndName before_column = before_block_header.getByName(header_column.name);
        /// If the processed_stage greater than FetchColumns and the block structure between streams is different.
        /// the where expression maybe invalid because of convertingBlockInputStream.
        /// So we need to throw exception.
        if (!header_column.type->equals(*before_column.type.get()) && processed_stage > QueryProcessingStage::FetchColumns)
        {
            NamesAndTypesList source_columns = getSampleBlock().getNamesAndTypesList();
            NameAndTypePair virtual_column = getColumn("_table");
            source_columns.insert(source_columns.end(), virtual_column);
            auto syntax_result = SyntaxAnalyzer(context).analyze(where_expression, source_columns);
            ExpressionActionsPtr actions = ExpressionAnalyzer{where_expression, syntax_result, context}.getActions(false, false);
            Names required_columns = actions->getRequiredColumns();

            for (const auto & required_column : required_columns)
            {
                if (required_column == header_column.name)
                    throw Exception("Block structure mismatch in Merge Storage: different types:\n" + before_block_header.dumpStructure()
                                    + "\n" + header.dumpStructure(), ErrorCodes::BLOCKS_HAVE_DIFFERENT_STRUCTURE);
            }
        }

    }
}
コード例 #5
0
SquashingBlockInputStream::SquashingBlockInputStream(
    const BlockInputStreamPtr & src, size_t min_block_size_rows, size_t min_block_size_bytes)
    : header(src->getHeader()), transform(min_block_size_rows, min_block_size_bytes)
{
    children.emplace_back(src);
}
コード例 #6
0
int main(int argc, char ** argv)
{
    using namespace DB;

    try
    {
        size_t n = argc == 2 ? atoi(argv[1]) : 10;

        Block block;

        ColumnWithTypeAndName column_x;
        column_x.name = "x";
        column_x.type = std::make_shared<DataTypeInt16>();
        auto x = std::make_shared<ColumnInt16>();
        column_x.column = x;
        auto & vec_x = x->getData();

        vec_x.resize(n);
        for (size_t i = 0; i < n; ++i)
            vec_x[i] = i % 9;

        block.insert(column_x);

        const char * strings[] = {"abc", "def", "abcd", "defg", "ac"};

        ColumnWithTypeAndName column_s1;
        column_s1.name = "s1";
        column_s1.type = std::make_shared<DataTypeString>();
        column_s1.column = std::make_shared<ColumnString>();

        for (size_t i = 0; i < n; ++i)
            column_s1.column->insert(std::string(strings[i % 5]));

        block.insert(column_s1);

        ColumnWithTypeAndName column_s2;
        column_s2.name = "s2";
        column_s2.type = std::make_shared<DataTypeString>();
        column_s2.column = std::make_shared<ColumnString>();

        for (size_t i = 0; i < n; ++i)
            column_s2.column->insert(std::string(strings[i % 3]));

        block.insert(column_s2);

        Names key_column_names;
        key_column_names.emplace_back("x");

        AggregateFunctionFactory factory;

        AggregateDescriptions aggregate_descriptions(1);

        DataTypes empty_list_of_types;
        aggregate_descriptions[0].function = factory.get("count", empty_list_of_types);

        DataTypes result_types
        {
            std::make_shared<DataTypeInt16>(),
        //    std::make_shared<DataTypeString>(),
            std::make_shared<DataTypeUInt64>(),
        };

        Block sample;
        for (DataTypes::const_iterator it = result_types.begin(); it != result_types.end(); ++it)
        {
            ColumnWithTypeAndName col;
            col.type = *it;
            sample.insert(std::move(col));
        }

        Aggregator::Params params(key_column_names, aggregate_descriptions, false);

        BlockInputStreamPtr stream = std::make_shared<OneBlockInputStream>(block);
        stream = std::make_shared<AggregatingBlockInputStream>(stream, params, true);

        WriteBufferFromOStream ob(std::cout);
        RowOutputStreamPtr row_out = std::make_shared<TabSeparatedRowOutputStream>(ob, sample);
        BlockOutputStreamPtr out = std::make_shared<BlockOutputStreamFromRowOutputStream>(row_out);

        {
            Stopwatch stopwatch;
            stopwatch.start();

            copyData(*stream, *out);

            stopwatch.stop();
            std::cout << std::fixed << std::setprecision(2)
                << "Elapsed " << stopwatch.elapsedSeconds() << " sec."
                << ", " << n / stopwatch.elapsedSeconds() << " rows/sec."
                << std::endl;
        }

        std::cout << std::endl;
        stream->dumpTree(std::cout);
        std::cout << std::endl;
    }
    catch (const Exception & e)
    {
        std::cerr << e.displayText() << std::endl;
    }

    return 0;
}