BlockIO InterpreterSelectWithUnionQuery::execute()
{
const Settings & settings = context.getSettingsRef();
BlockInputStreams nested_streams = executeWithMultipleStreams();
BlockInputStreamPtr result_stream;
if (nested_streams.empty())
{
result_stream = std::make_shared<NullBlockInputStream>(getSampleBlock());
}
else if (nested_streams.size() == 1)
{
result_stream = nested_streams.front();
nested_streams.clear();
}
else
{
result_stream = std::make_shared<UnionBlockInputStream>(nested_streams, nullptr, settings.max_threads);
nested_streams.clear();
}
BlockIO res;
res.in = result_stream;
return res;
}
BlockInputStreams StorageSystemEvents::read(
const Names & column_names,
const ASTPtr & query,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
const size_t max_block_size,
const unsigned threads)
{
check(column_names);
processed_stage = QueryProcessingStage::FetchColumns;
Block block = getSampleBlock();
for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
{
UInt64 value = ProfileEvents::counters[i];
if (0 != value)
{
block.getByPosition(0).column->insert(String(ProfileEvents::getDescription(ProfileEvents::Event(i))));
block.getByPosition(1).column->insert(value);
}
}
return BlockInputStreams(1, std::make_shared<OneBlockInputStream>(block));
}
BlockInputStreams StorageSystemTables::read(
const Names & column_names,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum /*processed_stage*/,
const size_t max_block_size,
const unsigned /*num_streams*/)
{
check(column_names);
/// Create a mask of what columns are needed in the result.
NameSet names_set(column_names.begin(), column_names.end());
Block sample_block = getSampleBlock();
Block res_block;
std::vector<UInt8> columns_mask(sample_block.columns());
for (size_t i = 0, size = columns_mask.size(); i < size; ++i)
{
if (names_set.count(sample_block.getByPosition(i).name))
{
columns_mask[i] = 1;
res_block.insert(sample_block.getByPosition(i));
}
}
ColumnPtr filtered_databases_column = getFilteredDatabases(query_info.query, context);
return {std::make_shared<TablesBlockInputStream>(
std::move(columns_mask), std::move(res_block), max_block_size, std::move(filtered_databases_column), context)};
}
void StorageJoin::truncate(const ASTPtr &)
{
Poco::File(path).remove(true);
Poco::File(path).createDirectories();
Poco::File(path + "tmp/").createDirectories();
increment = 0;
join = std::make_shared<Join>(key_names, key_names, NameSet(), false /* use_nulls */, SizeLimits(), kind, strictness);
join->setSampleBlock(getSampleBlock().sortColumns());
}
BlockInputStreams StorageSystemBuildOptions::read(
const Names & column_names,
const SelectQueryInfo &,
const Context &,
QueryProcessingStage::Enum & processed_stage,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/)
{
check(column_names);
processed_stage = QueryProcessingStage::FetchColumns;
MutableColumns res_columns = getSampleBlock().cloneEmptyColumns();
for (auto it = auto_config_build; *it; it += 2)
{
res_columns[0]->insert(String(it[0]));
res_columns[1]->insert(String(it[1]));
}
return BlockInputStreams(1, std::make_shared<OneBlockInputStream>(getSampleBlock().cloneWithColumns(std::move(res_columns))));
}
BlockIO InterpreterInsertQuery::execute()
{
ASTInsertQuery & query = typeid_cast<ASTInsertQuery &>(*query_ptr);
checkAccess(query);
StoragePtr table = getTable();
auto table_lock = table->lockStructure(true, __PRETTY_FUNCTION__);
NamesAndTypesList required_columns = table->getColumnsList();
/// We create a pipeline of several streams, into which we will write data.
BlockOutputStreamPtr out;
out = std::make_shared<PushingToViewsBlockOutputStream>(query.database, query.table, table, context, query_ptr, query.no_destination);
out = std::make_shared<MaterializingBlockOutputStream>(out);
out = std::make_shared<AddingDefaultBlockOutputStream>(
out, required_columns, table->column_defaults, context, static_cast<bool>(context.getSettingsRef().strict_insert_defaults));
if (!allow_materialized)
out = std::make_shared<ProhibitColumnsBlockOutputStream>(out, table->materialized_columns);
out = std::make_shared<SquashingBlockOutputStream>(
out, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes);
auto out_wrapper = std::make_shared<CountingBlockOutputStream>(out);
out_wrapper->setProcessListElement(context.getProcessListElement());
out = std::move(out_wrapper);
BlockIO res;
res.out_sample = getSampleBlock();
/// What type of query: INSERT or INSERT SELECT?
if (!query.select)
{
res.out = out;
}
else
{
InterpreterSelectQuery interpreter_select{query.select, context};
res.in_sample = interpreter_select.getSampleBlock();
res.in = interpreter_select.execute().in;
res.in = std::make_shared<NullableAdapterBlockInputStream>(res.in, res.in_sample, res.out_sample);
res.in = std::make_shared<CastTypeBlockInputStream>(context, res.in, res.out_sample);
res.in = std::make_shared<NullAndDoCopyBlockInputStream>(res.in, out);
}
return res;
}
void StorageSet::truncate(const ASTPtr &)
{
Poco::File(path).remove(true);
Poco::File(path).createDirectories();
Poco::File(path + "tmp/").createDirectories();
Block header = getSampleBlock();
header = header.sortColumns();
increment = 0;
set = std::make_shared<Set>(SizeLimits(), false);
set->setHeader(header);
}
StorageSet::StorageSet(
const String & path_,
const String & name_,
const ColumnsDescription & columns_)
: StorageSetOrJoinBase{path_, name_, columns_},
set(std::make_shared<Set>(SizeLimits(), false))
{
Block header = getSampleBlock();
header = header.sortColumns();
set->setHeader(header);
restore();
}
BlockInputStreamPtr InterpreterExplainQuery::executeImpl()
{
const ASTExplainQuery & ast = typeid_cast<const ASTExplainQuery &>(*query);
std::stringstream ss;
dumpAST(ast, ss);
Block sample_block = getSampleBlock();
MutableColumns res_columns = sample_block.cloneEmptyColumns();
res_columns[0]->insert(ss.str());
return std::make_shared<OneBlockInputStream>(sample_block.cloneWithColumns(std::move(res_columns)));
}
BlockInputStreams StorageSystemGraphite::read(
const Names & column_names,
const SelectQueryInfo &,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
size_t /*max_block_size*/,
unsigned /*num_streams*/)
{
check(column_names);
processed_stage = QueryProcessingStage::FetchColumns;
MutableColumns res_columns = getSampleBlock().cloneEmptyColumns();
const auto & config = context.getConfigRef();
Strings sections = getAllGraphiteSections(config);
for (const auto & section : sections)
{
const auto patterns = readPatterns(config, section);
for (const auto & pattern : patterns)
{
for (const auto & ret : pattern.retentions)
{
res_columns[0]->insert(Field(section));
res_columns[1]->insert(Field(pattern.regexp));
res_columns[2]->insert(Field(pattern.function));
res_columns[3]->insert(nearestFieldType(ret.age));
res_columns[4]->insert(nearestFieldType(ret.precision));
res_columns[5]->insert(nearestFieldType(pattern.priority));
res_columns[6]->insert(nearestFieldType(pattern.is_default));
}
}
}
return BlockInputStreams(1, std::make_shared<OneBlockInputStream>(getSampleBlock().cloneWithColumns(std::move(res_columns))));
}
BlockInputStreams StorageHDFS::read(
const Names & /*column_names*/,
const SelectQueryInfo & /*query_info*/,
const Context & context,
QueryProcessingStage::Enum /*processed_stage*/,
size_t max_block_size,
unsigned /*num_streams*/)
{
return {std::make_shared<StorageHDFSBlockInputStream>(
uri,
format_name,
getName(),
getSampleBlock(),
context,
max_block_size)};
}
StorageJoin::StorageJoin(
const String & path_,
const String & name_,
const Names & key_names_,
ASTTableJoin::Kind kind_, ASTTableJoin::Strictness strictness_,
const ColumnsDescription & columns_)
: StorageSetOrJoinBase{path_, name_, columns_},
key_names(key_names_), kind(kind_), strictness(strictness_)
{
for (const auto & key : key_names)
if (!getColumns().hasPhysical(key))
throw Exception{"Key column (" + key + ") does not exist in table declaration.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE};
/// NOTE StorageJoin doesn't use join_use_nulls setting.
join = std::make_shared<Join>(key_names, key_names, NameSet(), false /* use_nulls */, SizeLimits(), kind, strictness);
join->setSampleBlock(getSampleBlock().sortColumns());
restore();
}
BlockInputStreams StorageSystemMerges::read(
const Names & column_names,
ASTPtr query,
const Context & context,
const Settings & settings,
QueryProcessingStage::Enum & processed_stage,
const size_t max_block_size,
const unsigned)
{
check(column_names);
processed_stage = QueryProcessingStage::FetchColumns;
Block block = getSampleBlock();
for (const auto & merge : context.getMergeList().get())
{
size_t i = 0;
block.getByPosition(i++).column->insert(merge.database);
block.getByPosition(i++).column->insert(merge.table);
block.getByPosition(i++).column->insert(merge.elapsed);
block.getByPosition(i++).column->insert(std::min(1., merge.progress)); /// little cheat
block.getByPosition(i++).column->insert(merge.num_parts);
block.getByPosition(i++).column->insert(merge.source_part_names);
block.getByPosition(i++).column->insert(merge.result_part_name);
block.getByPosition(i++).column->insert(merge.total_size_bytes_compressed);
block.getByPosition(i++).column->insert(merge.total_size_marks);
block.getByPosition(i++).column->insert(merge.bytes_read_uncompressed);
block.getByPosition(i++).column->insert(merge.rows_read);
block.getByPosition(i++).column->insert(merge.bytes_written_uncompressed);
block.getByPosition(i++).column->insert(merge.rows_written);
block.getByPosition(i++).column->insert(merge.columns_written);
block.getByPosition(i++).column->insert(merge.memory_usage);
block.getByPosition(i++).column->insert(merge.thread_number);
}
return BlockInputStreams{1, std::make_shared<OneBlockInputStream>(block)};
}
void StorageMerge::convertingSourceStream(const Block & header, const Context & context, ASTPtr & query,
BlockInputStreamPtr & source_stream, QueryProcessingStage::Enum processed_stage)
{
Block before_block_header = source_stream->getHeader();
source_stream = std::make_shared<ConvertingBlockInputStream>(context, source_stream, header, ConvertingBlockInputStream::MatchColumnsMode::Name);
ASTPtr where_expression = typeid_cast<ASTSelectQuery *>(query.get())->where_expression;
if (!where_expression)
return;
for (size_t column_index : ext::range(0, header.columns()))
{
ColumnWithTypeAndName header_column = header.getByPosition(column_index);
ColumnWithTypeAndName before_column = before_block_header.getByName(header_column.name);
/// If the processed_stage greater than FetchColumns and the block structure between streams is different.
/// the where expression maybe invalid because of convertingBlockInputStream.
/// So we need to throw exception.
if (!header_column.type->equals(*before_column.type.get()) && processed_stage > QueryProcessingStage::FetchColumns)
{
NamesAndTypesList source_columns = getSampleBlock().getNamesAndTypesList();
NameAndTypePair virtual_column = getColumn("_table");
source_columns.insert(source_columns.end(), virtual_column);
auto syntax_result = SyntaxAnalyzer(context).analyze(where_expression, source_columns);
ExpressionActionsPtr actions = ExpressionAnalyzer{where_expression, syntax_result, context}.getActions(false, false);
Names required_columns = actions->getRequiredColumns();
for (const auto & required_column : required_columns)
{
if (required_column == header_column.name)
throw Exception("Block structure mismatch in Merge Storage: different types:\n" + before_block_header.dumpStructure()
+ "\n" + header.dumpStructure(), ErrorCodes::BLOCKS_HAVE_DIFFERENT_STRUCTURE);
}
}
}
}
BlockIO InterpreterInsertQuery::execute()
{
ASTInsertQuery & query = typeid_cast<ASTInsertQuery &>(*query_ptr);
StoragePtr table = getTable();
auto table_lock = table->lockStructure(true);
NamesAndTypesListPtr required_columns = std::make_shared<NamesAndTypesList>(table->getColumnsList());
/// Создаем кортеж из нескольких стримов, в которые будем писать данные.
BlockOutputStreamPtr out =
std::make_shared<ProhibitColumnsBlockOutputStream>(
std::make_shared<AddingDefaultBlockOutputStream>(
std::make_shared<MaterializingBlockOutputStream>(
std::make_shared<PushingToViewsBlockOutputStream>(query.database, query.table, context, query_ptr)),
required_columns, table->column_defaults, context, static_cast<bool>(context.getSettingsRef().strict_insert_defaults)),
table->materialized_columns);
BlockIO res;
res.out_sample = getSampleBlock();
/// Какой тип запроса: INSERT или INSERT SELECT?
if (!query.select)
{
res.out = out;
}
else
{
InterpreterSelectQuery interpreter_select{query.select, context};
BlockInputStreamPtr in{interpreter_select.execute().in};
res.in = std::make_shared<NullAndDoCopyBlockInputStream>(in, out);
res.in_sample = interpreter_select.getSampleBlock();
}
return res;
}
BlockInputStreams StorageSystemReplicationQueue::read(
const Names & column_names,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/)
{
check(column_names);
processed_stage = QueryProcessingStage::FetchColumns;
std::map<String, std::map<String, StoragePtr>> replicated_tables;
for (const auto & db : context.getDatabases())
for (auto iterator = db.second->getIterator(context); iterator->isValid(); iterator->next())
if (dynamic_cast<const StorageReplicatedMergeTree *>(iterator->table().get()))
replicated_tables[db.first][iterator->name()] = iterator->table();
MutableColumnPtr col_database_mut = ColumnString::create();
MutableColumnPtr col_table_mut = ColumnString::create();
for (auto & db : replicated_tables)
{
for (auto & table : db.second)
{
col_database_mut->insert(db.first);
col_table_mut->insert(table.first);
}
}
ColumnPtr col_database_to_filter = std::move(col_database_mut);
ColumnPtr col_table_to_filter = std::move(col_table_mut);
/// Determine what tables are needed by the conditions in the query.
{
Block filtered_block
{
{ col_database_to_filter, std::make_shared<DataTypeString>(), "database" },
{ col_table_to_filter, std::make_shared<DataTypeString>(), "table" },
};
VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context);
if (!filtered_block.rows())
return BlockInputStreams();
col_database_to_filter = filtered_block.getByName("database").column;
col_table_to_filter = filtered_block.getByName("table").column;
}
StorageReplicatedMergeTree::LogEntriesData queue;
String replica_name;
MutableColumns res_columns = getSampleBlock().cloneEmptyColumns();
for (size_t i = 0, tables_size = col_database_to_filter->size(); i < tables_size; ++i)
{
String database = (*col_database_to_filter)[i].safeGet<const String &>();
String table = (*col_table_to_filter)[i].safeGet<const String &>();
dynamic_cast<StorageReplicatedMergeTree &>(*replicated_tables[database][table]).getQueue(queue, replica_name);
for (size_t j = 0, queue_size = queue.size(); j < queue_size; ++j)
{
const auto & entry = queue[j];
Array parts_to_merge;
parts_to_merge.reserve(entry.source_parts.size());
for (const auto & name : entry.source_parts)
parts_to_merge.push_back(name);
size_t col_num = 0;
res_columns[col_num++]->insert(database);
res_columns[col_num++]->insert(table);
res_columns[col_num++]->insert(replica_name);
res_columns[col_num++]->insert(UInt64(j));
res_columns[col_num++]->insert(entry.znode_name);
res_columns[col_num++]->insert(entry.typeToString());
res_columns[col_num++]->insert(UInt64(entry.create_time));
res_columns[col_num++]->insert(UInt64(entry.quorum));
res_columns[col_num++]->insert(entry.source_replica);
res_columns[col_num++]->insert(entry.new_part_name);
res_columns[col_num++]->insert(parts_to_merge);
res_columns[col_num++]->insert(UInt64(entry.detach));
res_columns[col_num++]->insert(UInt64(entry.currently_executing));
res_columns[col_num++]->insert(UInt64(entry.num_tries));
res_columns[col_num++]->insert(entry.exception ? getExceptionMessage(entry.exception, false) : "");
res_columns[col_num++]->insert(UInt64(entry.last_attempt_time));
res_columns[col_num++]->insert(UInt64(entry.num_postponed));
res_columns[col_num++]->insert(entry.postpone_reason);
res_columns[col_num++]->insert(UInt64(entry.last_postpone_time));
}
}
return BlockInputStreams(1, std::make_shared<OneBlockInputStream>(getSampleBlock().cloneWithColumns(std::move(res_columns))));
}
BlockOutputStreamPtr IStorageURLBase::write(const ASTPtr & /*query*/, const Context & /*context*/)
{
return std::make_shared<StorageURLBlockOutputStream>(
uri, format_name, getSampleBlock(), context_global, ConnectionTimeouts::getHTTPTimeouts(context_global.getSettingsRef()));
}
/// AST to the list of columns with types. Columns of Nested type are expanded into a list of real columns.
static ColumnsAndDefaults parseColumns(const ASTExpressionList & column_list_ast, const Context & context)
{
/// list of table columns in correct order
NamesAndTypesList columns{};
ColumnDefaults defaults{};
/// Columns requiring type-deduction or default_expression type-check
std::vector<std::pair<NameAndTypePair *, ASTColumnDeclaration *>> defaulted_columns{};
/** all default_expressions as a single expression list,
* mixed with conversion-columns for each explicitly specified type */
ASTPtr default_expr_list = std::make_shared<ASTExpressionList>();
default_expr_list->children.reserve(column_list_ast.children.size());
for (const auto & ast : column_list_ast.children)
{
auto & col_decl = typeid_cast<ASTColumnDeclaration &>(*ast);
if (col_decl.type)
{
columns.emplace_back(col_decl.name, DataTypeFactory::instance().get(col_decl.type));
}
else
/// we're creating dummy DataTypeUInt8 in order to prevent the NullPointerException in ExpressionActions
columns.emplace_back(col_decl.name, std::make_shared<DataTypeUInt8>());
/// add column to postprocessing if there is a default_expression specified
if (col_decl.default_expression)
{
defaulted_columns.emplace_back(&columns.back(), &col_decl);
/** for columns with explicitly-specified type create two expressions:
* 1. default_expression aliased as column name with _tmp suffix
* 2. conversion of expression (1) to explicitly-specified type alias as column name */
if (col_decl.type)
{
const auto & final_column_name = col_decl.name;
const auto tmp_column_name = final_column_name + "_tmp";
const auto data_type_ptr = columns.back().type.get();
default_expr_list->children.emplace_back(setAlias(
makeASTFunction("CAST", std::make_shared<ASTIdentifier>(tmp_column_name),
std::make_shared<ASTLiteral>(Field(data_type_ptr->getName()))), final_column_name));
default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), tmp_column_name));
}
else
default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), col_decl.name));
}
}
/// set missing types and wrap default_expression's in a conversion-function if necessary
if (!defaulted_columns.empty())
{
const auto actions = ExpressionAnalyzer{default_expr_list, context, {}, columns}.getActions(true);
const auto block = actions->getSampleBlock();
for (auto & column : defaulted_columns)
{
const auto name_and_type_ptr = column.first;
const auto col_decl_ptr = column.second;
const auto & column_name = col_decl_ptr->name;
const auto has_explicit_type = nullptr != col_decl_ptr->type;
auto & explicit_type = name_and_type_ptr->type;
/// if column declaration contains explicit type, name_and_type_ptr->type is not null
if (has_explicit_type)
{
const auto & tmp_column = block.getByName(column_name + "_tmp");
const auto & deduced_type = tmp_column.type;
/// type mismatch between explicitly specified and deduced type, add conversion for non-array types
if (!explicit_type->equals(*deduced_type))
{
col_decl_ptr->default_expression = makeASTFunction("CAST", col_decl_ptr->default_expression,
std::make_shared<ASTLiteral>(explicit_type->getName()));
col_decl_ptr->children.clear();
col_decl_ptr->children.push_back(col_decl_ptr->type);
col_decl_ptr->children.push_back(col_decl_ptr->default_expression);
}
}
else
/// no explicit type, name_and_type_ptr->type is null, set to deduced type
explicit_type = block.getByName(column_name).type;
defaults.emplace(column_name, ColumnDefault{
columnDefaultKindFromString(col_decl_ptr->default_specifier),
col_decl_ptr->default_expression
});
}
}
return {Nested::flatten(columns), defaults};
}
