Block StorageMerge::getQueryHeader( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum processed_stage) { switch (processed_stage) { case QueryProcessingStage::FetchColumns: { Block header = getSampleBlockForColumns(column_names); if (query_info.prewhere_info) { query_info.prewhere_info->prewhere_actions->execute(header); header = materializeBlock(header); if (query_info.prewhere_info->remove_prewhere_column) header.erase(query_info.prewhere_info->prewhere_column_name); } return header; } case QueryProcessingStage::WithMergeableState: case QueryProcessingStage::Complete: return materializeBlock(InterpreterSelectQuery( query_info.query, context, std::make_shared<OneBlockInputStream>(getSampleBlockForColumns(column_names)), processed_stage, true).getSampleBlock()); } throw Exception("Logical Error: unknown processed stage.", ErrorCodes::LOGICAL_ERROR); }
void Join::setSampleBlock(const Block & block) { std::unique_lock lock(rwlock); if (!empty()) return; size_t keys_size = key_names_right.size(); ColumnRawPtrs key_columns(keys_size); Columns materialized_columns(keys_size); for (size_t i = 0; i < keys_size; ++i) { materialized_columns[i] = recursiveRemoveLowCardinality(block.getByName(key_names_right[i]).column); key_columns[i] = materialized_columns[i].get(); /// We will join only keys, where all components are not NULL. if (key_columns[i]->isColumnNullable()) key_columns[i] = &static_cast<const ColumnNullable &>(*key_columns[i]).getNestedColumn(); } /// Choose data structure to use for JOIN. init(chooseMethod(key_columns, key_sizes)); sample_block_with_columns_to_add = materializeBlock(block); /// Move from `sample_block_with_columns_to_add` key columns to `sample_block_with_keys`, keeping the order. size_t pos = 0; while (pos < sample_block_with_columns_to_add.columns()) { const auto & name = sample_block_with_columns_to_add.getByPosition(pos).name; if (key_names_right.end() != std::find(key_names_right.begin(), key_names_right.end(), name)) { auto & col = sample_block_with_columns_to_add.getByPosition(pos); col.column = recursiveRemoveLowCardinality(col.column); col.type = recursiveRemoveLowCardinality(col.type); sample_block_with_keys.insert(col); sample_block_with_columns_to_add.erase(pos); } else ++pos; } size_t num_columns_to_add = sample_block_with_columns_to_add.columns(); for (size_t i = 0; i < num_columns_to_add; ++i) { auto & column = sample_block_with_columns_to_add.getByPosition(i); if (!column.column) column.column = column.type->createColumn(); } /// In case of LEFT and FULL joins, if use_nulls, convert joined columns to Nullable. if (use_nulls && (kind == ASTTableJoin::Kind::Left || kind == ASTTableJoin::Kind::Full)) for (size_t i = 0; i < num_columns_to_add; ++i) convertColumnToNullable(sample_block_with_columns_to_add.getByPosition(i)); }
NonJoinedBlockInputStream(const Join & parent_, const Block & left_sample_block, size_t max_block_size_) : parent(parent_), max_block_size(max_block_size_) { /** left_sample_block contains keys and "left" columns. * result_sample_block - keys, "left" columns, and "right" columns. */ size_t num_keys = parent.key_names_left.size(); size_t num_columns_left = left_sample_block.columns() - num_keys; size_t num_columns_right = parent.sample_block_with_columns_to_add.columns(); result_sample_block = materializeBlock(left_sample_block); /// Add columns from the right-side table to the block. for (size_t i = 0; i < num_columns_right; ++i) { const ColumnWithTypeAndName & src_column = parent.sample_block_with_columns_to_add.getByPosition(i); result_sample_block.insert(src_column.cloneEmpty()); } column_indices_left.reserve(num_columns_left); column_indices_keys_and_right.reserve(num_keys + num_columns_right); std::vector<bool> is_key_column_in_left_block(num_keys + num_columns_left, false); for (const std::string & key : parent.key_names_left) { size_t key_pos = left_sample_block.getPositionByName(key); is_key_column_in_left_block[key_pos] = true; /// Here we establish the mapping between key columns of the left- and right-side tables. /// key_pos index is inserted in the position corresponding to key column in parent.blocks /// (saved blocks of the right-side table) and points to the same key column /// in the left_sample_block and thus in the result_sample_block. column_indices_keys_and_right.push_back(key_pos); } for (size_t i = 0; i < num_keys + num_columns_left; ++i) { if (!is_key_column_in_left_block[i]) column_indices_left.push_back(i); } for (size_t i = 0; i < num_columns_right; ++i) column_indices_keys_and_right.push_back(num_keys + num_columns_left + i); /// If use_nulls, convert left columns to Nullable. if (parent.use_nulls) { for (size_t i = 0; i < num_columns_left; ++i) { convertColumnToNullable(result_sample_block.getByPosition(column_indices_left[i])); } } columns_left.resize(num_columns_left); columns_keys_and_right.resize(num_keys + num_columns_right); }
Block MaterializingBlockInputStream::readImpl() { return materializeBlock(children.back()->read()); }
void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) { LOG_TRACE(log, (subquery.set ? "Creating set. " : "") << (subquery.join ? "Creating join. " : "") << (subquery.table ? "Filling temporary table. " : "")); Stopwatch watch; BlockOutputStreamPtr table_out; if (subquery.table) table_out = subquery.table->write({}, context); bool done_with_set = !subquery.set; bool done_with_join = !subquery.join; bool done_with_table = !subquery.table; if (done_with_set && done_with_join && done_with_table) throw Exception("Logical error: nothing to do with subquery", ErrorCodes::LOGICAL_ERROR); if (table_out) table_out->writePrefix(); while (Block block = subquery.source->read()) { if (isCancelled()) { LOG_DEBUG(log, "Query was cancelled during set / join or temporary table creation."); return; } if (!done_with_set) { if (!subquery.set->insertFromBlock(block)) done_with_set = true; } if (!done_with_join) { subquery.renameColumns(block); if (subquery.joined_block_actions) subquery.joined_block_actions->execute(block); if (!subquery.join->insertFromBlock(block)) done_with_join = true; } if (!done_with_table) { block = materializeBlock(block); table_out->write(block); rows_to_transfer += block.rows(); bytes_to_transfer += block.bytes(); if (!network_transfer_limits.check(rows_to_transfer, bytes_to_transfer, "IN/JOIN external table", ErrorCodes::SET_SIZE_LIMIT_EXCEEDED)) done_with_table = true; } if (done_with_set && done_with_join && done_with_table) { subquery.source->cancel(false); break; } } if (table_out) table_out->writeSuffix(); watch.stop(); size_t head_rows = 0; const BlockStreamProfileInfo & profile_info = subquery.source->getProfileInfo(); head_rows = profile_info.rows; if (subquery.join) subquery.join->setTotals(subquery.source->getTotals()); if (head_rows != 0) { std::stringstream msg; msg << std::fixed << std::setprecision(3); msg << "Created. "; if (subquery.set) msg << "Set with " << subquery.set->getTotalRowCount() << " entries from " << head_rows << " rows. "; if (subquery.join) msg << "Join with " << subquery.join->getTotalRowCount() << " entries from " << head_rows << " rows. "; if (subquery.table) msg << "Table with " << head_rows << " rows. "; msg << "In " << watch.elapsedSeconds() << " sec."; LOG_DEBUG(log, msg.rdbuf()); } else { LOG_DEBUG(log, "Subquery has empty result."); } }