static ColumnPtr getFilteredDatabases(const ASTPtr & query, const Context & context)
{
MutableColumnPtr column = ColumnString::create();
for (const auto & db : context.getDatabases())
column->insert(db.first);
Block block { ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database") };
VirtualColumnUtils::filterBlockWithQuery(query, block, context);
return block.getByPosition(0).column;
}
Block TinyLogBlockInputStream::readImpl()
{
Block res;
if (finished || (!streams.empty() && streams.begin()->second->compressed.eof()))
{
/** Close the files (before destroying the object).
* When many sources are created, but simultaneously reading only a few of them,
* buffers don't waste memory.
*/
finished = true;
streams.clear();
return res;
}
{
/// if there are no files in the folder, it means that the table is empty
if (Poco::DirectoryIterator(storage.full_path()) == Poco::DirectoryIterator())
return res;
}
for (const auto & name_type : columns)
{
MutableColumnPtr column = name_type.type->createColumn();
try
{
readData(name_type.name, *name_type.type, *column, block_size);
}
catch (Exception & e)
{
e.addMessage("while reading column " + name_type.name + " at " + storage.full_path());
throw;
}
if (column->size())
res.insert(ColumnWithTypeAndName(std::move(column), name_type.type, name_type.name));
}
if (!res || streams.begin()->second->compressed.eof())
{
finished = true;
streams.clear();
}
return Nested::flatten(res);
}
int main(int, char **)
{
ColumnPtr x = ConcreteColumn::create(1);
ColumnPtr y = x;//x->test();
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
{
MutableColumnPtr mut = y->mutate();
mut->set(2);
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << ", " << mut.get() << "\n";
y = std::move(mut);
}
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
x = ConcreteColumn::create(0);
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
{
MutableColumnPtr mut = y->mutate();
mut->set(3);
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << ", " << mut.get() << "\n";
y = std::move(mut);
}
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
return 0;
}
void executeImpl(Block & block, const ColumnNumbers & args, size_t result, size_t input_rows_count) override
{
/** We will gather values from columns in branches to result column,
* depending on values of conditions.
*/
struct Instruction
{
const IColumn * condition = nullptr;
const IColumn * source = nullptr;
bool condition_always_true = false;
bool condition_is_nullable = false;
bool source_is_constant = false;
};
std::vector<Instruction> instructions;
instructions.reserve(args.size() / 2 + 1);
Columns converted_columns_holder;
converted_columns_holder.reserve(instructions.size());
const DataTypePtr & return_type = block.getByPosition(result).type;
for (size_t i = 0; i < args.size(); i += 2)
{
Instruction instruction;
size_t source_idx = i + 1;
if (source_idx == args.size())
{
/// The last, "else" branch can be treated as a branch with always true condition "else if (true)".
--source_idx;
instruction.condition_always_true = true;
}
else
{
const ColumnWithTypeAndName & cond_col = block.getByPosition(args[i]);
/// We skip branches that are always false.
/// If we encounter a branch that is always true, we can finish.
if (cond_col.column->onlyNull())
continue;
if (cond_col.column->isColumnConst())
{
Field value = typeid_cast<const ColumnConst &>(*cond_col.column).getField();
if (value.isNull())
continue;
if (value.get<UInt64>() == 0)
continue;
instruction.condition_always_true = true;
}
else
{
if (cond_col.column->isColumnNullable())
instruction.condition_is_nullable = true;
instruction.condition = cond_col.column.get();
}
}
const ColumnWithTypeAndName & source_col = block.getByPosition(args[source_idx]);
if (source_col.type->equals(*return_type))
{
instruction.source = source_col.column.get();
}
else
{
/// Cast all columns to result type.
converted_columns_holder.emplace_back(castColumn(source_col, return_type, context));
instruction.source = converted_columns_holder.back().get();
}
if (instruction.source && instruction.source->isColumnConst())
instruction.source_is_constant = true;
instructions.emplace_back(std::move(instruction));
if (instructions.back().condition_always_true)
break;
}
size_t rows = input_rows_count;
MutableColumnPtr res = return_type->createColumn();
for (size_t i = 0; i < rows; ++i)
{
for (const auto & instruction : instructions)
{
bool insert = false;
if (instruction.condition_always_true)
insert = true;
else if (!instruction.condition_is_nullable)
insert = static_cast<const ColumnUInt8 &>(*instruction.condition).getData()[i];
else
{
const ColumnNullable & condition_nullable = static_cast<const ColumnNullable &>(*instruction.condition);
const ColumnUInt8 & condition_nested = static_cast<const ColumnUInt8 &>(condition_nullable.getNestedColumn());
const NullMap & condition_null_map = condition_nullable.getNullMapData();
insert = !condition_null_map[i] && condition_nested.getData()[i];
}
if (insert)
{
if (!instruction.source_is_constant)
res->insertFrom(*instruction.source, i);
else
res->insertFrom(static_cast<const ColumnConst &>(*instruction.source).getDataColumn(), 0);
break;
}
}
}
block.getByPosition(result).column = std::move(res);
}
size_t MergeTreeReader::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res)
{
size_t read_rows = 0;
try
{
/// Pointers to offset columns that are common to the nested data structure columns.
/// If append is true, then the value will be equal to nullptr and will be used only to
/// check that the offsets column has been already read.
OffsetColumns offset_columns;
for (const NameAndTypePair & it : columns)
{
/// The column is already present in the block so we will append the values to the end.
bool append = res.has(it.name);
if (!append)
res.insert(ColumnWithTypeAndName(it.type->createColumn(), it.type, it.name));
/// To keep offsets shared. TODO Very dangerous. Get rid of this.
MutableColumnPtr column = res.getByName(it.name).column->assumeMutable();
bool read_offsets = true;
/// For nested data structures collect pointers to offset columns.
if (const DataTypeArray * type_arr = typeid_cast<const DataTypeArray *>(it.type.get()))
{
String name = Nested::extractTableName(it.name);
auto it_inserted = offset_columns.emplace(name, nullptr);
/// offsets have already been read on the previous iteration and we don't need to read it again
if (!it_inserted.second)
read_offsets = false;
/// need to create new offsets
if (it_inserted.second && !append)
it_inserted.first->second = ColumnArray::ColumnOffsets::create();
/// share offsets in all elements of nested structure
if (!append)
column = ColumnArray::create(type_arr->getNestedType()->createColumn(), it_inserted.first->second);
}
try
{
size_t column_size_before_reading = column->size();
readData(it.name, *it.type, *column, from_mark, continue_reading, max_rows_to_read, read_offsets);
/// For elements of Nested, column_size_before_reading may be greater than column size
/// if offsets are not empty and were already read, but elements are empty.
if (column->size())
read_rows = std::max(read_rows, column->size() - column_size_before_reading);
}
catch (Exception & e)
{
/// Better diagnostics.
e.addMessage("(while reading column " + it.name + ")");
throw;
}
if (column->size())
res.getByName(it.name).column = std::move(column);
else
res.erase(it.name);
}
/// NOTE: positions for all streams must be kept in sync. In particular, even if for some streams there are no rows to be read,
/// you must ensure that no seeks are skipped and at this point they all point to to_mark.
}
catch (Exception & e)
{
if (e.code() != ErrorCodes::MEMORY_LIMIT_EXCEEDED)
storage.reportBrokenPart(data_part->name);
/// Better diagnostics.
e.addMessage("(while reading from part " + path + " from mark " + toString(from_mark) + " with max_rows_to_read = " + toString(max_rows_to_read) + ")");
throw;
}
catch (...)
{
storage.reportBrokenPart(data_part->name);
throw;
}
return read_rows;
}