void GDAFile::add(Common::SeekableReadStream *gda) {
try {
_gff4s.push_back(new GFF4File(gda, kG2DAID));
const GFF4Struct &top = _gff4s.back()->getTopLevel();
_rows.push_back(&top.getList(kGFF4G2DARowList));
_rowStarts.push_back(_rowCount);
_rowCount += _rows.back()->size();
Columns columns = &top.getList(kGFF4G2DAColumnList);
if (columns->size() != _columns->size())
throw Common::Exception("Column counts don't match (%u vs. %u)",
(uint)columns->size(), (uint)_columns->size());
for (size_t i = 0; i < columns->size(); i++) {
const uint32 hash1 = (uint32) (* columns)[i]->getUint(kGFF4G2DAColumnHash);
const uint32 hash2 = (uint32) (*_columns)[i]->getUint(kGFF4G2DAColumnHash);
const uint32 type1 = identifyType( columns, _rows.back(), i);
const uint32 type2 = identifyType(_columns, _rows[0] , i);
if ((hash1 != hash2) || (type1 != type2))
throw Common::Exception("Columns don't match (%u: %u+%u vs. %u+%u)", (uint) i,
hash1, type1, hash2, type2);
}
} catch (Common::Exception &e) {
clear();
e.add("Failed adding GDA file");
throw;
}
}
FunctionArrayIntersect::UnpackedArrays FunctionArrayIntersect::prepareArrays(const Columns & columns) const
{
UnpackedArrays arrays;
size_t columns_number = columns.size();
arrays.is_const.assign(columns_number, false);
arrays.null_maps.resize(columns_number);
arrays.offsets.resize(columns_number);
arrays.nested_columns.resize(columns_number);
for (auto i : ext::range(0, columns_number))
{
auto argument_column = columns[i].get();
if (auto argument_column_const = typeid_cast<const ColumnConst *>(argument_column))
{
arrays.is_const[i] = true;
argument_column = argument_column_const->getDataColumnPtr().get();
}
if (auto argument_column_array = typeid_cast<const ColumnArray *>(argument_column))
{
arrays.offsets[i] = &argument_column_array->getOffsets();
arrays.nested_columns[i] = &argument_column_array->getData();
if (auto column_nullable = typeid_cast<const ColumnNullable *>(arrays.nested_columns[i]))
{
arrays.null_maps[i] = &column_nullable->getNullMapData();
arrays.nested_columns[i] = &column_nullable->getNestedColumn();
}
}
else
throw Exception{"Arguments for function " + getName() + " must be arrays.", ErrorCodes::LOGICAL_ERROR};
}
return arrays;
}
uint32 GDAFile::identifyType(const Columns &columns, const Row &rows, size_t column) const {
if (!columns || (column >= columns->size()) || !(*columns)[column])
return -1;
if ((*columns)[column]->hasField(kGFF4G2DAColumnType))
return (uint32) (*columns)[column]->getUint(kGFF4G2DAColumnType, -1);
if (!rows || rows->empty() || !(*rows)[0])
return -1;
GFF4Struct::FieldType fieldType = (*rows)[0]->getFieldType(kGFF4G2DAColumn1 + column);
switch (fieldType) {
case GFF4Struct::kFieldTypeString:
return 0;
case GFF4Struct::kFieldTypeUint:
case GFF4Struct::kFieldTypeSint:
return 1;
case GFF4Struct::kFieldTypeDouble:
return 2;
default:
break;
}
return -1;
}
void text_tree::prepare (int current_indent, int indent, Columns& columns)
{
num_siblings = 1; // including ourselves
for ( Children::iterator i=children.begin(); i!=children.end(); ++i )
{
if ( (*i)->shown )
{
(*i)->prepare(current_indent+indent, indent, columns);
num_siblings += (*i)->num_siblings;
}
}
if ( columns.size() < strings.size() )
{
columns.resize(strings.size());
}
Strings::iterator j = strings.begin();
Columns::iterator c = columns.begin();
// only count as column if theres more then 1 on the line!
if ( strings.size() > 1 )
{
for ( ; j!=strings.end(); ++j, ++c )
{
int string_size = (int)(*j).size();
string_size += (j==strings.begin()) ? current_indent : 0;
*c = std::max(string_size, *c);
}
}
}
void ExternalQueryBuilder::composeKeyTuple(const Columns & key_columns, const size_t row, WriteBuffer & out) const
{
writeString("(", out);
const auto keys_size = key_columns.size();
auto first = true;
for (const auto i : ext::range(0, keys_size))
{
if (!first)
writeString(", ", out);
first = false;
(*dict_struct.key)[i].type->serializeAsTextQuoted(*key_columns[i], row, out, format_settings);
}
writeString(")", out);
}
void ExternalQueryBuilder::composeKeyCondition(const Columns & key_columns, const size_t row, WriteBuffer & out) const
{
writeString("(", out);
const auto keys_size = key_columns.size();
auto first = true;
for (const auto i : ext::range(0, keys_size))
{
if (!first)
writeString(" AND ", out);
first = false;
const auto & key_description = (*dict_struct.key)[i];
/// key_i=value_i
writeString(key_description.name, out);
writeString("=", out);
key_description.type->serializeAsTextQuoted(*key_columns[i], row, out, format_settings);
}
writeString(")", out);
}
ColumnPtr recursiveLowCardinalityConversion(const ColumnPtr & column, const DataTypePtr & from_type, const DataTypePtr & to_type)
{
if (!column)
return column;
if (from_type->equals(*to_type))
return column;
if (const auto * column_const = typeid_cast<const ColumnConst *>(column.get()))
return ColumnConst::create(recursiveLowCardinalityConversion(column_const->getDataColumnPtr(), from_type, to_type),
column_const->size());
if (const auto * low_cardinality_type = typeid_cast<const DataTypeLowCardinality *>(from_type.get()))
{
if (to_type->equals(*low_cardinality_type->getDictionaryType()))
return column->convertToFullColumnIfLowCardinality();
}
if (const auto * low_cardinality_type = typeid_cast<const DataTypeLowCardinality *>(to_type.get()))
{
if (from_type->equals(*low_cardinality_type->getDictionaryType()))
{
auto col = low_cardinality_type->createColumn();
static_cast<ColumnLowCardinality &>(*col).insertRangeFromFullColumn(*column, 0, column->size());
return std::move(col);
}
}
if (const auto * from_array_type = typeid_cast<const DataTypeArray *>(from_type.get()))
{
if (const auto * to_array_type = typeid_cast<const DataTypeArray *>(to_type.get()))
{
const auto * column_array = typeid_cast<const ColumnArray *>(column.get());
if (!column_array)
throw Exception("Unexpected column " + column->getName() + " for type " + from_type->getName(),
ErrorCodes::ILLEGAL_COLUMN);
auto & nested_from = from_array_type->getNestedType();
auto & nested_to = to_array_type->getNestedType();
return ColumnArray::create(
recursiveLowCardinalityConversion(column_array->getDataPtr(), nested_from, nested_to),
column_array->getOffsetsPtr());
}
}
if (const auto * from_tuple_type = typeid_cast<const DataTypeTuple *>(from_type.get()))
{
if (const auto * to_tuple_type = typeid_cast<const DataTypeTuple *>(to_type.get()))
{
const auto * column_tuple = typeid_cast<const ColumnTuple *>(column.get());
if (!column_tuple)
throw Exception("Unexpected column " + column->getName() + " for type " + from_type->getName(),
ErrorCodes::ILLEGAL_COLUMN);
Columns columns = column_tuple->getColumns();
auto & from_elements = from_tuple_type->getElements();
auto & to_elements = to_tuple_type->getElements();
for (size_t i = 0; i < columns.size(); ++i)
{
auto & element = columns[i];
element = recursiveLowCardinalityConversion(element, from_elements.at(i), to_elements.at(i));
}
return ColumnTuple::create(columns);
}
}
throw Exception("Cannot convert: " + from_type->getName() + " to " + to_type->getName(), ErrorCodes::TYPE_MISMATCH);
}
BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_columns, const std::vector<std::size_t> & requested_rows)
{
LOG_TRACE(log, "loadKeys " << toString() << " size = " << requested_rows.size());
auto holder = std::make_unique<ClickHouseLibrary::Row[]>(key_columns.size());
std::vector<std::unique_ptr<ClickHouseLibrary::Field[]>> column_data_holders;
for (size_t i = 0; i < key_columns.size(); ++i)
{
auto cell_holder = std::make_unique<ClickHouseLibrary::Field[]>(requested_rows.size());
for (size_t j = 0; j < requested_rows.size(); ++j)
{
auto data_ref = key_columns[i]->getDataAt(requested_rows[j]);
cell_holder[j] = ClickHouseLibrary::Field{.data = static_cast<const void *>(data_ref.data), .size = data_ref.size};
}
holder[i]
= ClickHouseLibrary::Row{.data = static_cast<ClickHouseLibrary::Field *>(cell_holder.get()), .size = requested_rows.size()};
column_data_holders.push_back(std::move(cell_holder));
}
ClickHouseLibrary::Table request_cols{.data = static_cast<ClickHouseLibrary::Row *>(holder.get()), .size = key_columns.size()};
void * data_ptr = nullptr;
/// Get function pointer before dataNew call because library->get may throw.
auto func_loadKeys = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&request_cols))>(
"ClickHouseDictionary_v3_loadKeys");
data_ptr = library->get<decltype(data_ptr) (*)(decltype(lib_data))>("ClickHouseDictionary_v3_dataNew")(lib_data);
auto data = func_loadKeys(data_ptr, &settings->strings, &request_cols);
auto block = dataToBlock(description.sample_block, data);
SCOPE_EXIT(library->get<void (*)(decltype(lib_data), decltype(data_ptr))>("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr));
return std::make_shared<OneBlockInputStream>(block);
}
bool LibraryDictionarySource::isModified() const
{
if (auto func_isModified
= library->tryGet<bool (*)(decltype(lib_data), decltype(&settings->strings))>("ClickHouseDictionary_v3_isModified"))
return func_isModified(lib_data, &settings->strings);
return true;
}
bool LibraryDictionarySource::supportsSelectiveLoad() const
{
if (auto func_supportsSelectiveLoad
= library->tryGet<bool (*)(decltype(lib_data), decltype(&settings->strings))>("ClickHouseDictionary_v3_supportsSelectiveLoad"))
return func_supportsSelectiveLoad(lib_data, &settings->strings);
return true;
}
DictionarySourcePtr LibraryDictionarySource::clone() const
{
return std::make_unique<LibraryDictionarySource>(*this);
}
std::string LibraryDictionarySource::toString() const
{
return path;
}
void registerDictionarySourceLibrary(DictionarySourceFactory & factory)
{
auto createTableSource = [=](const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
Block & sample_block,
const Context &) -> DictionarySourcePtr
{
return std::make_unique<LibraryDictionarySource>(dict_struct, config, config_prefix + ".library", sample_block);
};
factory.registerSource("library", createTableSource);
}
}