static void checkTypeOfKey(const Block & block_left, const Block & block_right)
{
auto & [c1, left_type_origin, left_name] = block_left.safeGetByPosition(0);
auto & [c2, right_type_origin, right_name] = block_right.safeGetByPosition(0);
auto left_type = removeNullable(left_type_origin);
auto right_type = removeNullable(right_type_origin);
if (!left_type->equals(*right_type))
throw Exception("Type mismatch of columns to joinGet by: "
+ left_name + " " + left_type->getName() + " at left, "
+ right_name + " " + right_type->getName() + " at right",
ErrorCodes::TYPE_MISMATCH);
}
void FunctionIfNull::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result)
{
/// Always null.
if (block.getByPosition(arguments[0]).type->onlyNull())
{
block.getByPosition(result).column = block.getByPosition(arguments[1]).column;
return;
}
/// Could not contain nulls, so nullIf makes no sense.
if (!block.getByPosition(arguments[0]).type->isNullable())
{
block.getByPosition(result).column = block.getByPosition(arguments[0]).column;
return;
}
/// ifNull(col1, col2) == if(isNotNull(col1), assumeNotNull(col1), col2)
Block temp_block = block;
size_t is_not_null_pos = temp_block.columns();
temp_block.insert({nullptr, std::make_shared<DataTypeUInt8>(), ""});
size_t assume_not_null_pos = temp_block.columns();
temp_block.insert({nullptr, removeNullable(block.getByPosition(arguments[0]).type), ""});
FunctionIsNotNull{}.execute(temp_block, {arguments[0]}, is_not_null_pos);
FunctionAssumeNotNull{}.execute(temp_block, {arguments[0]}, assume_not_null_pos);
FunctionIf{}.execute(temp_block, {is_not_null_pos, assume_not_null_pos, arguments[1]}, result);
block.getByPosition(result).column = std::move(temp_block.getByPosition(result).column);
}
DataTypePtr FunctionCoalesce::getReturnTypeImpl(const DataTypes & arguments) const
{
/// Skip all NULL arguments. If any argument is non-Nullable, skip all next arguments.
DataTypes filtered_args;
filtered_args.reserve(arguments.size());
for (const auto & arg : arguments)
{
if (arg->onlyNull())
continue;
filtered_args.push_back(arg);
if (!arg->isNullable())
break;
}
DataTypes new_args;
for (size_t i = 0; i < filtered_args.size(); ++i)
{
bool is_last = i + 1 == filtered_args.size();
if (is_last)
{
new_args.push_back(filtered_args[i]);
}
else
{
new_args.push_back(std::make_shared<DataTypeUInt8>());
new_args.push_back(removeNullable(filtered_args[i]));
}
}
if (new_args.empty())
return std::make_shared<DataTypeNullable>(std::make_shared<DataTypeNothing>());
if (new_args.size() == 1)
return new_args.front();
auto res = FunctionMultiIf{context}.getReturnTypeImpl(new_args);
/// if last argument is not nullable, result should be also not nullable
if (!new_args.back()->isNullable() && res->isNullable())
res = removeNullable(res);
return res;
}
void Join::checkTypesOfKeys(const Block & block_left, const Block & block_right) const
{
size_t keys_size = key_names_left.size();
for (size_t i = 0; i < keys_size; ++i)
{
/// Compare up to Nullability.
DataTypePtr left_type = removeNullable(block_left.getByName(key_names_left[i]).type);
DataTypePtr right_type = removeNullable(block_right.getByName(key_names_right[i]).type);
if (!left_type->equals(*right_type))
throw Exception("Type mismatch of columns to JOIN by: "
+ key_names_left[i] + " " + left_type->getName() + " at left, "
+ key_names_right[i] + " " + right_type->getName() + " at right",
ErrorCodes::TYPE_MISMATCH);
}
}
DataTypePtr FunctionIfNull::getReturnTypeImpl(const DataTypes & arguments) const
{
if (arguments[0]->onlyNull())
return arguments[1];
if (!arguments[0]->isNullable())
return arguments[0];
return FunctionIf{}.getReturnTypeImpl({std::make_shared<DataTypeUInt8>(), removeNullable(arguments[0]), arguments[1]});
}
void FunctionArrayIntersect::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count)
{
const auto & return_type = block.getByPosition(result).type;
auto return_type_array = checkAndGetDataType<DataTypeArray>(return_type.get());
if (!return_type)
throw Exception{"Return type for function " + getName() + " must be array.", ErrorCodes::LOGICAL_ERROR};
const auto & nested_return_type = return_type_array->getNestedType();
if (typeid_cast<const DataTypeNothing *>(nested_return_type.get()))
{
block.getByPosition(result).column = return_type->createColumnConstWithDefaultValue(input_rows_count);
return;
}
auto num_args = arguments.size();
DataTypes data_types;
data_types.reserve(num_args);
for (size_t i = 0; i < num_args; ++i)
data_types.push_back(block.getByPosition(arguments[i]).type);
auto return_type_with_nulls = getMostSubtype(data_types, true, true);
Columns columns = castColumns(block, arguments, return_type, return_type_with_nulls);
UnpackedArrays arrays = prepareArrays(columns);
ColumnPtr result_column;
auto not_nullable_nested_return_type = removeNullable(nested_return_type);
TypeListNumbers::forEach(NumberExecutor(arrays, not_nullable_nested_return_type, result_column));
using DateMap = ClearableHashMap<DataTypeDate::FieldType, size_t, DefaultHash<DataTypeDate::FieldType>,
HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(DataTypeDate::FieldType)>>;
using DateTimeMap = ClearableHashMap<DataTypeDateTime::FieldType, size_t, DefaultHash<DataTypeDateTime::FieldType>,
HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(DataTypeDateTime::FieldType)>>;
using StringMap = ClearableHashMap<StringRef, size_t, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
if (!result_column)
{
auto column = not_nullable_nested_return_type->createColumn();
WhichDataType which(not_nullable_nested_return_type);
if (which.isDate())
result_column = execute<DateMap, ColumnVector<DataTypeDate::FieldType>, true>(arrays, std::move(column));
else if (which.isDateTime())
result_column = execute<DateTimeMap, ColumnVector<DataTypeDateTime::FieldType>, true>(arrays, std::move(column));
else if (which.isString())
result_column = execute<StringMap, ColumnString, false>(arrays, std::move(column));
else if (which.isFixedString())
result_column = execute<StringMap, ColumnFixedString, false>(arrays, std::move(column));
else
{
column = static_cast<const DataTypeArray &>(*return_type_with_nulls).getNestedType()->createColumn();
result_column = castRemoveNullable(execute<StringMap, IColumn, false>(arrays, std::move(column)), return_type);
}
}
block.getByPosition(result).column = std::move(result_column);
}
Columns FunctionArrayIntersect::castColumns(
Block & block, const ColumnNumbers & arguments, const DataTypePtr & return_type,
const DataTypePtr & return_type_with_nulls) const
{
size_t num_args = arguments.size();
Columns columns(num_args);
auto type_array = checkAndGetDataType<DataTypeArray>(return_type.get());
auto & type_nested = type_array->getNestedType();
auto type_not_nullable_nested = removeNullable(type_nested);
const bool is_numeric_or_string = isNumber(type_not_nullable_nested)
|| isDateOrDateTime(type_not_nullable_nested)
|| isStringOrFixedString(type_not_nullable_nested);
DataTypePtr nullable_return_type;
if (is_numeric_or_string)
{
auto type_nullable_nested = makeNullable(type_nested);
nullable_return_type = std::make_shared<DataTypeArray>(type_nullable_nested);
}
const bool nested_is_nullable = type_nested->isNullable();
for (size_t i = 0; i < num_args; ++i)
{
const ColumnWithTypeAndName & arg = block.getByPosition(arguments[i]);
auto & column = columns[i];
if (is_numeric_or_string)
{
/// Cast to Array(T) or Array(Nullable(T)).
if (nested_is_nullable)
{
if (arg.type->equals(*return_type))
column = arg.column;
else
column = castColumn(arg, return_type, context);
}
else
{
/// If result has array type Array(T) still cast Array(Nullable(U)) to Array(Nullable(T))
/// because cannot cast Nullable(T) to T.
if (arg.type->equals(*return_type) || arg.type->equals(*nullable_return_type))
column = arg.column;
else if (static_cast<const DataTypeArray &>(*arg.type).getNestedType()->isNullable())
column = castColumn(arg, nullable_return_type, context);
else
column = castColumn(arg, return_type, context);
}
}
else
{
/// return_type_with_nulls is the most common subtype with possible nullable parts.
if (arg.type->equals(*return_type_with_nulls))
column = arg.column;
else
column = castColumn(arg, return_type_with_nulls, context);
}
}
return columns;
}
DataTypePtr FunctionAssumeNotNull::getReturnTypeImpl(const DataTypes & arguments) const
{
return removeNullable(arguments[0]);
}
void FunctionCoalesce::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result)
{
/// coalesce(arg0, arg1, ..., argN) is essentially
/// multiIf(isNotNull(arg0), assumeNotNull(arg0), isNotNull(arg1), assumeNotNull(arg1), ..., argN)
/// with constant NULL arguments removed.
ColumnNumbers filtered_args;
filtered_args.reserve(arguments.size());
for (const auto & arg : arguments)
{
const auto & type = block.getByPosition(arg).type;
if (type->onlyNull())
continue;
filtered_args.push_back(arg);
if (!type->isNullable())
break;
}
FunctionIsNotNull is_not_null;
FunctionAssumeNotNull assume_not_null;
ColumnNumbers multi_if_args;
Block temp_block = block;
for (size_t i = 0; i < filtered_args.size(); ++i)
{
size_t res_pos = temp_block.columns();
bool is_last = i + 1 == filtered_args.size();
if (is_last)
{
multi_if_args.push_back(filtered_args[i]);
}
else
{
temp_block.insert({nullptr, std::make_shared<DataTypeUInt8>(), ""});
is_not_null.execute(temp_block, {filtered_args[i]}, res_pos);
temp_block.insert({nullptr, removeNullable(block.getByPosition(filtered_args[i]).type), ""});
assume_not_null.execute(temp_block, {filtered_args[i]}, res_pos + 1);
multi_if_args.push_back(res_pos);
multi_if_args.push_back(res_pos + 1);
}
}
/// If all arguments appeared to be NULL.
if (multi_if_args.empty())
{
block.getByPosition(result).column = block.getByPosition(result).type->createColumnConstWithDefaultValue(block.rows());
return;
}
if (multi_if_args.size() == 1)
{
block.getByPosition(result).column = block.getByPosition(multi_if_args.front()).column;
return;
}
FunctionMultiIf{context}.execute(temp_block, multi_if_args, result);
ColumnPtr res = std::move(temp_block.getByPosition(result).column);
/// if last argument is not nullable, result should be also not nullable
if (!block.getByPosition(multi_if_args.back()).column->isColumnNullable() && res->isColumnNullable())
res = static_cast<const ColumnNullable &>(*res).getNestedColumnPtr();
block.getByPosition(result).column = std::move(res);
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
{
/// coalesce(arg0, arg1, ..., argN) is essentially
/// multiIf(isNotNull(arg0), assumeNotNull(arg0), isNotNull(arg1), assumeNotNull(arg1), ..., argN)
/// with constant NULL arguments removed.
ColumnNumbers filtered_args;
filtered_args.reserve(arguments.size());
for (const auto & arg : arguments)
{
const auto & type = block.getByPosition(arg).type;
if (type->onlyNull())
continue;
filtered_args.push_back(arg);
if (!type->isNullable())
break;
}
auto is_not_null = FunctionFactory::instance().get("isNotNull", context);
auto assume_not_null = FunctionFactory::instance().get("assumeNotNull", context);
auto multi_if = FunctionFactory::instance().get("multiIf", context);
ColumnNumbers multi_if_args;
Block temp_block = block;
for (size_t i = 0; i < filtered_args.size(); ++i)
{
size_t res_pos = temp_block.columns();
bool is_last = i + 1 == filtered_args.size();
if (is_last)
{
multi_if_args.push_back(filtered_args[i]);
}
else
{
temp_block.insert({nullptr, std::make_shared<DataTypeUInt8>(), ""});
is_not_null->build({temp_block.getByPosition(filtered_args[i])})->execute(temp_block, {filtered_args[i]}, res_pos, input_rows_count);
temp_block.insert({nullptr, removeNullable(block.getByPosition(filtered_args[i]).type), ""});
assume_not_null->build({temp_block.getByPosition(filtered_args[i])})->execute(temp_block, {filtered_args[i]}, res_pos + 1, input_rows_count);
multi_if_args.push_back(res_pos);
multi_if_args.push_back(res_pos + 1);
}
}
/// If all arguments appeared to be NULL.
if (multi_if_args.empty())
{
block.getByPosition(result).column = block.getByPosition(result).type->createColumnConstWithDefaultValue(input_rows_count);
return;
}
if (multi_if_args.size() == 1)
{
block.getByPosition(result).column = block.getByPosition(multi_if_args.front()).column;
return;
}
ColumnsWithTypeAndName multi_if_args_elems;
multi_if_args_elems.reserve(multi_if_args.size());
for (auto column_num : multi_if_args)
multi_if_args_elems.emplace_back(temp_block.getByPosition(column_num));
multi_if->build(multi_if_args_elems)->execute(temp_block, multi_if_args, result, input_rows_count);
ColumnPtr res = std::move(temp_block.getByPosition(result).column);
/// if last argument is not nullable, result should be also not nullable
if (!block.getByPosition(multi_if_args.back()).column->isColumnNullable() && res->isColumnNullable())
res = static_cast<const ColumnNullable &>(*res).getNestedColumnPtr();
block.getByPosition(result).column = std::move(res);
}
