/// NOTE: This ID is used to create part names which are then persisted in ZK and as directory names on the file system.
/// So if you want to change this method, be sure to guarantee compatibility with existing table data.
String MergeTreePartition::getID(const Block & partition_key_sample) const
{
if (value.size() != partition_key_sample.columns())
throw Exception("Invalid partition key size: " + toString(value.size()), ErrorCodes::LOGICAL_ERROR);
if (value.empty())
return "all"; /// It is tempting to use an empty string here. But that would break directory structure in ZK.
/// In case all partition fields are represented by integral types, try to produce a human-readable ID.
/// Otherwise use a hex-encoded hash.
bool are_all_integral = true;
for (const Field & field : value)
{
if (field.getType() != Field::Types::UInt64 && field.getType() != Field::Types::Int64)
{
are_all_integral = false;
break;
}
}
String result;
if (are_all_integral)
{
FieldVisitorToString to_string_visitor;
for (size_t i = 0; i < value.size(); ++i)
{
if (i > 0)
result += '-';
if (typeid_cast<const DataTypeDate *>(partition_key_sample.getByPosition(i).type.get()))
result += toString(DateLUT::instance().toNumYYYYMMDD(DayNum(value[i].safeGet<UInt64>())));
else
result += applyVisitor(to_string_visitor, value[i]);
/// It is tempting to output DateTime as YYYYMMDDhhmmss, but that would make partition ID
/// timezone-dependent.
}
return result;
}
SipHash hash;
FieldVisitorHash hashing_visitor(hash);
for (const Field & field : value)
applyVisitor(hashing_visitor, field);
char hash_data[16];
hash.get128(hash_data);
result.resize(32);
for (size_t i = 0; i < 16; ++i)
writeHexByteLowercase(hash_data[i], &result[2 * i]);
return result;
}
void SettingsConstraints::check(const Settings & current_settings, const SettingChange & change) const
{
const String & name = change.name;
size_t setting_index = Settings::findIndex(name);
if (setting_index == Settings::npos)
return;
Field new_value = convertSettingValueToComparableType(change.value, setting_index);
Field current_value = current_settings.get(setting_index);
/// Setting isn't checked if value wasn't changed.
if (current_value == new_value)
return;
if (!current_settings.allow_ddl && name == "allow_ddl")
throw Exception("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);
/** The `readonly` value is understood as follows:
* 0 - everything allowed.
* 1 - only read queries can be made; you can not change the settings.
* 2 - You can only do read queries and you can change the settings, except for the `readonly` setting.
*/
if (current_settings.readonly == 1)
throw Exception("Cannot modify '" + name + "' setting in readonly mode", ErrorCodes::READONLY);
if (current_settings.readonly > 1 && name == "readonly")
throw Exception("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY);
const Constraint * constraint = tryGetConstraint(setting_index);
if (constraint)
{
if (constraint->read_only)
throw Exception("Setting " + name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION);
if (!constraint->min_value.isNull() && (new_value < constraint->min_value))
throw Exception(
"Setting " + name + " shouldn't be less than " + applyVisitor(FieldVisitorToString(), constraint->min_value),
ErrorCodes::SETTING_CONSTRAINT_VIOLATION);
if (!constraint->max_value.isNull() && (new_value > constraint->max_value))
throw Exception(
"Setting " + name + " shouldn't be greater than " + applyVisitor(FieldVisitorToString(), constraint->max_value),
ErrorCodes::SETTING_CONSTRAINT_VIOLATION);
}
}
void FieldVisitorHash::operator() (const Array & x) const
{
UInt8 type = Field::Types::Array;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
size_t size = x.size();
hash.update(reinterpret_cast<const char *>(&size), sizeof(size));
for (const auto & elem : x)
applyVisitor(*this, elem);
}
DataTypePtr FieldToDataType::operator() (const Array & x) const
{
DataTypes element_types;
element_types.reserve(x.size());
for (const Field & elem : x)
element_types.emplace_back(applyVisitor(FieldToDataType(), elem));
return std::make_shared<DataTypeArray>(getLeastSupertype(element_types));
}
DataTypePtr FieldToDataType::operator() (const Tuple & x) const
{
auto & tuple = static_cast<const TupleBackend &>(x);
if (tuple.empty())
throw Exception("Cannot infer type of an empty tuple", ErrorCodes::EMPTY_DATA_PASSED);
DataTypes element_types;
element_types.reserve(ext::size(tuple));
for (const auto & element : tuple)
element_types.push_back(applyVisitor(FieldToDataType(), element));
return std::make_shared<DataTypeTuple>(element_types);
}
String FieldVisitorDump::operator() (const Array & x) const
{
String res;
WriteBufferFromString wb(res);
wb.write("Array_[", 7);
for (auto it = x.begin(); it != x.end(); ++it)
{
if (it != x.begin())
wb.write(", ", 2);
writeString(applyVisitor(*this, *it), wb);
}
writeChar(']', wb);
return res;
}
String FieldVisitorDump::operator() (const Tuple & x_def) const
{
auto & x = x_def.t;
String res;
WriteBufferFromString wb(res);
wb.write("Tuple_(", 7);
for (auto it = x.begin(); it != x.end(); ++it)
{
if (it != x.begin())
wb.write(", ", 2);
writeString(applyVisitor(*this, *it), wb);
}
writeChar(')', wb);
return res;
}
BlockInputStreamPtr MongoDBDictionarySource::loadKeys(
const ConstColumnPlainPtrs & key_columns, const std::vector<std::size_t> & requested_rows)
{
if (!dict_struct.key)
throw Exception{"'key' is required for selective loading", ErrorCodes::UNSUPPORTED_METHOD};
auto cursor = createCursor(db, collection, sample_block);
Poco::MongoDB::Array::Ptr keys_array(new Poco::MongoDB::Array);
for (const auto row_idx : requested_rows)
{
auto & key = keys_array->addNewDocument(DB::toString(row_idx));
for (const auto attr : ext::enumerate(*dict_struct.key))
{
switch (attr.second.underlying_type)
{
case AttributeUnderlyingType::UInt8:
case AttributeUnderlyingType::UInt16:
case AttributeUnderlyingType::UInt32:
case AttributeUnderlyingType::UInt64:
case AttributeUnderlyingType::Int8:
case AttributeUnderlyingType::Int16:
case AttributeUnderlyingType::Int32:
case AttributeUnderlyingType::Int64:
key.add(attr.second.name, Int32(key_columns[attr.first]->get64(row_idx)));
break;
case AttributeUnderlyingType::Float32:
case AttributeUnderlyingType::Float64:
key.add(attr.second.name, applyVisitor(FieldVisitorConvertToNumber<Float64>(), (*key_columns[attr.first])[row_idx]));
break;
case AttributeUnderlyingType::String:
key.add(attr.second.name, get<String>((*key_columns[attr.first])[row_idx]));
break;
}
}
}
cursor->query().selector().add("$or", keys_array);
return std::make_shared<MongoDBBlockInputStream>(
connection, std::move(cursor), sample_block, max_block_size);
}
void TypeAndConstantInference::dump(WriteBuffer & out) const
{
/// For need of tests, we need to dump result in some fixed order.
std::vector<Info::const_iterator> vec;
vec.reserve(info.size());
for (auto it = info.begin(); it != info.end(); ++it)
vec.emplace_back(it);
std::sort(vec.begin(), vec.end(), [](const auto & a, const auto & b) { return a->first < b->first; });
for (const auto & it : vec)
{
writeString(it->first, out);
writeCString(" -> ", out);
writeString(it->second.data_type->getName(), out);
if (it->second.is_constant_expression)
{
writeCString(" = ", out);
String value = applyVisitor(FieldVisitorToString(), it->second.value);
writeString(value, out);
}
writeCString(". AST: ", out);
if (!it->second.node)
writeCString("(none)", out);
else
{
std::stringstream formatted_ast;
formatAST(*it->second.node, formatted_ast, false, true);
writeString(formatted_ast.str(), out);
}
writeChar('\n', out);
}
}
bool ValuesRowInputStream::read(MutableColumns & columns)
{
size_t num_columns = columns.size();
skipWhitespaceIfAny(istr);
if (istr.eof() || *istr.position() == ';')
return false;
/** Typically, this is the usual format for streaming parsing.
* But as an exception, it also supports processing arbitrary expressions instead of values.
* This is very inefficient. But if there are no expressions, then there is no overhead.
*/
ParserExpression parser;
assertChar('(', istr);
for (size_t i = 0; i < num_columns; ++i)
{
skipWhitespaceIfAny(istr);
char * prev_istr_position = istr.position();
size_t prev_istr_bytes = istr.count() - istr.offset();
bool rollback_on_exception = false;
try
{
header.getByPosition(i).type->deserializeTextQuoted(*columns[i], istr, format_settings);
rollback_on_exception = true;
skipWhitespaceIfAny(istr);
if (i != num_columns - 1)
assertChar(',', istr);
else
assertChar(')', istr);
}
catch (const Exception & e)
{
if (!format_settings.values.interpret_expressions)
throw;
/** The normal streaming parser could not parse the value.
* Let's try to parse it with a SQL parser as a constant expression.
* This is an exceptional case.
*/
if (e.code() == ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED
|| e.code() == ErrorCodes::CANNOT_PARSE_QUOTED_STRING
|| e.code() == ErrorCodes::CANNOT_PARSE_NUMBER
|| e.code() == ErrorCodes::CANNOT_PARSE_DATE
|| e.code() == ErrorCodes::CANNOT_PARSE_DATETIME
|| e.code() == ErrorCodes::CANNOT_READ_ARRAY_FROM_TEXT)
{
/// TODO Case when the expression does not fit entirely in the buffer.
/// If the beginning of the value is no longer in the buffer.
if (istr.count() - istr.offset() != prev_istr_bytes)
throw;
if (rollback_on_exception)
columns[i]->popBack(1);
const IDataType & type = *header.getByPosition(i).type;
Expected expected;
Tokens tokens(prev_istr_position, istr.buffer().end());
TokenIterator token_iterator(tokens);
ASTPtr ast;
if (!parser.parse(token_iterator, ast, expected))
throw Exception("Cannot parse expression of type " + type.getName() + " here: "
+ String(prev_istr_position, std::min(SHOW_CHARS_ON_SYNTAX_ERROR, istr.buffer().end() - prev_istr_position)),
ErrorCodes::SYNTAX_ERROR);
istr.position() = const_cast<char *>(token_iterator->begin);
std::pair<Field, DataTypePtr> value_raw = evaluateConstantExpression(ast, *context);
Field value = convertFieldToType(value_raw.first, type, value_raw.second.get());
/// Check that we are indeed allowed to insert a NULL.
if (value.isNull())
{
if (!type.isNullable())
throw Exception{"Expression returns value " + applyVisitor(FieldVisitorToString(), value)
+ ", that is out of range of type " + type.getName()
+ ", at: " + String(prev_istr_position, std::min(SHOW_CHARS_ON_SYNTAX_ERROR, istr.buffer().end() - prev_istr_position)),
ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE};
}
columns[i]->insert(value);
skipWhitespaceIfAny(istr);
if (i != num_columns - 1)
assertChar(',', istr);
else
assertChar(')', istr);
}
else
throw;
}
}
skipWhitespaceIfAny(istr);
if (!istr.eof() && *istr.position() == ',')
++istr.position();
return true;
}
void writeText(const Tuple & x, WriteBuffer & buf)
{
DB::String res = applyVisitor(DB::FieldVisitorToString(), DB::Field(x));
buf.write(res.data(), res.size());
}
bool ValuesRowInputStream::read(Block & block)
{
size_t size = block.columns();
skipWhitespaceIfAny(istr);
if (istr.eof() || *istr.position() == ';')
return false;
/** Как правило, это обычный формат для потокового парсинга.
* Но в качестве исключения, поддерживается также обработка произвольных выражений вместо значений.
* Это очень неэффективно. Но если выражений нет, то оверхед отсутствует.
*/
ParserExpressionWithOptionalAlias parser(false);
assertChar('(', istr);
for (size_t i = 0; i < size; ++i)
{
skipWhitespaceIfAny(istr);
char * prev_istr_position = istr.position();
size_t prev_istr_bytes = istr.count() - istr.offset();
auto & col = block.getByPosition(i);
bool rollback_on_exception = false;
try
{
col.type.get()->deserializeTextQuoted(*col.column.get(), istr);
rollback_on_exception = true;
skipWhitespaceIfAny(istr);
if (i != size - 1)
assertChar(',', istr);
else
assertChar(')', istr);
}
catch (const Exception & e)
{
if (!interpret_expressions)
throw;
/** Обычный потоковый парсер не смог распарсить значение.
* Попробуем распарсить его SQL-парсером как константное выражение.
* Это исключительный случай.
*/
if (e.code() == ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED
|| e.code() == ErrorCodes::CANNOT_PARSE_QUOTED_STRING
|| e.code() == ErrorCodes::CANNOT_PARSE_DATE
|| e.code() == ErrorCodes::CANNOT_PARSE_DATETIME
|| e.code() == ErrorCodes::CANNOT_READ_ARRAY_FROM_TEXT)
{
/// TODO Работоспособность, если выражение не помещается целиком до конца буфера.
/// Если начало значения уже не лежит в буфере.
if (istr.count() - istr.offset() != prev_istr_bytes)
throw;
if (rollback_on_exception)
col.column.get()->popBack(1);
IDataType & type = *block.safeGetByPosition(i).type;
IParser::Pos pos = prev_istr_position;
Expected expected = "";
IParser::Pos max_parsed_pos = pos;
ASTPtr ast;
if (!parser.parse(pos, istr.buffer().end(), ast, max_parsed_pos, expected))
throw Exception("Cannot parse expression of type " + type.getName() + " here: "
+ String(prev_istr_position, std::min(SHOW_CHARS_ON_SYNTAX_ERROR, istr.buffer().end() - prev_istr_position)),
ErrorCodes::SYNTAX_ERROR);
istr.position() = const_cast<char *>(max_parsed_pos);
std::pair<Field, DataTypePtr> value_raw = evaluateConstantExpression(ast, context);
Field value = convertFieldToType(value_raw.first, type, value_raw.second.get());
if (value.isNull())
{
/// Check that we are indeed allowed to insert a NULL.
bool is_null_allowed = false;
if (type.isNullable())
is_null_allowed = true;
else
{
/// NOTE: For now we support only one level of null values, i.e.
/// there are not yet such things as Array(Nullable(Array(Nullable(T))).
/// Therefore the code below is valid within the current limitations.
const auto array_type = typeid_cast<const DataTypeArray *>(&type);
if (array_type != nullptr)
{
const auto & nested_type = array_type->getMostNestedType();
if (nested_type->isNullable())
is_null_allowed = true;
}
}
if (!is_null_allowed)
throw Exception{"Expression returns value " + applyVisitor(FieldVisitorToString(), value)
+ ", that is out of range of type " + type.getName()
+ ", at: " + String(prev_istr_position, std::min(SHOW_CHARS_ON_SYNTAX_ERROR, istr.buffer().end() - prev_istr_position)),
ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE};
}
col.column->insert(value);
skipWhitespaceIfAny(istr);
if (i != size - 1)
assertChar(',', istr);
else
assertChar(')', istr);
}
else
throw;
}
}
skipWhitespaceIfAny(istr);
if (!istr.eof() && *istr.position() == ',')
++istr.position();
return true;
}
static IFunction::Monotonicity get(const Field & left, const Field & right)
{
Float64 left_float = left.isNull() ? -std::numeric_limits<Float64>::infinity() : applyVisitor(FieldVisitorConvertToNumber<Float64>(), left);
Float64 right_float = right.isNull() ? std::numeric_limits<Float64>::infinity() : applyVisitor(FieldVisitorConvertToNumber<Float64>(), right);
if (left_float < 0 || right_float > 63)
return {};
return { true };
}
