void IndexForNativeFormat::read(ReadBuffer & istr, const NameSet & required_columns)
{
while (!istr.eof())
{
blocks.emplace_back();
IndexOfBlockForNativeFormat & block = blocks.back();
readVarUInt(block.num_columns, istr);
readVarUInt(block.num_rows, istr);
if (block.num_columns < required_columns.size())
throw Exception("Index contain less than required columns", ErrorCodes::INCORRECT_INDEX);
for (size_t i = 0; i < block.num_columns; ++i)
{
IndexOfOneColumnForNativeFormat column_index;
readBinary(column_index.name, istr);
readBinary(column_index.type, istr);
readBinary(column_index.location.offset_in_compressed_file, istr);
readBinary(column_index.location.offset_in_decompressed_block, istr);
if (required_columns.count(column_index.name))
block.columns.push_back(std::move(column_index));
}
if (block.columns.size() < required_columns.size())
throw Exception("Index contain less than required columns", ErrorCodes::INCORRECT_INDEX);
if (block.columns.size() > required_columns.size())
throw Exception("Index contain duplicate columns", ErrorCodes::INCORRECT_INDEX);
block.num_columns = block.columns.size();
}
}
void BlockStreamProfileInfo::read(ReadBuffer & in)
{
readVarUInt(rows, in);
readVarUInt(blocks, in);
readVarUInt(bytes, in);
readBinary(applied_limit, in);
readVarUInt(rows_before_limit, in);
readBinary(calculated_rows_before_limit, in);
}
void DataTypeArray::deserializeBinary(IColumn & column, ReadBuffer & istr) const
{
ColumnArray & column_array = static_cast<ColumnArray &>(column);
ColumnArray::Offsets & offsets = column_array.getOffsets();
size_t size;
readVarUInt(size, istr);
IColumn & nested_column = column_array.getData();
size_t i = 0;
try
{
for (; i < size; ++i)
nested->deserializeBinary(nested_column, istr);
}
catch (...)
{
if (i)
nested_column.popBack(i);
throw;
}
offsets.push_back((offsets.empty() ? 0 : offsets.back()) + size);
}
void DataTypeAggregateFunction::deserializeBinary(Field & field, ReadBuffer & istr) const
{
UInt64 size;
readVarUInt(size, istr);
field = String();
String & s = get<String &>(field);
s.resize(size);
istr.readStrict(&s[0], size);
}
void DataTypeArray::deserializeBinary(Field & field, ReadBuffer & istr) const
{
size_t size;
readVarUInt(size, istr);
field = Array(size);
Array & arr = get<Array &>(field);
for (size_t i = 0; i < size; ++i)
nested->deserializeBinary(arr[i], istr);
}
ReshardingJob::ReshardingJob(const std::string & serialized_job)
{
ReadBufferFromString buf{serialized_job};
readBinary(database_name, buf);
readBinary(table_name, buf);
readBinary(partition, buf);
std::string expr;
readBinary(expr, buf);
IParser::Pos pos = expr.data();
IParser::Pos max_parsed_pos = pos;
const char * end = pos + expr.size();
ParserExpressionWithOptionalAlias parser(false);
Expected expected = "";
if (!parser.parse(pos, end, sharding_key_expr, max_parsed_pos, expected))
throw Exception{"ReshardingJob: Internal error", ErrorCodes::LOGICAL_ERROR};
readBinary(coordinator_id, buf);
readVarUInt(block_number, buf);
readBinary(do_copy, buf);
size_t s;
readVarUInt(s, buf);
for (size_t i = 0; i < s; ++i)
{
std::string path;
readBinary(path, buf);
UInt64 weight;
readVarUInt(weight, buf);
paths.emplace_back(path, weight);
}
}
Block NativeBlockInputStream::readImpl()
{
Block res;
const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
if (use_index && index_block_it == index_block_end)
return res;
if (istr.eof())
{
if (use_index)
throw Exception("Input doesn't contain all data for index.", ErrorCodes::CANNOT_READ_ALL_DATA);
return res;
}
/// Дополнительная информация о блоке.
if (server_revision >= DBMS_MIN_REVISION_WITH_BLOCK_INFO)
res.info.read(istr);
/// Размеры
size_t columns = 0;
size_t rows = 0;
if (!use_index)
{
readVarUInt(columns, istr);
readVarUInt(rows, istr);
}
else
{
columns = index_block_it->num_columns;
rows = index_block_it->num_rows;
}
for (size_t i = 0; i < columns; ++i)
{
if (use_index)
{
/// Если текущая позиция и так какая требуется, то реального seek-а не происходит.
istr_concrete->seek(index_column_it->location.offset_in_compressed_file, index_column_it->location.offset_in_decompressed_block);
}
ColumnWithTypeAndName column;
/// Name
readBinary(column.name, istr);
/// Type
String type_name;
readBinary(type_name, istr);
column.type = data_type_factory.get(type_name);
if (use_index)
{
/// Index allows to do more checks.
if (index_column_it->name != column.name)
throw Exception("Index points to column with wrong name: corrupted index or data", ErrorCodes::INCORRECT_INDEX);
if (index_column_it->type != type_name)
throw Exception("Index points to column with wrong type: corrupted index or data", ErrorCodes::INCORRECT_INDEX);
}
/// Data
column.column = column.type->createColumn();
if (rows) /// If no rows, nothing to read.
readData(*column.type, *column.column, istr, rows);
res.insert(column);
if (use_index)
++index_column_it;
}
if (use_index)
{
if (index_column_it != index_block_it->columns.end())
throw Exception("Inconsistent index: not all columns were read", ErrorCodes::INCORRECT_INDEX);
++index_block_it;
if (index_block_it != index_block_end)
index_column_it = index_block_it->columns.begin();
}
return res;
}
