void MergeTreeReader::readData( const String & name, const IDataType & type, IColumn & column, size_t from_mark, bool continue_reading, size_t max_rows_to_read, bool with_offsets) { IDataType::InputStreamGetter stream_getter = [&] (const IDataType::SubstreamPath & path) -> ReadBuffer * { /// If offsets for arrays have already been read. if (!with_offsets && !path.empty() && path.back().type == IDataType::Substream::ArraySizes) return nullptr; String stream_name = IDataType::getFileNameForStream(name, path); auto it = streams.find(stream_name); if (it == streams.end()) return nullptr; Stream & stream = *it->second; if (!continue_reading) stream.seekToMark(from_mark); return stream.data_buffer; }; double & avg_value_size_hint = avg_value_size_hints[name]; type.deserializeBinaryBulkWithMultipleStreams(column, stream_getter, max_rows_to_read, avg_value_size_hint, true, {}); IDataType::updateAvgValueSizeHint(column, avg_value_size_hint); }
void TinyLogBlockInputStream::readData(const String & name, const IDataType & type, IColumn & column, UInt64 limit) { IDataType::DeserializeBinaryBulkSettings settings; /// TODO Use avg_value_size_hint. settings.getter = [&] (const IDataType::SubstreamPath & path) -> ReadBuffer * { String stream_name = IDataType::getFileNameForStream(name, path); if (!streams.count(stream_name)) streams[stream_name] = std::make_unique<Stream>(storage.files[stream_name].data_file.path(), max_read_buffer_size); return &streams[stream_name]->compressed; }; if (deserialize_states.count(name) == 0) type.deserializeBinaryBulkStatePrefix(settings, deserialize_states[name]); type.deserializeBinaryBulkWithMultipleStreams(column, limit, settings, deserialize_states[name]); }