void test1()
{
Context context;
StoragePtr table = StorageSystemNumbers::create("numbers");
Names column_names;
column_names.push_back("number");
QueryProcessingStage::Enum stage1;
QueryProcessingStage::Enum stage2;
QueryProcessingStage::Enum stage3;
BlockInputStreams streams;
streams.emplace_back(std::make_shared<LimitBlockInputStream>(table->read(column_names, 0, context, Settings(), stage1, 1)[0], 30, 30000));
streams.emplace_back(std::make_shared<LimitBlockInputStream>(table->read(column_names, 0, context, Settings(), stage2, 1)[0], 30, 2000));
streams.emplace_back(std::make_shared<LimitBlockInputStream>(table->read(column_names, 0, context, Settings(), stage3, 1)[0], 30, 100));
UnionBlockInputStream<> union_stream(streams, nullptr, 2);
WriteBufferFromFileDescriptor wb(STDERR_FILENO);
Block sample = table->getSampleBlock();
BlockOutputStreamPtr out = context.getOutputFormat("TabSeparated", wb, sample);
while (Block block = union_stream.read())
{
out->write(block);
wb.next();
}
//copyData(union_stream, *out);
}
int main(int argc, char ** argv)
try
{
using namespace DB;
StoragePtr table = StorageSystemNumbers::create("Numbers");
Names column_names;
column_names.push_back("number");
Block sample;
ColumnWithTypeAndName col;
col.type = std::make_shared<DataTypeUInt64>();
sample.insert(std::move(col));
WriteBufferFromOStream out_buf(std::cout);
QueryProcessingStage::Enum stage;
LimitBlockInputStream input(table->read(column_names, 0, Context{}, Settings(), stage, 10)[0], 10, 96);
RowOutputStreamPtr output_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample);
BlockOutputStreamFromRowOutputStream output(output_);
copyData(input, output);
return 0;
}
catch (const DB::Exception & e)
{
std::cerr << e.what() << ", " << e.displayText() << std::endl;
return 1;
}
int main(int argc, char ** argv)
try
{
using namespace DB;
size_t n = argc == 2 ? parse<UInt64>(argv[1]) : 10ULL;
std::string input = "SELECT number, number / 3, number * number";
ParserSelectQuery parser;
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "");
Context context;
ExpressionAnalyzer analyzer(ast, context, {}, {NameAndTypePair("number", std::make_shared<DataTypeUInt64>())});
ExpressionActionsChain chain;
analyzer.appendSelect(chain, false);
analyzer.appendProjectResult(chain, false);
chain.finalize();
ExpressionActionsPtr expression = chain.getLastActions();
StoragePtr table = StorageSystemNumbers::create("Numbers");
Names column_names;
column_names.push_back("number");
QueryProcessingStage::Enum stage;
BlockInputStreamPtr in;
in = table->read(column_names, 0, context, Settings(), stage)[0];
in = std::make_shared<ExpressionBlockInputStream>(in, expression);
in = std::make_shared<LimitBlockInputStream>(in, 10, std::max(static_cast<Int64>(0), static_cast<Int64>(n) - 10));
WriteBufferFromOStream out1(std::cout);
RowOutputStreamPtr out2 = std::make_shared<TabSeparatedRowOutputStream>(out1, expression->getSampleBlock());
BlockOutputStreamFromRowOutputStream out(out2);
{
Stopwatch stopwatch;
stopwatch.start();
copyData(*in, out);
stopwatch.stop();
std::cout << std::fixed << std::setprecision(2)
<< "Elapsed " << stopwatch.elapsedSeconds() << " sec."
<< ", " << n / stopwatch.elapsedSeconds() << " rows/sec."
<< std::endl;
}
return 0;
}
catch (const DB::Exception & e)
{
std::cerr << e.what() << ", " << e.displayText() << std::endl;
throw;
}
int main(int argc, char ** argv)
{
using namespace DB;
try
{
NamesAndTypesList names_and_types_list
{
{"WatchID", std::make_shared<DataTypeUInt64>()},
{"JavaEnable", std::make_shared<DataTypeUInt8>()},
{"Title", std::make_shared<DataTypeString>()},
{"EventTime", std::make_shared<DataTypeDateTime>()},
{"CounterID", std::make_shared<DataTypeUInt32>()},
{"ClientIP", std::make_shared<DataTypeUInt32>()},
{"RegionID", std::make_shared<DataTypeUInt32>()},
{"UniqID", std::make_shared<DataTypeUInt64>()},
{"CounterClass", std::make_shared<DataTypeUInt8>()},
{"OS", std::make_shared<DataTypeUInt8>()},
{"UserAgent", std::make_shared<DataTypeUInt8>()},
{"URL", std::make_shared<DataTypeString>()},
{"Referer", std::make_shared<DataTypeString>()},
{"ResolutionWidth", std::make_shared<DataTypeUInt16>()},
{"ResolutionHeight", std::make_shared<DataTypeUInt16>()},
{"ResolutionDepth", std::make_shared<DataTypeUInt8>()},
{"FlashMajor", std::make_shared<DataTypeUInt8>()},
{"FlashMinor", std::make_shared<DataTypeUInt8>()},
{"FlashMinor2", std::make_shared<DataTypeString>()},
{"NetMajor", std::make_shared<DataTypeUInt8>()},
{"NetMinor", std::make_shared<DataTypeUInt8>()},
{"UserAgentMajor", std::make_shared<DataTypeUInt16>()},
{"UserAgentMinor", std::make_shared<DataTypeFixedString>(2)},
{"CookieEnable", std::make_shared<DataTypeUInt8>()},
{"JavascriptEnable", std::make_shared<DataTypeUInt8>()},
{"IsMobile", std::make_shared<DataTypeUInt8>()},
{"MobilePhone", std::make_shared<DataTypeUInt8>()},
{"MobilePhoneModel", std::make_shared<DataTypeString>()},
{"Params", std::make_shared<DataTypeString>()},
{"IPNetworkID", std::make_shared<DataTypeUInt32>()},
{"TraficSourceID", std::make_shared<DataTypeInt8>()},
{"SearchEngineID", std::make_shared<DataTypeUInt16>()},
{"SearchPhrase", std::make_shared<DataTypeString>()},
{"AdvEngineID", std::make_shared<DataTypeUInt8>()},
{"IsArtifical", std::make_shared<DataTypeUInt8>()},
{"WindowClientWidth", std::make_shared<DataTypeUInt16>()},
{"WindowClientHeight", std::make_shared<DataTypeUInt16>()},
{"ClientTimeZone", std::make_shared<DataTypeInt16>()},
{"ClientEventTime", std::make_shared<DataTypeDateTime>()},
{"SilverlightVersion1", std::make_shared<DataTypeUInt8>()},
{"SilverlightVersion2", std::make_shared<DataTypeUInt8>()},
{"SilverlightVersion3", std::make_shared<DataTypeUInt32>()},
{"SilverlightVersion4", std::make_shared<DataTypeUInt16>()},
{"PageCharset", std::make_shared<DataTypeString>()},
{"CodeVersion", std::make_shared<DataTypeUInt32>()},
{"IsLink", std::make_shared<DataTypeUInt8>()},
{"IsDownload", std::make_shared<DataTypeUInt8>()},
{"IsNotBounce", std::make_shared<DataTypeUInt8>()},
{"FUniqID", std::make_shared<DataTypeUInt64>()},
{"OriginalURL", std::make_shared<DataTypeString>()},
{"HID", std::make_shared<DataTypeUInt32>()},
{"IsOldCounter", std::make_shared<DataTypeUInt8>()},
{"IsEvent", std::make_shared<DataTypeUInt8>()},
{"IsParameter", std::make_shared<DataTypeUInt8>()},
{"DontCountHits", std::make_shared<DataTypeUInt8>()},
{"WithHash", std::make_shared<DataTypeUInt8>()},
};
Context context;
std::string input = "SELECT UniqID, URL, CounterID, IsLink WHERE URL = 'http://mail.yandex.ru/neo2/#inbox'";
ParserSelectQuery parser;
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "");
formatAST(*ast, std::cerr);
std::cerr << std::endl;
std::cerr << ast->getTreeID() << std::endl;
/// create an object of an existing hit log table
StoragePtr table = StorageLog::create("./", "HitLog", std::make_shared<NamesAndTypesList>(names_and_types_list));
/// read from it, apply the expression, filter, and write in tsv form to the console
ExpressionAnalyzer analyzer(ast, context, nullptr, names_and_types_list);
ExpressionActionsChain chain;
analyzer.appendSelect(chain, false);
analyzer.appendWhere(chain, false);
chain.finalize();
ExpressionActionsPtr expression = chain.getLastActions();
Names column_names
{
"UniqID",
"URL",
"CounterID",
"IsLink",
};
QueryProcessingStage::Enum stage;
BlockInputStreamPtr in = table->read(column_names, 0, context, stage)[0];
in = std::make_shared<FilterBlockInputStream>(in, expression, 4);
//in = std::make_shared<LimitBlockInputStream>(in, 10, 0);
WriteBufferFromOStream ob(std::cout);
RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(ob, expression->getSampleBlock());
BlockOutputStreamFromRowOutputStream out(out_);
copyData(*in, out);
}
catch (const Exception & e)
{
std::cerr << e.what() << ", " << e.displayText() << std::endl;
return 1;
}
return 0;
}
int main(int argc, char ** argv)
try
{
NamesAndTypesList names_and_types_list
{
{"WatchID", std::make_shared<DataTypeUInt64>()},
{"JavaEnable", std::make_shared<DataTypeUInt8>()},
{"Title", std::make_shared<DataTypeString>()},
{"EventTime", std::make_shared<DataTypeDateTime>()},
{"CounterID", std::make_shared<DataTypeUInt32>()},
{"ClientIP", std::make_shared<DataTypeUInt32>()},
{"RegionID", std::make_shared<DataTypeUInt32>()},
{"UniqID", std::make_shared<DataTypeUInt64>()},
{"CounterClass", std::make_shared<DataTypeUInt8>()},
{"OS", std::make_shared<DataTypeUInt8>()},
{"UserAgent", std::make_shared<DataTypeUInt8>()},
{"URL", std::make_shared<DataTypeString>()},
{"Referer", std::make_shared<DataTypeString>()},
{"ResolutionWidth", std::make_shared<DataTypeUInt16>()},
{"ResolutionHeight", std::make_shared<DataTypeUInt16>()},
{"ResolutionDepth", std::make_shared<DataTypeUInt8>()},
{"FlashMajor", std::make_shared<DataTypeUInt8>()},
{"FlashMinor", std::make_shared<DataTypeUInt8>()},
{"FlashMinor2", std::make_shared<DataTypeString>()},
{"NetMajor", std::make_shared<DataTypeUInt8>()},
{"NetMinor", std::make_shared<DataTypeUInt8>()},
{"UserAgentMajor", std::make_shared<DataTypeUInt16>()},
{"UserAgentMinor", std::make_shared<DataTypeFixedString>(2)},
{"CookieEnable", std::make_shared<DataTypeUInt8>()},
{"JavascriptEnable", std::make_shared<DataTypeUInt8>()},
{"IsMobile", std::make_shared<DataTypeUInt8>()},
{"MobilePhone", std::make_shared<DataTypeUInt8>()},
{"MobilePhoneModel", std::make_shared<DataTypeString>()},
{"Params", std::make_shared<DataTypeString>()},
{"IPNetworkID", std::make_shared<DataTypeUInt32>()},
{"TraficSourceID", std::make_shared<DataTypeInt8>()},
{"SearchEngineID", std::make_shared<DataTypeUInt16>()},
{"SearchPhrase", std::make_shared<DataTypeString>()},
{"AdvEngineID", std::make_shared<DataTypeUInt8>()},
{"IsArtifical", std::make_shared<DataTypeUInt8>()},
{"WindowClientWidth", std::make_shared<DataTypeUInt16>()},
{"WindowClientHeight", std::make_shared<DataTypeUInt16>()},
{"ClientTimeZone", std::make_shared<DataTypeInt16>()},
{"ClientEventTime", std::make_shared<DataTypeDateTime>()},
{"SilverlightVersion1", std::make_shared<DataTypeUInt8>()},
{"SilverlightVersion2", std::make_shared<DataTypeUInt8>()},
{"SilverlightVersion3", std::make_shared<DataTypeUInt32>()},
{"SilverlightVersion4", std::make_shared<DataTypeUInt16>()},
{"PageCharset", std::make_shared<DataTypeString>()},
{"CodeVersion", std::make_shared<DataTypeUInt32>()},
{"IsLink", std::make_shared<DataTypeUInt8>()},
{"IsDownload", std::make_shared<DataTypeUInt8>()},
{"IsNotBounce", std::make_shared<DataTypeUInt8>()},
{"FUniqID", std::make_shared<DataTypeUInt64>()},
{"OriginalURL", std::make_shared<DataTypeString>()},
{"HID", std::make_shared<DataTypeUInt32>()},
{"IsOldCounter", std::make_shared<DataTypeUInt8>()},
{"IsEvent", std::make_shared<DataTypeUInt8>()},
{"IsParameter", std::make_shared<DataTypeUInt8>()},
{"DontCountHits", std::make_shared<DataTypeUInt8>()},
{"WithHash", std::make_shared<DataTypeUInt8>()},
};
DataTypes data_types;
Names column_names;
for (const auto & name_type : names_and_types_list)
{
data_types.push_back(name_type.type);
column_names.push_back(name_type.name);
}
/// create a hit log table
StoragePtr table = StorageLog::create(
"./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
table->startup();
/// create a description of how to read data from the tab separated dump
Block sample;
for (const auto & name_type : names_and_types_list)
{
ColumnWithTypeAndName elem;
elem.name = name_type.name;
elem.type = name_type.type;
elem.column = elem.type->createColumn();
sample.insert(std::move(elem));
}
FormatSettings format_settings;
/// read the data from tsv file and simultaneously write to table
if (argc == 2 && 0 == strcmp(argv[1], "write"))
{
ReadBufferFromFileDescriptor in_buf(STDIN_FILENO);
RowInputStreamPtr in_ = std::make_shared<TabSeparatedRowInputStream>(in_buf, sample, false, false, format_settings);
BlockInputStreamFromRowInputStream in(in_, sample, DEFAULT_INSERT_BLOCK_SIZE, 0, 0);
BlockOutputStreamPtr out = table->write({}, {});
copyData(in, *out);
}
/// read from it
if (argc == 2 && 0 == strcmp(argv[1], "read"))
{
WriteBufferFromFileDescriptor out_buf(STDOUT_FILENO);
QueryProcessingStage::Enum stage;
BlockInputStreamPtr in = table->read(column_names, {}, Context::createGlobal(), stage, 8192, 1)[0];
RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample, false, false, format_settings);
BlockOutputStreamFromRowOutputStream out(out_, sample);
copyData(*in, out);
}
return 0;
}
catch (const Exception & e)
{
std::cerr << e.what() << ", " << e.displayText() << std::endl;
throw;
}
int main(int argc, char ** argv)
try
{
using namespace DB;
const size_t rows = 10000000;
/// создаём таблицу с парой столбцов
NamesAndTypesListPtr names_and_types = std::make_shared<NamesAndTypesList>();
names_and_types->push_back(NameAndTypePair("a", std::make_shared<DataTypeUInt64>()));
names_and_types->push_back(NameAndTypePair("b", std::make_shared<DataTypeUInt8>()));
StoragePtr table = StorageLog::create("./", "test", names_and_types);
/// пишем в неё
{
Block block;
ColumnWithTypeAndName column1;
column1.name = "a";
column1.type = table->getDataTypeByName("a");
column1.column = column1.type->createColumn();
ColumnUInt64::Container_t & vec1 = typeid_cast<ColumnUInt64&>(*column1.column).getData();
vec1.resize(rows);
for (size_t i = 0; i < rows; ++i)
vec1[i] = i;
block.insert(column1);
ColumnWithTypeAndName column2;
column2.name = "b";
column2.type = table->getDataTypeByName("b");
column2.column = column2.type->createColumn();
ColumnUInt8::Container_t & vec2 = typeid_cast<ColumnUInt8&>(*column2.column).getData();
vec2.resize(rows);
for (size_t i = 0; i < rows; ++i)
vec2[i] = i * 2;
block.insert(column2);
BlockOutputStreamPtr out = table->write({}, {});
out->write(block);
}
/// читаем из неё
{
Names column_names;
column_names.push_back("a");
column_names.push_back("b");
QueryProcessingStage::Enum stage;
BlockInputStreamPtr in = table->read(column_names, 0, Context{}, Settings(), stage)[0];
Block sample;
{
ColumnWithTypeAndName col;
col.type = std::make_shared<DataTypeUInt64>();
sample.insert(col);
}
{
ColumnWithTypeAndName col;
col.type = std::make_shared<DataTypeUInt8>();
sample.insert(col);
}
WriteBufferFromOStream out_buf(std::cout);
LimitBlockInputStream in_limit(in, 10, 0);
RowOutputStreamPtr output_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample);
BlockOutputStreamFromRowOutputStream output(output_);
copyData(in_limit, output);
}
return 0;
}
catch (const DB::Exception & e)
{
std::cerr << e.what() << ", " << e.displayText() << std::endl;
return 1;
}
int main(int argc, char ** argv)
try
{
using namespace DB;
NamesAndTypesList names_and_types_list
{
{"WatchID", std::make_shared<DataTypeUInt64>()},
{"JavaEnable", std::make_shared<DataTypeUInt8>()},
{"Title", std::make_shared<DataTypeString>()},
{"EventTime", std::make_shared<DataTypeDateTime>()},
{"CounterID", std::make_shared<DataTypeUInt32>()},
{"ClientIP", std::make_shared<DataTypeUInt32>()},
{"RegionID", std::make_shared<DataTypeUInt32>()},
{"UniqID", std::make_shared<DataTypeUInt64>()},
{"CounterClass", std::make_shared<DataTypeUInt8>()},
{"OS", std::make_shared<DataTypeUInt8>()},
{"UserAgent", std::make_shared<DataTypeUInt8>()},
{"URL", std::make_shared<DataTypeString>()},
{"Referer", std::make_shared<DataTypeString>()},
{"ResolutionWidth", std::make_shared<DataTypeUInt16>()},
{"ResolutionHeight", std::make_shared<DataTypeUInt16>()},
{"ResolutionDepth", std::make_shared<DataTypeUInt8>()},
{"FlashMajor", std::make_shared<DataTypeUInt8>()},
{"FlashMinor", std::make_shared<DataTypeUInt8>()},
{"FlashMinor2", std::make_shared<DataTypeString>()},
{"NetMajor", std::make_shared<DataTypeUInt8>()},
{"NetMinor", std::make_shared<DataTypeUInt8>()},
{"UserAgentMajor", std::make_shared<DataTypeUInt16>()},
{"UserAgentMinor", std::make_shared<DataTypeFixedString>(2)},
{"CookieEnable", std::make_shared<DataTypeUInt8>()},
{"JavascriptEnable", std::make_shared<DataTypeUInt8>()},
{"IsMobile", std::make_shared<DataTypeUInt8>()},
{"MobilePhone", std::make_shared<DataTypeUInt8>()},
{"MobilePhoneModel", std::make_shared<DataTypeString>()},
{"Params", std::make_shared<DataTypeString>()},
{"IPNetworkID", std::make_shared<DataTypeUInt32>()},
{"TraficSourceID", std::make_shared<DataTypeInt8>()},
{"SearchEngineID", std::make_shared<DataTypeUInt16>()},
{"SearchPhrase", std::make_shared<DataTypeString>()},
{"AdvEngineID", std::make_shared<DataTypeUInt8>()},
{"IsArtifical", std::make_shared<DataTypeUInt8>()},
{"WindowClientWidth", std::make_shared<DataTypeUInt16>()},
{"WindowClientHeight", std::make_shared<DataTypeUInt16>()},
{"ClientTimeZone", std::make_shared<DataTypeInt16>()},
{"ClientEventTime", std::make_shared<DataTypeDateTime>()},
{"SilverlightVersion1", std::make_shared<DataTypeUInt8>()},
{"SilverlightVersion2", std::make_shared<DataTypeUInt8>()},
{"SilverlightVersion3", std::make_shared<DataTypeUInt32>()},
{"SilverlightVersion4", std::make_shared<DataTypeUInt16>()},
{"PageCharset", std::make_shared<DataTypeString>()},
{"CodeVersion", std::make_shared<DataTypeUInt32>()},
{"IsLink", std::make_shared<DataTypeUInt8>()},
{"IsDownload", std::make_shared<DataTypeUInt8>()},
{"IsNotBounce", std::make_shared<DataTypeUInt8>()},
{"FUniqID", std::make_shared<DataTypeUInt64>()},
{"OriginalURL", std::make_shared<DataTypeString>()},
{"HID", std::make_shared<DataTypeUInt32>()},
{"IsOldCounter", std::make_shared<DataTypeUInt8>()},
{"IsEvent", std::make_shared<DataTypeUInt8>()},
{"IsParameter", std::make_shared<DataTypeUInt8>()},
{"DontCountHits", std::make_shared<DataTypeUInt8>()},
{"WithHash", std::make_shared<DataTypeUInt8>()},
};
Names column_names;
for (const auto & name_type : names_and_types_list)
column_names.push_back(name_type.name);
/// create an object of an existing hit log table
StoragePtr table = StorageLog::create("./", "HitLog", std::make_shared<NamesAndTypesList>(names_and_types_list));
/// read from it
if (argc == 2 && 0 == strcmp(argv[1], "read"))
{
QueryProcessingStage::Enum stage;
BlockInputStreamPtr in = table->read(column_names, 0, Context{}, stage)[0];
WriteBufferFromFileDescriptor out1(STDOUT_FILENO);
CompressedWriteBuffer out2(out1);
NativeBlockOutputStream out3(out2, ClickHouseRevision::get());
copyData(*in, out3);
}
/// read the data from the native file and simultaneously write to the table
if (argc == 2 && 0 == strcmp(argv[1], "write"))
{
ReadBufferFromFileDescriptor in1(STDIN_FILENO);
CompressedReadBuffer in2(in1);
NativeBlockInputStream in3(in2, ClickHouseRevision::get());
BlockOutputStreamPtr out = table->write({}, {});
copyData(in3, *out);
}
return 0;
}
catch (const DB::Exception & e)
{
std::cerr << e.what() << ", " << e.displayText() << std::endl;
throw;
}
BlockInputStreams StorageMerge::createSourceStreams(const SelectQueryInfo & query_info, const QueryProcessingStage::Enum & processed_stage,
const size_t max_block_size, const Block & header, const StoragePtr & storage,
const TableStructureReadLockPtr & struct_lock, Names & real_column_names,
Context & modified_context, size_t streams_num, bool has_table_virtual_column,
bool concat_streams)
{
SelectQueryInfo modified_query_info = query_info;
modified_query_info.query = query_info.query->clone();
VirtualColumnUtils::rewriteEntityInAst(modified_query_info.query, "_table", storage ? storage->getTableName() : "");
if (!storage)
return BlockInputStreams{
InterpreterSelectQuery(modified_query_info.query, modified_context, std::make_shared<OneBlockInputStream>(header),
processed_stage, true).execute().in};
BlockInputStreams source_streams;
if (processed_stage <= storage->getQueryProcessingStage(modified_context))
{
/// If there are only virtual columns in query, you must request at least one other column.
if (real_column_names.size() ==0)
real_column_names.push_back(ExpressionActions::getSmallestColumn(storage->getColumns().getAllPhysical()));
source_streams = storage->read(real_column_names, modified_query_info, modified_context, processed_stage, max_block_size,
UInt32(streams_num));
}
else if (processed_stage > storage->getQueryProcessingStage(modified_context))
{
typeid_cast<ASTSelectQuery *>(modified_query_info.query.get())->replaceDatabaseAndTable(source_database, storage->getTableName());
/// Maximum permissible parallelism is streams_num
modified_context.getSettingsRef().max_threads = UInt64(streams_num);
modified_context.getSettingsRef().max_streams_to_max_threads_ratio = 1;
InterpreterSelectQuery interpreter{modified_query_info.query, modified_context, Names{}, processed_stage};
BlockInputStreamPtr interpreter_stream = interpreter.execute().in;
/** Materialization is needed, since from distributed storage the constants come materialized.
* If you do not do this, different types (Const and non-Const) columns will be produced in different threads,
* And this is not allowed, since all code is based on the assumption that in the block stream all types are the same.
*/
source_streams.emplace_back(std::make_shared<MaterializingBlockInputStream>(interpreter_stream));
}
if (!source_streams.empty())
{
if (concat_streams)
{
BlockInputStreamPtr stream =
source_streams.size() > 1 ? std::make_shared<ConcatBlockInputStream>(source_streams) : source_streams[0];
source_streams.resize(1);
source_streams[0] = stream;
}
for (BlockInputStreamPtr & source_stream : source_streams)
{
if (has_table_virtual_column)
source_stream = std::make_shared<AddingConstColumnBlockInputStream<String>>(
source_stream, std::make_shared<DataTypeString>(), storage->getTableName(), "_table");
/// Subordinary tables could have different but convertible types, like numeric types of different width.
/// We must return streams with structure equals to structure of Merge table.
convertingSourceStream(header, modified_context, modified_query_info.query, source_stream, processed_stage);
source_stream->addTableLock(struct_lock);
}
}
return source_streams;
}
void test2()
{
Context context;
StoragePtr table = StorageSystemNumbers::create("numbers");
Names column_names;
column_names.push_back("number");
QueryProcessingStage::Enum stage1;
QueryProcessingStage::Enum stage2;
QueryProcessingStage::Enum stage3;
BlockExtraInfo extra_info1;
extra_info1.host = "host1";
extra_info1.resolved_address = "127.0.0.1";
extra_info1.port = 9000;
extra_info1.user = "******";
BlockExtraInfo extra_info2;
extra_info2.host = "host2";
extra_info2.resolved_address = "127.0.0.2";
extra_info2.port = 9001;
extra_info2.user = "******";
BlockExtraInfo extra_info3;
extra_info3.host = "host3";
extra_info3.resolved_address = "127.0.0.3";
extra_info3.port = 9003;
extra_info3.user = "******";
BlockInputStreams streams;
BlockInputStreamPtr stream1 = std::make_shared<LimitBlockInputStream>(table->read(column_names, 0, context, Settings(), stage1, 1)[0], 30, 30000);
stream1 = std::make_shared<BlockExtraInfoInputStream>(stream1, extra_info1);
streams.emplace_back(stream1);
BlockInputStreamPtr stream2 = std::make_shared<LimitBlockInputStream>(table->read(column_names, 0, context, Settings(), stage2, 1)[0], 30, 2000);
stream2 = std::make_shared<BlockExtraInfoInputStream>(stream2, extra_info2);
streams.emplace_back(stream2);
BlockInputStreamPtr stream3 = std::make_shared<LimitBlockInputStream>(table->read(column_names, 0, context, Settings(), stage3, 1)[0], 30, 100);
stream3 = std::make_shared<BlockExtraInfoInputStream>(stream3, extra_info3);
streams.emplace_back(stream3);
UnionBlockInputStream<StreamUnionMode::ExtraInfo> union_stream(streams, nullptr, 2);
auto getSampleBlock = []()
{
Block block;
ColumnWithTypeAndName col;
col.name = "number";
col.type = std::make_shared<DataTypeUInt64>();
col.column = col.type->createColumn();
block.insert(col);
col.name = "host_name";
col.type = std::make_shared<DataTypeString>();
col.column = col.type->createColumn();
block.insert(col);
col.name = "host_address";
col.type = std::make_shared<DataTypeString>();
col.column = col.type->createColumn();
block.insert(col);
col.name = "port";
col.type = std::make_shared<DataTypeUInt16>();
col.column = col.type->createColumn();
block.insert(col);
col.name = "user";
col.type = std::make_shared<DataTypeString>();
col.column = col.type->createColumn();
block.insert(col);
return block;
};
WriteBufferFromFileDescriptor wb(STDERR_FILENO);
Block sample = getSampleBlock();
BlockOutputStreamPtr out = context.getOutputFormat("TabSeparated", wb, sample);
while (Block block = union_stream.read())
{
const auto & col = block.safeGetByPosition(0);
auto extra_info = union_stream.getBlockExtraInfo();
ColumnPtr host_name_column = std::make_shared<ColumnString>();
ColumnPtr host_address_column = std::make_shared<ColumnString>();
ColumnPtr port_column = std::make_shared<ColumnUInt16>();
ColumnPtr user_column = std::make_shared<ColumnString>();
size_t row_count = block.rows();
for (size_t i = 0; i < row_count; ++i)
{
host_name_column->insert(extra_info.resolved_address);
host_address_column->insert(extra_info.host);
port_column->insert(static_cast<UInt64>(extra_info.port));
user_column->insert(extra_info.user);
}
Block out_block;
out_block.insert(ColumnWithTypeAndName(col.column->clone(), col.type, col.name));
out_block.insert(ColumnWithTypeAndName(host_name_column, std::make_shared<DataTypeString>(), "host_name"));
out_block.insert(ColumnWithTypeAndName(host_address_column, std::make_shared<DataTypeString>(), "host_address"));
out_block.insert(ColumnWithTypeAndName(port_column, std::make_shared<DataTypeUInt16>(), "port"));
out_block.insert(ColumnWithTypeAndName(user_column, std::make_shared<DataTypeString>(), "user"));
out->write(out_block);
wb.next();
}
//copyData(union_stream, *out);
}
int main(int argc, char ** argv)
try
{
using namespace DB;
std::string input = "SELECT number, number % 10000000 == 1";
ParserSelectQuery parser;
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "");
formatAST(*ast, std::cerr);
std::cerr << std::endl;
Context context;
ExpressionAnalyzer analyzer(ast, context, {}, {NameAndTypePair("number", std::make_shared<DataTypeUInt64>())});
ExpressionActionsChain chain;
analyzer.appendSelect(chain, false);
analyzer.appendProjectResult(chain, false);
chain.finalize();
ExpressionActionsPtr expression = chain.getLastActions();
StoragePtr table = StorageSystemNumbers::create("Numbers");
Names column_names;
column_names.push_back("number");
QueryProcessingStage::Enum stage;
BlockInputStreamPtr in = table->read(column_names, 0, context, stage)[0];
ForkBlockInputStreams fork(in);
BlockInputStreamPtr in1 = fork.createInput();
BlockInputStreamPtr in2 = fork.createInput();
in1 = std::make_shared<FilterBlockInputStream>(in1, expression, 1);
in1 = std::make_shared<LimitBlockInputStream>(in1, 10, 0);
in2 = std::make_shared<FilterBlockInputStream>(in2, expression, 1);
in2 = std::make_shared<LimitBlockInputStream>(in2, 20, 5);
Block out_sample = expression->getSampleBlock();
WriteBufferFromOStream ob1(std::cout);
WriteBufferFromOStream ob2(std::cerr);
BlockOutputStreamPtr out1 = context.getOutputFormat("TabSeparated", ob1, out_sample);
BlockOutputStreamPtr out2 = context.getOutputFormat("TabSeparated", ob2, out_sample);
std::thread thr1(std::bind(thread1, in1, out1, std::ref(ob1)));
std::thread thr2(std::bind(thread2, in2, out2, std::ref(ob2)));
fork.run();
thr1.join();
thr2.join();
return 0;
}
catch (const DB::Exception & e)
{
std::cerr << e.what() << ", " << e.displayText() << std::endl;
throw;
}