int main(int argc, char ** argv)
try
{
using namespace DB;
size_t n = argc == 2 ? parse<UInt64>(argv[1]) : 10ULL;
std::string input = "SELECT number, number / 3, number * number";
ParserSelectQuery parser;
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "");
Context context;
ExpressionAnalyzer analyzer(ast, context, {}, {NameAndTypePair("number", std::make_shared<DataTypeUInt64>())});
ExpressionActionsChain chain;
analyzer.appendSelect(chain, false);
analyzer.appendProjectResult(chain, false);
chain.finalize();
ExpressionActionsPtr expression = chain.getLastActions();
StoragePtr table = StorageSystemNumbers::create("Numbers");
Names column_names;
column_names.push_back("number");
QueryProcessingStage::Enum stage;
BlockInputStreamPtr in;
in = table->read(column_names, 0, context, Settings(), stage)[0];
in = std::make_shared<ExpressionBlockInputStream>(in, expression);
in = std::make_shared<LimitBlockInputStream>(in, 10, std::max(static_cast<Int64>(0), static_cast<Int64>(n) - 10));
WriteBufferFromOStream out1(std::cout);
RowOutputStreamPtr out2 = std::make_shared<TabSeparatedRowOutputStream>(out1, expression->getSampleBlock());
BlockOutputStreamFromRowOutputStream out(out2);
{
Stopwatch stopwatch;
stopwatch.start();
copyData(*in, out);
stopwatch.stop();
std::cout << std::fixed << std::setprecision(2)
<< "Elapsed " << stopwatch.elapsedSeconds() << " sec."
<< ", " << n / stopwatch.elapsedSeconds() << " rows/sec."
<< std::endl;
}
return 0;
}
catch (const DB::Exception & e)
{
std::cerr << e.what() << ", " << e.displayText() << std::endl;
throw;
}
int main(int argc, char ** argv)
{
using namespace DB;
try
{
NamesAndTypesList names_and_types_list
{
{"WatchID", std::make_shared<DataTypeUInt64>()},
{"JavaEnable", std::make_shared<DataTypeUInt8>()},
{"Title", std::make_shared<DataTypeString>()},
{"EventTime", std::make_shared<DataTypeDateTime>()},
{"CounterID", std::make_shared<DataTypeUInt32>()},
{"ClientIP", std::make_shared<DataTypeUInt32>()},
{"RegionID", std::make_shared<DataTypeUInt32>()},
{"UniqID", std::make_shared<DataTypeUInt64>()},
{"CounterClass", std::make_shared<DataTypeUInt8>()},
{"OS", std::make_shared<DataTypeUInt8>()},
{"UserAgent", std::make_shared<DataTypeUInt8>()},
{"URL", std::make_shared<DataTypeString>()},
{"Referer", std::make_shared<DataTypeString>()},
{"ResolutionWidth", std::make_shared<DataTypeUInt16>()},
{"ResolutionHeight", std::make_shared<DataTypeUInt16>()},
{"ResolutionDepth", std::make_shared<DataTypeUInt8>()},
{"FlashMajor", std::make_shared<DataTypeUInt8>()},
{"FlashMinor", std::make_shared<DataTypeUInt8>()},
{"FlashMinor2", std::make_shared<DataTypeString>()},
{"NetMajor", std::make_shared<DataTypeUInt8>()},
{"NetMinor", std::make_shared<DataTypeUInt8>()},
{"UserAgentMajor", std::make_shared<DataTypeUInt16>()},
{"UserAgentMinor", std::make_shared<DataTypeFixedString>(2)},
{"CookieEnable", std::make_shared<DataTypeUInt8>()},
{"JavascriptEnable", std::make_shared<DataTypeUInt8>()},
{"IsMobile", std::make_shared<DataTypeUInt8>()},
{"MobilePhone", std::make_shared<DataTypeUInt8>()},
{"MobilePhoneModel", std::make_shared<DataTypeString>()},
{"Params", std::make_shared<DataTypeString>()},
{"IPNetworkID", std::make_shared<DataTypeUInt32>()},
{"TraficSourceID", std::make_shared<DataTypeInt8>()},
{"SearchEngineID", std::make_shared<DataTypeUInt16>()},
{"SearchPhrase", std::make_shared<DataTypeString>()},
{"AdvEngineID", std::make_shared<DataTypeUInt8>()},
{"IsArtifical", std::make_shared<DataTypeUInt8>()},
{"WindowClientWidth", std::make_shared<DataTypeUInt16>()},
{"WindowClientHeight", std::make_shared<DataTypeUInt16>()},
{"ClientTimeZone", std::make_shared<DataTypeInt16>()},
{"ClientEventTime", std::make_shared<DataTypeDateTime>()},
{"SilverlightVersion1", std::make_shared<DataTypeUInt8>()},
{"SilverlightVersion2", std::make_shared<DataTypeUInt8>()},
{"SilverlightVersion3", std::make_shared<DataTypeUInt32>()},
{"SilverlightVersion4", std::make_shared<DataTypeUInt16>()},
{"PageCharset", std::make_shared<DataTypeString>()},
{"CodeVersion", std::make_shared<DataTypeUInt32>()},
{"IsLink", std::make_shared<DataTypeUInt8>()},
{"IsDownload", std::make_shared<DataTypeUInt8>()},
{"IsNotBounce", std::make_shared<DataTypeUInt8>()},
{"FUniqID", std::make_shared<DataTypeUInt64>()},
{"OriginalURL", std::make_shared<DataTypeString>()},
{"HID", std::make_shared<DataTypeUInt32>()},
{"IsOldCounter", std::make_shared<DataTypeUInt8>()},
{"IsEvent", std::make_shared<DataTypeUInt8>()},
{"IsParameter", std::make_shared<DataTypeUInt8>()},
{"DontCountHits", std::make_shared<DataTypeUInt8>()},
{"WithHash", std::make_shared<DataTypeUInt8>()},
};
Context context;
std::string input = "SELECT UniqID, URL, CounterID, IsLink WHERE URL = 'http://mail.yandex.ru/neo2/#inbox'";
ParserSelectQuery parser;
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "");
formatAST(*ast, std::cerr);
std::cerr << std::endl;
std::cerr << ast->getTreeID() << std::endl;
/// create an object of an existing hit log table
StoragePtr table = StorageLog::create("./", "HitLog", std::make_shared<NamesAndTypesList>(names_and_types_list));
/// read from it, apply the expression, filter, and write in tsv form to the console
ExpressionAnalyzer analyzer(ast, context, nullptr, names_and_types_list);
ExpressionActionsChain chain;
analyzer.appendSelect(chain, false);
analyzer.appendWhere(chain, false);
chain.finalize();
ExpressionActionsPtr expression = chain.getLastActions();
Names column_names
{
"UniqID",
"URL",
"CounterID",
"IsLink",
};
QueryProcessingStage::Enum stage;
BlockInputStreamPtr in = table->read(column_names, 0, context, stage)[0];
in = std::make_shared<FilterBlockInputStream>(in, expression, 4);
//in = std::make_shared<LimitBlockInputStream>(in, 10, 0);
WriteBufferFromOStream ob(std::cout);
RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(ob, expression->getSampleBlock());
BlockOutputStreamFromRowOutputStream out(out_);
copyData(*in, out);
}
catch (const Exception & e)
{
std::cerr << e.what() << ", " << e.displayText() << std::endl;
return 1;
}
return 0;
}
int main(int argc, char ** argv)
{
using namespace DB;
try
{
std::string input = "SELECT x, s1, s2, "
"/*"
"2 + x * 2, x * 2, x % 3 == 1, "
"s1 == 'abc', s1 == s2, s1 != 'abc', s1 != s2, "
"s1 < 'abc', s1 < s2, s1 > 'abc', s1 > s2, "
"s1 <= 'abc', s1 <= s2, s1 >= 'abc', s1 >= s2, "
"*/"
"s1 < s2 AND x % 3 < x % 5";
ParserSelectQuery parser;
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "");
formatAST(*ast, std::cerr);
std::cerr << std::endl;
Context context;
NamesAndTypesList columns
{
{"x", std::make_shared<DataTypeInt16>()},
{"s1", std::make_shared<DataTypeString>()},
{"s2", std::make_shared<DataTypeString>()}
};
ExpressionAnalyzer analyzer(ast, context, {}, columns);
ExpressionActionsChain chain;
analyzer.appendSelect(chain, false);
analyzer.appendProjectResult(chain, false);
chain.finalize();
ExpressionActionsPtr expression = chain.getLastActions();
size_t n = argc == 2 ? atoi(argv[1]) : 10;
Block block;
ColumnWithTypeAndName column_x;
column_x.name = "x";
column_x.type = std::make_shared<DataTypeInt16>();
auto x = std::make_shared<ColumnInt16>();
column_x.column = x;
auto & vec_x = x->getData();
vec_x.resize(n);
for (size_t i = 0; i < n; ++i)
vec_x[i] = i;
block.insert(column_x);
const char * strings[] = {"abc", "def", "abcd", "defg", "ac"};
ColumnWithTypeAndName column_s1;
column_s1.name = "s1";
column_s1.type = std::make_shared<DataTypeString>();
column_s1.column = std::make_shared<ColumnString>();
for (size_t i = 0; i < n; ++i)
column_s1.column->insert(String(strings[i % 5]));
block.insert(column_s1);
ColumnWithTypeAndName column_s2;
column_s2.name = "s2";
column_s2.type = std::make_shared<DataTypeString>();
column_s2.column = std::make_shared<ColumnString>();
for (size_t i = 0; i < n; ++i)
column_s2.column->insert(String(strings[i % 3]));
block.insert(column_s2);
{
Stopwatch stopwatch;
stopwatch.start();
expression->execute(block);
stopwatch.stop();
std::cout << std::fixed << std::setprecision(2)
<< "Elapsed " << stopwatch.elapsedSeconds() << " sec."
<< ", " << n / stopwatch.elapsedSeconds() << " rows/sec."
<< std::endl;
}
auto is = std::make_shared<OneBlockInputStream>(block);
LimitBlockInputStream lis(is, 20, std::max(0, static_cast<int>(n) - 20));
WriteBufferFromOStream out_buf(std::cout);
RowOutputStreamPtr os_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, block);
BlockOutputStreamFromRowOutputStream os(os_);
copyData(lis, os);
}
catch (const Exception & e)
{
std::cerr << e.displayText() << std::endl;
}
return 0;
}
int main(int argc, char ** argv)
{
using namespace DB;
try
{
if (argc < 2)
{
std::cerr << "at least 1 argument expected" << std::endl;
return 1;
}
Context context;
NamesAndTypesList columns;
for (int i = 2; i + 1 < argc; i += 2)
{
NameAndTypePair col;
col.name = argv[i];
col.type = DataTypeFactory::instance().get(argv[i + 1]);
columns.push_back(col);
}
ASTPtr root;
ParserPtr parsers[] = {std::make_unique<ParserSelectQuery>(), std::make_unique<ParserExpressionList>(false)};
for (size_t i = 0; i < sizeof(parsers)/sizeof(parsers[0]); ++i)
{
IParser & parser = *parsers[i];
const char * pos = argv[1];
const char * end = argv[1] + strlen(argv[1]);
const char * max_parsed_pos = pos;
Expected expected = "";
if (parser.parse(pos, end, root, max_parsed_pos, expected))
break;
else
root = nullptr;
}
if (!root)
{
std::cerr << "invalid expression (should be select query or expression list)" << std::endl;
return 2;
}
formatAST(*root, std::cout);
std::cout << std::endl;
ExpressionAnalyzer analyzer(root, context, {}, columns);
Names required = analyzer.getRequiredColumns();
std::cout << "required columns:\n";
for (size_t i = 0; i < required.size(); ++i)
{
std::cout << required[i] << "\n";
}
std::cout << "\n";
std::cout << "only consts:\n\n" << analyzer.getConstActions()->dumpActions() << "\n";
if (analyzer.hasAggregation())
{
Names key_names;
AggregateDescriptions aggregates;
analyzer.getAggregateInfo(key_names, aggregates);
std::cout << "keys:\n";
for (size_t i = 0; i < key_names.size(); ++i)
std::cout << key_names[i] << "\n";
std::cout << "\n";
std::cout << "aggregates:\n";
for (size_t i = 0; i < aggregates.size(); ++i)
{
AggregateDescription desc = aggregates[i];
std::cout << desc.column_name << " = " << desc.function->getName() << " ( ";
for (size_t j = 0; j < desc.argument_names.size(); ++j)
std::cout << desc.argument_names[j] << " ";
std::cout << ")\n";
}
std::cout << "\n";
ExpressionActionsChain before;
if (analyzer.appendWhere(before, false))
before.addStep();
analyzer.appendAggregateFunctionsArguments(before, false);
analyzer.appendGroupBy(before, false);
before.finalize();
ExpressionActionsChain after;
if (analyzer.appendHaving(after, false))
after.addStep();
analyzer.appendSelect(after, false);
analyzer.appendOrderBy(after, false);
after.addStep();
analyzer.appendProjectResult(after, false);
after.finalize();
std::cout << "before aggregation:\n\n";
for (size_t i = 0; i < before.steps.size(); ++i)
{
std::cout << before.steps[i].actions->dumpActions();
std::cout << std::endl;
}
std::cout << "\nafter aggregation:\n\n";
for (size_t i = 0; i < after.steps.size(); ++i)
{
std::cout << after.steps[i].actions->dumpActions();
std::cout << std::endl;
}
}
else
{
if (typeid_cast<ASTSelectQuery *>(&*root))
{
ExpressionActionsChain chain;
if (analyzer.appendWhere(chain, false))
chain.addStep();
analyzer.appendSelect(chain, false);
analyzer.appendOrderBy(chain, false);
chain.addStep();
analyzer.appendProjectResult(chain, false);
chain.finalize();
for (size_t i = 0; i < chain.steps.size(); ++i)
{
std::cout << chain.steps[i].actions->dumpActions();
std::cout << std::endl;
}
}
else
{
std::cout << "unprojected actions:\n\n" << analyzer.getActions(false)->dumpActions() << "\n";
std::cout << "projected actions:\n\n" << analyzer.getActions(true)->dumpActions() << "\n";
}
}
}
catch (Exception & e)
{
std::cerr << "Exception " << e.what() << ": " << e.displayText() << "\n" << e.getStackTrace().toString();
return 3;
}
return 0;
}
int main(int argc, char ** argv)
try
{
using namespace DB;
std::string input = "SELECT number, number % 10000000 == 1";
ParserSelectQuery parser;
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "");
formatAST(*ast, std::cerr);
std::cerr << std::endl;
Context context;
ExpressionAnalyzer analyzer(ast, context, {}, {NameAndTypePair("number", std::make_shared<DataTypeUInt64>())});
ExpressionActionsChain chain;
analyzer.appendSelect(chain, false);
analyzer.appendProjectResult(chain, false);
chain.finalize();
ExpressionActionsPtr expression = chain.getLastActions();
StoragePtr table = StorageSystemNumbers::create("Numbers");
Names column_names;
column_names.push_back("number");
QueryProcessingStage::Enum stage;
BlockInputStreamPtr in = table->read(column_names, 0, context, stage)[0];
ForkBlockInputStreams fork(in);
BlockInputStreamPtr in1 = fork.createInput();
BlockInputStreamPtr in2 = fork.createInput();
in1 = std::make_shared<FilterBlockInputStream>(in1, expression, 1);
in1 = std::make_shared<LimitBlockInputStream>(in1, 10, 0);
in2 = std::make_shared<FilterBlockInputStream>(in2, expression, 1);
in2 = std::make_shared<LimitBlockInputStream>(in2, 20, 5);
Block out_sample = expression->getSampleBlock();
WriteBufferFromOStream ob1(std::cout);
WriteBufferFromOStream ob2(std::cerr);
BlockOutputStreamPtr out1 = context.getOutputFormat("TabSeparated", ob1, out_sample);
BlockOutputStreamPtr out2 = context.getOutputFormat("TabSeparated", ob2, out_sample);
std::thread thr1(std::bind(thread1, in1, out1, std::ref(ob1)));
std::thread thr2(std::bind(thread2, in2, out2, std::ref(ob2)));
fork.run();
thr1.join();
thr2.join();
return 0;
}
catch (const DB::Exception & e)
{
std::cerr << e.what() << ", " << e.displayText() << std::endl;
throw;
}