コード例 #1
0
int main(int argc, char ** argv)
try
{
	using namespace DB;

	size_t n = argc == 2 ? parse<UInt64>(argv[1]) : 10ULL;

	std::string input = "SELECT number, number / 3, number * number";

	ParserSelectQuery parser;
	ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "");

	Context context;

	ExpressionAnalyzer analyzer(ast, context, {}, {NameAndTypePair("number", std::make_shared<DataTypeUInt64>())});
	ExpressionActionsChain chain;
	analyzer.appendSelect(chain, false);
	analyzer.appendProjectResult(chain, false);
	chain.finalize();
	ExpressionActionsPtr expression = chain.getLastActions();

	StoragePtr table = StorageSystemNumbers::create("Numbers");

	Names column_names;
	column_names.push_back("number");

	QueryProcessingStage::Enum stage;

	BlockInputStreamPtr in;
	in = table->read(column_names, 0, context, Settings(), stage)[0];
	in = std::make_shared<ExpressionBlockInputStream>(in, expression);
	in = std::make_shared<LimitBlockInputStream>(in, 10, std::max(static_cast<Int64>(0), static_cast<Int64>(n) - 10));

	WriteBufferFromOStream out1(std::cout);
	RowOutputStreamPtr out2 = std::make_shared<TabSeparatedRowOutputStream>(out1, expression->getSampleBlock());
	BlockOutputStreamFromRowOutputStream out(out2);

	{
		Stopwatch stopwatch;
		stopwatch.start();

		copyData(*in, out);

		stopwatch.stop();
		std::cout << std::fixed << std::setprecision(2)
			<< "Elapsed " << stopwatch.elapsedSeconds() << " sec."
			<< ", " << n / stopwatch.elapsedSeconds() << " rows/sec."
			<< std::endl;
	}

	return 0;
}
catch (const DB::Exception & e)
{
	std::cerr << e.what() << ", " << e.displayText() << std::endl;
	throw;
}
コード例 #2
0
int main(int argc, char ** argv)
{
    using namespace DB;

    try
    {
        NamesAndTypesList names_and_types_list
        {
            {"WatchID",                std::make_shared<DataTypeUInt64>()},
            {"JavaEnable",            std::make_shared<DataTypeUInt8>()},
            {"Title",                std::make_shared<DataTypeString>()},
            {"EventTime",            std::make_shared<DataTypeDateTime>()},
            {"CounterID",            std::make_shared<DataTypeUInt32>()},
            {"ClientIP",            std::make_shared<DataTypeUInt32>()},
            {"RegionID",            std::make_shared<DataTypeUInt32>()},
            {"UniqID",                std::make_shared<DataTypeUInt64>()},
            {"CounterClass",        std::make_shared<DataTypeUInt8>()},
            {"OS",                    std::make_shared<DataTypeUInt8>()},
            {"UserAgent",            std::make_shared<DataTypeUInt8>()},
            {"URL",                    std::make_shared<DataTypeString>()},
            {"Referer",                std::make_shared<DataTypeString>()},
            {"ResolutionWidth",        std::make_shared<DataTypeUInt16>()},
            {"ResolutionHeight",    std::make_shared<DataTypeUInt16>()},
            {"ResolutionDepth",        std::make_shared<DataTypeUInt8>()},
            {"FlashMajor",            std::make_shared<DataTypeUInt8>()},
            {"FlashMinor",            std::make_shared<DataTypeUInt8>()},
            {"FlashMinor2",            std::make_shared<DataTypeString>()},
            {"NetMajor",            std::make_shared<DataTypeUInt8>()},
            {"NetMinor",            std::make_shared<DataTypeUInt8>()},
            {"UserAgentMajor",        std::make_shared<DataTypeUInt16>()},
            {"UserAgentMinor",        std::make_shared<DataTypeFixedString>(2)},
            {"CookieEnable",        std::make_shared<DataTypeUInt8>()},
            {"JavascriptEnable",    std::make_shared<DataTypeUInt8>()},
            {"IsMobile",            std::make_shared<DataTypeUInt8>()},
            {"MobilePhone",            std::make_shared<DataTypeUInt8>()},
            {"MobilePhoneModel",    std::make_shared<DataTypeString>()},
            {"Params",                std::make_shared<DataTypeString>()},
            {"IPNetworkID",            std::make_shared<DataTypeUInt32>()},
            {"TraficSourceID",        std::make_shared<DataTypeInt8>()},
            {"SearchEngineID",        std::make_shared<DataTypeUInt16>()},
            {"SearchPhrase",        std::make_shared<DataTypeString>()},
            {"AdvEngineID",            std::make_shared<DataTypeUInt8>()},
            {"IsArtifical",            std::make_shared<DataTypeUInt8>()},
            {"WindowClientWidth",    std::make_shared<DataTypeUInt16>()},
            {"WindowClientHeight",    std::make_shared<DataTypeUInt16>()},
            {"ClientTimeZone",        std::make_shared<DataTypeInt16>()},
            {"ClientEventTime",        std::make_shared<DataTypeDateTime>()},
            {"SilverlightVersion1",    std::make_shared<DataTypeUInt8>()},
            {"SilverlightVersion2",    std::make_shared<DataTypeUInt8>()},
            {"SilverlightVersion3",    std::make_shared<DataTypeUInt32>()},
            {"SilverlightVersion4",    std::make_shared<DataTypeUInt16>()},
            {"PageCharset",            std::make_shared<DataTypeString>()},
            {"CodeVersion",            std::make_shared<DataTypeUInt32>()},
            {"IsLink",                std::make_shared<DataTypeUInt8>()},
            {"IsDownload",            std::make_shared<DataTypeUInt8>()},
            {"IsNotBounce",            std::make_shared<DataTypeUInt8>()},
            {"FUniqID",                std::make_shared<DataTypeUInt64>()},
            {"OriginalURL",            std::make_shared<DataTypeString>()},
            {"HID",                    std::make_shared<DataTypeUInt32>()},
            {"IsOldCounter",        std::make_shared<DataTypeUInt8>()},
            {"IsEvent",                std::make_shared<DataTypeUInt8>()},
            {"IsParameter",            std::make_shared<DataTypeUInt8>()},
            {"DontCountHits",        std::make_shared<DataTypeUInt8>()},
            {"WithHash",            std::make_shared<DataTypeUInt8>()},
        };

        Context context;

        std::string input = "SELECT UniqID, URL, CounterID, IsLink WHERE URL = 'http://mail.yandex.ru/neo2/#inbox'";
        ParserSelectQuery parser;
        ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "");

        formatAST(*ast, std::cerr);
        std::cerr << std::endl;
        std::cerr << ast->getTreeID() << std::endl;

        /// create an object of an existing hit log table

        StoragePtr table = StorageLog::create("./", "HitLog", std::make_shared<NamesAndTypesList>(names_and_types_list));

        /// read from it, apply the expression, filter, and write in tsv form to the console

        ExpressionAnalyzer analyzer(ast, context, nullptr, names_and_types_list);
        ExpressionActionsChain chain;
        analyzer.appendSelect(chain, false);
        analyzer.appendWhere(chain, false);
        chain.finalize();
        ExpressionActionsPtr expression = chain.getLastActions();

        Names column_names
        {
            "UniqID",
            "URL",
            "CounterID",
            "IsLink",
        };

        QueryProcessingStage::Enum stage;

        BlockInputStreamPtr in = table->read(column_names, 0, context, stage)[0];
        in = std::make_shared<FilterBlockInputStream>(in, expression, 4);
        //in = std::make_shared<LimitBlockInputStream>(in, 10, 0);

        WriteBufferFromOStream ob(std::cout);
        RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(ob, expression->getSampleBlock());
        BlockOutputStreamFromRowOutputStream out(out_);

        copyData(*in, out);
    }
    catch (const Exception & e)
    {
        std::cerr << e.what() << ", " << e.displayText() << std::endl;
        return 1;
    }

    return 0;
}
コード例 #3
0
ファイル: expression.cpp プロジェクト: yurial/ClickHouse
int main(int argc, char ** argv)
{
    using namespace DB;

    try
    {
        std::string input = "SELECT x, s1, s2, "
            "/*"
            "2 + x * 2, x * 2, x % 3 == 1, "
            "s1 == 'abc', s1 == s2, s1 != 'abc', s1 != s2, "
            "s1 <  'abc', s1 <  s2, s1 >  'abc', s1 >  s2, "
            "s1 <= 'abc', s1 <= s2, s1 >= 'abc', s1 >= s2, "
            "*/"
            "s1 < s2 AND x % 3 < x % 5";

        ParserSelectQuery parser;
        ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "");

        formatAST(*ast, std::cerr);
        std::cerr << std::endl;

        Context context;
        NamesAndTypesList columns
        {
            {"x", std::make_shared<DataTypeInt16>()},
            {"s1", std::make_shared<DataTypeString>()},
            {"s2", std::make_shared<DataTypeString>()}
        };

        ExpressionAnalyzer analyzer(ast, context, {}, columns);
        ExpressionActionsChain chain;
        analyzer.appendSelect(chain, false);
        analyzer.appendProjectResult(chain, false);
        chain.finalize();
        ExpressionActionsPtr expression = chain.getLastActions();

        size_t n = argc == 2 ? atoi(argv[1]) : 10;

        Block block;

        ColumnWithTypeAndName column_x;
        column_x.name = "x";
        column_x.type = std::make_shared<DataTypeInt16>();
        auto x = std::make_shared<ColumnInt16>();
        column_x.column = x;
        auto & vec_x = x->getData();

        vec_x.resize(n);
        for (size_t i = 0; i < n; ++i)
            vec_x[i] = i;

        block.insert(column_x);

        const char * strings[] = {"abc", "def", "abcd", "defg", "ac"};

        ColumnWithTypeAndName column_s1;
        column_s1.name = "s1";
        column_s1.type = std::make_shared<DataTypeString>();
        column_s1.column = std::make_shared<ColumnString>();

        for (size_t i = 0; i < n; ++i)
            column_s1.column->insert(String(strings[i % 5]));

        block.insert(column_s1);

        ColumnWithTypeAndName column_s2;
        column_s2.name = "s2";
        column_s2.type = std::make_shared<DataTypeString>();
        column_s2.column = std::make_shared<ColumnString>();

        for (size_t i = 0; i < n; ++i)
            column_s2.column->insert(String(strings[i % 3]));

        block.insert(column_s2);

        {
            Stopwatch stopwatch;
            stopwatch.start();

            expression->execute(block);

            stopwatch.stop();
            std::cout << std::fixed << std::setprecision(2)
                << "Elapsed " << stopwatch.elapsedSeconds() << " sec."
                << ", " << n / stopwatch.elapsedSeconds() << " rows/sec."
                << std::endl;
        }

        auto is = std::make_shared<OneBlockInputStream>(block);
        LimitBlockInputStream lis(is, 20, std::max(0, static_cast<int>(n) - 20));
        WriteBufferFromOStream out_buf(std::cout);
        RowOutputStreamPtr os_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, block);
        BlockOutputStreamFromRowOutputStream os(os_);

        copyData(lis, os);
    }
    catch (const Exception & e)
    {
        std::cerr << e.displayText() << std::endl;
    }

    return 0;
}
コード例 #4
0
ファイル: fork_streams.cpp プロジェクト: yurial/ClickHouse
int main(int argc, char ** argv)
try
{
    using namespace DB;

    std::string input = "SELECT number, number % 10000000 == 1";

    ParserSelectQuery parser;
    ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "");

    formatAST(*ast, std::cerr);
    std::cerr << std::endl;

    Context context;

    ExpressionAnalyzer analyzer(ast, context, {}, {NameAndTypePair("number", std::make_shared<DataTypeUInt64>())});
    ExpressionActionsChain chain;
    analyzer.appendSelect(chain, false);
    analyzer.appendProjectResult(chain, false);
    chain.finalize();
    ExpressionActionsPtr expression = chain.getLastActions();

    StoragePtr table = StorageSystemNumbers::create("Numbers");

    Names column_names;
    column_names.push_back("number");

    QueryProcessingStage::Enum stage;

    BlockInputStreamPtr in = table->read(column_names, 0, context, stage)[0];

    ForkBlockInputStreams fork(in);

    BlockInputStreamPtr in1 = fork.createInput();
    BlockInputStreamPtr in2 = fork.createInput();

    in1 = std::make_shared<FilterBlockInputStream>(in1, expression, 1);
    in1 = std::make_shared<LimitBlockInputStream>(in1, 10, 0);

    in2 = std::make_shared<FilterBlockInputStream>(in2, expression, 1);
    in2 = std::make_shared<LimitBlockInputStream>(in2, 20, 5);

    Block out_sample = expression->getSampleBlock();

    WriteBufferFromOStream ob1(std::cout);
    WriteBufferFromOStream ob2(std::cerr);

    BlockOutputStreamPtr out1 = context.getOutputFormat("TabSeparated", ob1, out_sample);
    BlockOutputStreamPtr out2 = context.getOutputFormat("TabSeparated", ob2, out_sample);

    std::thread thr1(std::bind(thread1, in1, out1, std::ref(ob1)));
    std::thread thr2(std::bind(thread2, in2, out2, std::ref(ob2)));

    fork.run();

    thr1.join();
    thr2.join();

    return 0;
}
catch (const DB::Exception & e)
{
    std::cerr << e.what() << ", " << e.displayText() << std::endl;
    throw;
}