コード例 #1
0
PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
        String database, String table, StoragePtr storage,
        const Context & context_, const ASTPtr & query_ptr_, bool no_destination)
    : context(context_), query_ptr(query_ptr_)
{
    /** TODO This is a very important line. At any insertion into the table one of streams should own lock.
      * Although now any insertion into the table is done via PushingToViewsBlockOutputStream,
      *  but it's clear that here is not the best place for this functionality.
      */
    addTableLock(storage->lockStructure(true, __PRETTY_FUNCTION__));

    if (!table.empty())
    {
        Dependencies dependencies = context.getDependencies(database, table);

        /// We need special context for materialized views insertions
        if (!dependencies.empty())
        {
            views_context = std::make_unique<Context>(context);
            // Do not deduplicate insertions into MV if the main insertion is Ok
            views_context->getSettingsRef().insert_deduplicate = false;
        }

        for (const auto & database_table : dependencies)
        {
            auto dependent_table = context.getTable(database_table.first, database_table.second);
            auto & materialized_view = dynamic_cast<const StorageMaterializedView &>(*dependent_table);

            auto query = materialized_view.getInnerQuery();
            auto out = std::make_shared<PushingToViewsBlockOutputStream>(
                database_table.first, database_table.second, dependent_table, *views_context, ASTPtr());
            views.emplace_back(ViewInfo{std::move(query), database_table.first, database_table.second, std::move(out)});
        }
    }

    /* Do not push to destination table if the flag is set */
    if (!no_destination)
    {
        output = storage->write(query_ptr, context.getSettingsRef());
        replicated_output = dynamic_cast<ReplicatedMergeTreeBlockOutputStream *>(output.get());
    }
}
コード例 #2
0
ファイル: hit_log.cpp プロジェクト: kellylg/ClickHouse
int main(int argc, char ** argv)
try
{
    NamesAndTypesList names_and_types_list
    {
        {"WatchID", std::make_shared<DataTypeUInt64>()},
        {"JavaEnable", std::make_shared<DataTypeUInt8>()},
        {"Title", std::make_shared<DataTypeString>()},
        {"EventTime", std::make_shared<DataTypeDateTime>()},
        {"CounterID", std::make_shared<DataTypeUInt32>()},
        {"ClientIP", std::make_shared<DataTypeUInt32>()},
        {"RegionID", std::make_shared<DataTypeUInt32>()},
        {"UniqID", std::make_shared<DataTypeUInt64>()},
        {"CounterClass", std::make_shared<DataTypeUInt8>()},
        {"OS", std::make_shared<DataTypeUInt8>()},
        {"UserAgent", std::make_shared<DataTypeUInt8>()},
        {"URL", std::make_shared<DataTypeString>()},
        {"Referer", std::make_shared<DataTypeString>()},
        {"ResolutionWidth", std::make_shared<DataTypeUInt16>()},
        {"ResolutionHeight", std::make_shared<DataTypeUInt16>()},
        {"ResolutionDepth", std::make_shared<DataTypeUInt8>()},
        {"FlashMajor", std::make_shared<DataTypeUInt8>()},
        {"FlashMinor", std::make_shared<DataTypeUInt8>()},
        {"FlashMinor2", std::make_shared<DataTypeString>()},
        {"NetMajor", std::make_shared<DataTypeUInt8>()},
        {"NetMinor", std::make_shared<DataTypeUInt8>()},
        {"UserAgentMajor", std::make_shared<DataTypeUInt16>()},
        {"UserAgentMinor", std::make_shared<DataTypeFixedString>(2)},
        {"CookieEnable", std::make_shared<DataTypeUInt8>()},
        {"JavascriptEnable", std::make_shared<DataTypeUInt8>()},
        {"IsMobile", std::make_shared<DataTypeUInt8>()},
        {"MobilePhone", std::make_shared<DataTypeUInt8>()},
        {"MobilePhoneModel", std::make_shared<DataTypeString>()},
        {"Params", std::make_shared<DataTypeString>()},
        {"IPNetworkID", std::make_shared<DataTypeUInt32>()},
        {"TraficSourceID", std::make_shared<DataTypeInt8>()},
        {"SearchEngineID", std::make_shared<DataTypeUInt16>()},
        {"SearchPhrase", std::make_shared<DataTypeString>()},
        {"AdvEngineID", std::make_shared<DataTypeUInt8>()},
        {"IsArtifical", std::make_shared<DataTypeUInt8>()},
        {"WindowClientWidth", std::make_shared<DataTypeUInt16>()},
        {"WindowClientHeight", std::make_shared<DataTypeUInt16>()},
        {"ClientTimeZone", std::make_shared<DataTypeInt16>()},
        {"ClientEventTime", std::make_shared<DataTypeDateTime>()},
        {"SilverlightVersion1", std::make_shared<DataTypeUInt8>()},
        {"SilverlightVersion2", std::make_shared<DataTypeUInt8>()},
        {"SilverlightVersion3", std::make_shared<DataTypeUInt32>()},
        {"SilverlightVersion4", std::make_shared<DataTypeUInt16>()},
        {"PageCharset", std::make_shared<DataTypeString>()},
        {"CodeVersion", std::make_shared<DataTypeUInt32>()},
        {"IsLink", std::make_shared<DataTypeUInt8>()},
        {"IsDownload", std::make_shared<DataTypeUInt8>()},
        {"IsNotBounce", std::make_shared<DataTypeUInt8>()},
        {"FUniqID", std::make_shared<DataTypeUInt64>()},
        {"OriginalURL", std::make_shared<DataTypeString>()},
        {"HID", std::make_shared<DataTypeUInt32>()},
        {"IsOldCounter", std::make_shared<DataTypeUInt8>()},
        {"IsEvent", std::make_shared<DataTypeUInt8>()},
        {"IsParameter", std::make_shared<DataTypeUInt8>()},
        {"DontCountHits", std::make_shared<DataTypeUInt8>()},
        {"WithHash", std::make_shared<DataTypeUInt8>()},
    };

    DataTypes data_types;
    Names column_names;

    for (const auto & name_type : names_and_types_list)
    {
        data_types.push_back(name_type.type);
        column_names.push_back(name_type.name);
    }

    /// create a hit log table

    StoragePtr table = StorageLog::create(
        "./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
    table->startup();

    /// create a description of how to read data from the tab separated dump

    Block sample;
    for (const auto & name_type : names_and_types_list)
    {
        ColumnWithTypeAndName elem;
        elem.name = name_type.name;
        elem.type = name_type.type;
        elem.column = elem.type->createColumn();
        sample.insert(std::move(elem));
    }

    FormatSettings format_settings;

    /// read the data from tsv file and simultaneously write to table
    if (argc == 2 && 0 == strcmp(argv[1], "write"))
    {
        ReadBufferFromFileDescriptor in_buf(STDIN_FILENO);

        RowInputStreamPtr in_ = std::make_shared<TabSeparatedRowInputStream>(in_buf, sample, false, false, format_settings);
        BlockInputStreamFromRowInputStream in(in_, sample, DEFAULT_INSERT_BLOCK_SIZE, 0, 0);
        BlockOutputStreamPtr out = table->write({}, {});
        copyData(in, *out);
    }

    /// read from it
    if (argc == 2 && 0 == strcmp(argv[1], "read"))
    {
        WriteBufferFromFileDescriptor out_buf(STDOUT_FILENO);

        QueryProcessingStage::Enum stage;

        BlockInputStreamPtr in = table->read(column_names, {}, Context::createGlobal(), stage, 8192, 1)[0];
        RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample, false, false, format_settings);
        BlockOutputStreamFromRowOutputStream out(out_, sample);
        copyData(*in, out);
    }

    return 0;
}
catch (const Exception & e)
{
    std::cerr << e.what() << ", " << e.displayText() << std::endl;
    throw;
}
コード例 #3
0
ファイル: storage_log.cpp プロジェクト: Aahart911/ClickHouse
int main(int argc, char ** argv)
try
{
	using namespace DB;

	const size_t rows = 10000000;

	/// создаём таблицу с парой столбцов

	NamesAndTypesListPtr names_and_types = std::make_shared<NamesAndTypesList>();
	names_and_types->push_back(NameAndTypePair("a", std::make_shared<DataTypeUInt64>()));
	names_and_types->push_back(NameAndTypePair("b", std::make_shared<DataTypeUInt8>()));

	StoragePtr table = StorageLog::create("./", "test", names_and_types);

	/// пишем в неё
	{
		Block block;

		ColumnWithTypeAndName column1;
		column1.name = "a";
		column1.type = table->getDataTypeByName("a");
		column1.column = column1.type->createColumn();
		ColumnUInt64::Container_t & vec1 = typeid_cast<ColumnUInt64&>(*column1.column).getData();

		vec1.resize(rows);
		for (size_t i = 0; i < rows; ++i)
			vec1[i] = i;

		block.insert(column1);

		ColumnWithTypeAndName column2;
		column2.name = "b";
		column2.type = table->getDataTypeByName("b");
		column2.column = column2.type->createColumn();
		ColumnUInt8::Container_t & vec2 = typeid_cast<ColumnUInt8&>(*column2.column).getData();

		vec2.resize(rows);
		for (size_t i = 0; i < rows; ++i)
			vec2[i] = i * 2;

		block.insert(column2);

		BlockOutputStreamPtr out = table->write({}, {});
		out->write(block);
	}

	/// читаем из неё
	{
		Names column_names;
		column_names.push_back("a");
		column_names.push_back("b");

		QueryProcessingStage::Enum stage;

		BlockInputStreamPtr in = table->read(column_names, 0, Context{}, Settings(), stage)[0];

		Block sample;
		{
			ColumnWithTypeAndName col;
			col.type = std::make_shared<DataTypeUInt64>();
			sample.insert(col);
		}
		{
			ColumnWithTypeAndName col;
			col.type = std::make_shared<DataTypeUInt8>();
			sample.insert(col);
		}

		WriteBufferFromOStream out_buf(std::cout);

		LimitBlockInputStream in_limit(in, 10, 0);
		RowOutputStreamPtr output_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample);
		BlockOutputStreamFromRowOutputStream output(output_);

		copyData(in_limit, output);
	}

	return 0;
}
catch (const DB::Exception & e)
{
	std::cerr << e.what() << ", " << e.displayText() << std::endl;
	return 1;
}
コード例 #4
0
ファイル: native_streams.cpp プロジェクト: yurial/ClickHouse
int main(int argc, char ** argv)
try
{
    using namespace DB;

    NamesAndTypesList names_and_types_list
    {
        {"WatchID",                std::make_shared<DataTypeUInt64>()},
        {"JavaEnable",            std::make_shared<DataTypeUInt8>()},
        {"Title",                std::make_shared<DataTypeString>()},
        {"EventTime",            std::make_shared<DataTypeDateTime>()},
        {"CounterID",            std::make_shared<DataTypeUInt32>()},
        {"ClientIP",            std::make_shared<DataTypeUInt32>()},
        {"RegionID",            std::make_shared<DataTypeUInt32>()},
        {"UniqID",                std::make_shared<DataTypeUInt64>()},
        {"CounterClass",        std::make_shared<DataTypeUInt8>()},
        {"OS",                    std::make_shared<DataTypeUInt8>()},
        {"UserAgent",            std::make_shared<DataTypeUInt8>()},
        {"URL",                    std::make_shared<DataTypeString>()},
        {"Referer",                std::make_shared<DataTypeString>()},
        {"ResolutionWidth",        std::make_shared<DataTypeUInt16>()},
        {"ResolutionHeight",    std::make_shared<DataTypeUInt16>()},
        {"ResolutionDepth",        std::make_shared<DataTypeUInt8>()},
        {"FlashMajor",            std::make_shared<DataTypeUInt8>()},
        {"FlashMinor",            std::make_shared<DataTypeUInt8>()},
        {"FlashMinor2",            std::make_shared<DataTypeString>()},
        {"NetMajor",            std::make_shared<DataTypeUInt8>()},
        {"NetMinor",            std::make_shared<DataTypeUInt8>()},
        {"UserAgentMajor",        std::make_shared<DataTypeUInt16>()},
        {"UserAgentMinor",        std::make_shared<DataTypeFixedString>(2)},
        {"CookieEnable",        std::make_shared<DataTypeUInt8>()},
        {"JavascriptEnable",    std::make_shared<DataTypeUInt8>()},
        {"IsMobile",            std::make_shared<DataTypeUInt8>()},
        {"MobilePhone",            std::make_shared<DataTypeUInt8>()},
        {"MobilePhoneModel",    std::make_shared<DataTypeString>()},
        {"Params",                std::make_shared<DataTypeString>()},
        {"IPNetworkID",            std::make_shared<DataTypeUInt32>()},
        {"TraficSourceID",        std::make_shared<DataTypeInt8>()},
        {"SearchEngineID",        std::make_shared<DataTypeUInt16>()},
        {"SearchPhrase",        std::make_shared<DataTypeString>()},
        {"AdvEngineID",            std::make_shared<DataTypeUInt8>()},
        {"IsArtifical",            std::make_shared<DataTypeUInt8>()},
        {"WindowClientWidth",    std::make_shared<DataTypeUInt16>()},
        {"WindowClientHeight",    std::make_shared<DataTypeUInt16>()},
        {"ClientTimeZone",        std::make_shared<DataTypeInt16>()},
        {"ClientEventTime",        std::make_shared<DataTypeDateTime>()},
        {"SilverlightVersion1",    std::make_shared<DataTypeUInt8>()},
        {"SilverlightVersion2",    std::make_shared<DataTypeUInt8>()},
        {"SilverlightVersion3",    std::make_shared<DataTypeUInt32>()},
        {"SilverlightVersion4",    std::make_shared<DataTypeUInt16>()},
        {"PageCharset",            std::make_shared<DataTypeString>()},
        {"CodeVersion",            std::make_shared<DataTypeUInt32>()},
        {"IsLink",                std::make_shared<DataTypeUInt8>()},
        {"IsDownload",            std::make_shared<DataTypeUInt8>()},
        {"IsNotBounce",            std::make_shared<DataTypeUInt8>()},
        {"FUniqID",                std::make_shared<DataTypeUInt64>()},
        {"OriginalURL",            std::make_shared<DataTypeString>()},
        {"HID",                    std::make_shared<DataTypeUInt32>()},
        {"IsOldCounter",        std::make_shared<DataTypeUInt8>()},
        {"IsEvent",                std::make_shared<DataTypeUInt8>()},
        {"IsParameter",            std::make_shared<DataTypeUInt8>()},
        {"DontCountHits",        std::make_shared<DataTypeUInt8>()},
        {"WithHash",            std::make_shared<DataTypeUInt8>()},
    };

    Names column_names;

    for (const auto & name_type : names_and_types_list)
        column_names.push_back(name_type.name);

    /// create an object of an existing hit log table

    StoragePtr table = StorageLog::create("./", "HitLog", std::make_shared<NamesAndTypesList>(names_and_types_list));

    /// read from it
    if (argc == 2 && 0 == strcmp(argv[1], "read"))
    {
        QueryProcessingStage::Enum stage;
        BlockInputStreamPtr in = table->read(column_names, 0, Context{}, stage)[0];
        WriteBufferFromFileDescriptor out1(STDOUT_FILENO);
        CompressedWriteBuffer out2(out1);
        NativeBlockOutputStream out3(out2, ClickHouseRevision::get());
        copyData(*in, out3);
    }

    /// read the data from the native file and simultaneously write to the table
    if (argc == 2 && 0 == strcmp(argv[1], "write"))
    {
        ReadBufferFromFileDescriptor in1(STDIN_FILENO);
        CompressedReadBuffer in2(in1);
        NativeBlockInputStream in3(in2, ClickHouseRevision::get());
        BlockOutputStreamPtr out = table->write({}, {});
        copyData(in3, *out);
    }

    return 0;
}
catch (const DB::Exception & e)
{
    std::cerr << e.what() << ", " << e.displayText() << std::endl;
    throw;
}