void Quota::loadFromConfig(const String & config_elem, const String & name_, Poco::Util::AbstractConfiguration & config, pcg64 & rng) { name = name_; bool new_keyed_by_ip = config.has(config_elem + ".keyed_by_ip"); bool new_is_keyed = new_keyed_by_ip || config.has(config_elem + ".keyed"); if (new_is_keyed != is_keyed || new_keyed_by_ip != keyed_by_ip) { keyed_by_ip = new_keyed_by_ip; is_keyed = new_is_keyed; /// Meaning of keys has been changed. Throw away accumulated values. quota_for_keys.clear(); } ignore_key_if_not_keyed = config.has(config_elem + ".ignore_key_if_not_keyed"); QuotaForIntervals new_max(name, {}); new_max.initFromConfig(config_elem, config, rng); if (!new_max.hasEqualConfiguration(max)) { max = new_max; for (auto & quota : quota_for_keys) quota.second->setMax(max); } }
void SettingsConstraints::loadFromConfig(const String & path_to_constraints, const Poco::Util::AbstractConfiguration & config) { if (!config.has(path_to_constraints)) throw Exception("There is no path '" + path_to_constraints + "' in configuration file.", ErrorCodes::NO_ELEMENTS_IN_CONFIG); Poco::Util::AbstractConfiguration::Keys names; config.keys(path_to_constraints, names); for (const String & name : names) { String path_to_name = path_to_constraints + "." + name; Poco::Util::AbstractConfiguration::Keys constraint_types; config.keys(path_to_name, constraint_types); for (const String & constraint_type : constraint_types) { auto get_constraint_value = [&]{ return config.getString(path_to_name + "." + constraint_type); }; if (constraint_type == "min") setMinValue(name, get_constraint_value()); else if (constraint_type == "max") setMaxValue(name, get_constraint_value()); else if (constraint_type == "readonly") setReadOnly(name, true); else throw Exception("Setting " + constraint_type + " value for " + name + " isn't supported", ErrorCodes::NOT_IMPLEMENTED); } } }
DictionaryLifetime::DictionaryLifetime(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) { const auto & lifetime_min_key = config_prefix + ".min"; const auto has_min = config.has(lifetime_min_key); this->min_sec = has_min ? config.getInt(lifetime_min_key) : config.getInt(config_prefix); this->max_sec = has_min ? config.getInt(config_prefix + ".max") : this->min_sec; }
/*virtual*/ void Renderer::Initialize(uint width,uint height,bool fullScreen) { Poco::Util::AbstractConfiguration* pConfigurationFile = Poly::Application::Get()->ConfigurationFile(); mFpsLogging = pConfigurationFile->getBool("Renderer.LogFPS",false); mShadersPath = pConfigurationFile->getString("Renderer.ShadersPath",""); mVerticalSync = pConfigurationFile->getBool("Renderer.VerticalSync",false); }
DictionarySpecialAttribute::DictionarySpecialAttribute(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) : name{config.getString(config_prefix + ".name", "")}, expression{config.getString(config_prefix + ".expression", "")} { if (name.empty() && !expression.empty()) throw Exception{ "Element " + config_prefix + ".name is empty", ErrorCodes::BAD_ARGUMENTS}; }
Macros::Macros(const Poco::Util::AbstractConfiguration & config, const String & root_key) { Poco::Util::AbstractConfiguration::Keys keys; config.keys(root_key, keys); for (const String & key : keys) { macros[key] = config.getString(root_key + "." + key); } }
void QuotaValues<Counter>::initFromConfig(const String & config_elem, Poco::Util::AbstractConfiguration & config) { queries = config.getUInt64(config_elem + ".queries", 0); errors = config.getUInt64(config_elem + ".errors", 0); result_rows = config.getUInt64(config_elem + ".result_rows", 0); result_bytes = config.getUInt64(config_elem + ".result_bytes", 0); read_rows = config.getUInt64(config_elem + ".read_rows", 0); read_bytes = config.getUInt64(config_elem + ".read_bytes", 0); execution_time_usec = config.getUInt64(config_elem + ".execution_time", 0) * 1000000ULL; }
void SettingsConstraints::setProfile(const String & profile_name, const Poco::Util::AbstractConfiguration & config) { String parent_profile = "profiles." + profile_name + ".profile"; if (config.has(parent_profile)) setProfile(parent_profile, config); // Inheritance of one profile from another. String path_to_constraints = "profiles." + profile_name + ".constraints"; if (config.has(path_to_constraints)) loadFromConfig(path_to_constraints, config); }
std::unique_ptr<RegionsHierarchies> GeoDictionariesLoader::reloadRegionsHierarchies( const Poco::Util::AbstractConfiguration & config) { static constexpr auto config_key = "path_to_regions_hierarchy_file"; if (!config.has(config_key)) return {}; const auto default_hierarchy_file = config.getString(config_key); auto data_provider = std::make_unique<RegionsHierarchiesDataProvider>(default_hierarchy_file); return std::make_unique<RegionsHierarchies>(std::move(data_provider)); }
std::unique_ptr<RegionsNames> GeoDictionariesLoader::reloadRegionsNames( const Poco::Util::AbstractConfiguration & config) { static constexpr auto config_key = "path_to_regions_names_files"; if (!config.has(config_key)) return {}; const auto directory = config.getString(config_key); auto data_provider = std::make_unique<RegionsNamesDataProvider>(directory); return std::make_unique<RegionsNames>(std::move(data_provider)); }
void Settings::loadSettingsFromConfig(const String & path, const Poco::Util::AbstractConfiguration & config) { if (!config.has(path)) throw Exception("There is no path '" + path + "' in configuration file.", ErrorCodes::NO_ELEMENTS_IN_CONFIG); Poco::Util::AbstractConfiguration::Keys config_keys; config.keys(path, config_keys); for (const std::string & key : config_keys) { set(key, config.getString(path + "." + key)); } }
HTTPDictionarySource::HTTPDictionarySource(const DictionaryStructure & dict_struct_, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, const Context & context) : log(&Logger::get("HTTPDictionarySource")), update_time{std::chrono::system_clock::from_time_t(0)}, dict_struct{dict_struct_}, url{config.getString(config_prefix + ".url", "")}, update_field{config.getString(config_prefix + ".update_field", "")}, format{config.getString(config_prefix + ".format")}, sample_block{sample_block}, context(context), timeouts(ConnectionTimeouts::getHTTPTimeouts(context.getSettingsRef())) { }
MySQLDictionarySource::MySQLDictionarySource(const DictionaryStructure & dict_struct_, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, const Block & sample_block) : dict_struct{dict_struct_}, db{config.getString(config_prefix + ".db", "")}, table{config.getString(config_prefix + ".table")}, where{config.getString(config_prefix + ".where", "")}, dont_check_update_time{config.getBool(config_prefix + ".dont_check_update_time", false)}, sample_block{sample_block}, pool{config, config_prefix}, query_builder{dict_struct, db, table, where}, load_all_query{query_builder.composeLoadAllQuery()} { }
LibraryDictionarySource::LibraryDictionarySource( const DictionaryStructure & dict_struct_, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block) : log(&Logger::get("LibraryDictionarySource")) , dict_struct{dict_struct_} , config_prefix{config_prefix} , path{config.getString(config_prefix + ".path", "")} , sample_block{sample_block} { if (!Poco::File(path).exists()) throw Exception( "LibraryDictionarySource: Can't load lib " + toString() + ": " + Poco::File(path).path() + " - File doesn't exist", ErrorCodes::FILE_DOESNT_EXIST); description.init(sample_block); library = std::make_shared<SharedLibrary>(path, RTLD_LAZY #if defined(RTLD_DEEPBIND) // Does not exists in freebsd | RTLD_DEEPBIND #endif ); settings = std::make_shared<CStringsHolder>(getLibSettings(config, config_prefix + lib_config_settings)); if (auto libNew = library->tryGet<decltype(lib_data) (*)(decltype(&settings->strings), decltype(&ClickHouseLibrary::log))>( "ClickHouseDictionary_v3_libNew")) lib_data = libNew(&settings->strings, ClickHouseLibrary::log); }
SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfiguration & config) { query_log = createSystemLog<QueryLog>(global_context, "system", "query_log", config, "query_log"); query_thread_log = createSystemLog<QueryThreadLog>(global_context, "system", "query_thread_log", config, "query_thread_log"); part_log = createSystemLog<PartLog>(global_context, "system", "part_log", config, "part_log"); part_log_database = config.getString("part_log.database", "system"); }
void Utility::copyDeltaProperties(const Poco::Util::AbstractConfiguration& ref, const Poco::Util::AbstractConfiguration& source, Poco::Util::AbstractConfiguration& target, const std::set<std::string>& excludeSet, const std::string& root) { Poco::Util::AbstractConfiguration::Keys keys; source.keys(root, keys); if (keys.empty() && source.hasProperty(root)) { if ((ref.hasProperty(root) && ref.getRawString(root) != source.getRawString(root)) || !ref.hasProperty(root)) { target.setString(root, source.getRawString(root)); } } else { for (Poco::Util::AbstractConfiguration::Keys::const_iterator it = keys.begin(); it != keys.end(); ++it) { std::string fullKey = root; if (!fullKey.empty()) fullKey += '.'; fullKey.append(*it); if (excludeSet.find(fullKey) == excludeSet.end()) { copyDeltaProperties(ref, source, target, excludeSet, fullKey); } } } }
DictionaryStructure::DictionaryStructure(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) { const auto has_id = config.has(config_prefix + ".id"); const auto has_key = config.has(config_prefix + ".key"); if (has_key && has_id) throw Exception{"Only one of 'id' and 'key' should be specified", ErrorCodes::BAD_ARGUMENTS}; if (has_id) id.emplace(config, config_prefix + ".id"); else if (has_key) { key.emplace(getAttributes(config, config_prefix + ".key", false, false)); if (key->empty()) throw Exception{"Empty 'key' supplied", ErrorCodes::BAD_ARGUMENTS}; } else throw Exception{"Dictionary structure should specify either 'id' or 'key'", ErrorCodes::BAD_ARGUMENTS}; if (id) { if (id->name.empty()) throw Exception{"'id' cannot be empty", ErrorCodes::BAD_ARGUMENTS}; if (config.has(config_prefix + ".range_min")) range_min.emplace(config, config_prefix + ".range_min"); if (config.has(config_prefix + ".range_max")) range_max.emplace(config, config_prefix + ".range_max"); if (!id->expression.empty() || (range_min && !range_min->expression.empty()) || (range_max && !range_max->expression.empty())) has_expressions = true; } attributes = getAttributes(config, config_prefix); if (attributes.empty()) throw Exception{"Dictionary has no attributes defined", ErrorCodes::BAD_ARGUMENTS}; }
void SecurityManager::loadFromConfig(Poco::Util::AbstractConfiguration & config) { Container new_users; Poco::Util::AbstractConfiguration::Keys config_keys; config.keys("users", config_keys); for (const std::string & key : config_keys) { auto user = std::make_shared<const User>(key, "users." + key, config); new_users.emplace(key, std::move(user)); } users = std::move(new_users); }
void QuotaForIntervals::initFromConfig(const String & config_elem, Poco::Util::AbstractConfiguration & config, pcg64 & rng) { Poco::Util::AbstractConfiguration::Keys config_keys; config.keys(config_elem, config_keys); for (Poco::Util::AbstractConfiguration::Keys::const_iterator it = config_keys.begin(); it != config_keys.end(); ++it) { if (!startsWith(*it, "interval")) continue; String interval_config_elem = config_elem + "." + *it; time_t duration = config.getInt(interval_config_elem + ".duration", 0); time_t offset = 0; if (!duration) /// Skip quotas with zero duration continue; bool randomize = config.getBool(interval_config_elem + ".randomize", false); if (randomize) offset = std::uniform_int_distribution<decltype(duration)>(0, duration - 1)(rng); cont[duration].initFromConfig(interval_config_elem, duration, randomize, offset, config); } }
MongoDBDictionarySource::MongoDBDictionarySource( const DictionaryStructure & dict_struct, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block) : MongoDBDictionarySource( dict_struct, config.getString(config_prefix + ".host"), config.getUInt(config_prefix + ".port"), config.getString(config_prefix + ".user", ""), config.getString(config_prefix + ".password", ""), config.getString(config_prefix + ".db", ""), config.getString(config_prefix + ".collection"), sample_block) { }
void Quotas::loadFromConfig(Poco::Util::AbstractConfiguration & config) { pcg64 rng; Poco::Util::AbstractConfiguration::Keys config_keys; config.keys("quotas", config_keys); /// Remove keys, that now absent in config. std::set<std::string> keys_set(config_keys.begin(), config_keys.end()); for (Container::iterator it = cont.begin(); it != cont.end();) { if (keys_set.count(it->first)) ++it; else cont.erase(it++); } for (Poco::Util::AbstractConfiguration::Keys::const_iterator it = config_keys.begin(); it != config_keys.end(); ++it) { if (!cont.count(*it)) cont.try_emplace(*it); cont[*it].loadFromConfig("quotas." + *it, *it, config, rng); } }
DictionaryPtr DictionaryFactory::create(const std::string & name, Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Context & context) const { Poco::Util::AbstractConfiguration::Keys keys; const auto & layout_prefix = config_prefix + ".layout"; config.keys(layout_prefix, keys); if (keys.size() != 1) throw Exception{name + ": element dictionary.layout should have exactly one child element", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG}; const DictionaryStructure dict_struct{config, config_prefix + ".structure"}; auto source_ptr = DictionarySourceFactory::instance().create( name, config, config_prefix + ".source", dict_struct, context); const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false); const auto & layout_type = keys.front(); if ("range_hashed" == layout_type) { if (dict_struct.key) throw Exception{"'key' is not supported for dictionary of layout 'range_hashed'", ErrorCodes::UNSUPPORTED_METHOD}; if (!dict_struct.range_min || !dict_struct.range_max) throw Exception{name + ": dictionary of layout 'range_hashed' requires .structure.range_min and .structure.range_max", ErrorCodes::BAD_ARGUMENTS}; return std::make_unique<RangeHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); } else if ("complex_key_hashed" == layout_type) { if (!dict_struct.key) throw Exception{"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS}; return std::make_unique<ComplexKeyHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); } else if ("complex_key_cache" == layout_type) { if (!dict_struct.key) throw Exception{"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS}; const auto size = config.getInt(layout_prefix + ".complex_key_cache.size_in_cells"); if (size == 0) throw Exception{name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE}; if (require_nonempty) throw Exception{name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set", ErrorCodes::BAD_ARGUMENTS}; return std::make_unique<ComplexKeyCacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size); } else { if (dict_struct.key) throw Exception{"'key' is not supported for dictionary of layout '" + layout_type + "'", ErrorCodes::UNSUPPORTED_METHOD}; if (dict_struct.range_min || dict_struct.range_max) throw Exception{name + ": elements .structure.range_min and .structure.range_max should be defined only " "for a dictionary of layout 'range_hashed'", ErrorCodes::BAD_ARGUMENTS}; if ("flat" == layout_type) { return std::make_unique<FlatDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); } else if ("hashed" == layout_type) { return std::make_unique<HashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); } else if ("cache" == layout_type) { const auto size = config.getInt(layout_prefix + ".cache.size_in_cells"); if (size == 0) throw Exception{name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE}; if (require_nonempty) throw Exception{name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set", ErrorCodes::BAD_ARGUMENTS}; return std::make_unique<CacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size); } } throw Exception{name + ": unknown dictionary layout type: " + layout_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG}; };
std::vector<DictionaryAttribute> DictionaryStructure::getAttributes( const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, const bool hierarchy_allowed, const bool allow_null_values) { Poco::Util::AbstractConfiguration::Keys keys; config.keys(config_prefix, keys); auto has_hierarchy = false; std::vector<DictionaryAttribute> attributes; for (const auto & key : keys) { if (0 != strncmp(key.data(), "attribute", strlen("attribute"))) continue; const auto prefix = config_prefix + '.' + key + '.'; const auto name = config.getString(prefix + "name"); const auto type_string = config.getString(prefix + "type"); const auto type = DataTypeFactory::instance().get(type_string); const auto underlying_type = getAttributeUnderlyingType(type_string); const auto expression = config.getString(prefix + "expression", ""); if (!expression.empty()) has_expressions = true; Field null_value; if (allow_null_values) { const auto null_value_string = config.getString(prefix + "null_value"); try { ReadBufferFromString null_value_buffer{null_value_string}; ColumnPtr column_with_null_value = type->createColumn(); type->deserializeTextEscaped(*column_with_null_value, null_value_buffer); null_value = (*column_with_null_value)[0]; } catch (const std::exception & e) { throw Exception{ std::string{"Error parsing null_value: "} + e.what(), ErrorCodes::BAD_ARGUMENTS}; } } const auto hierarchical = config.getBool(prefix + "hierarchical", false); const auto injective = config.getBool(prefix + "injective", false); if (name.empty()) throw Exception{ "Properties 'name' and 'type' of an attribute cannot be empty", ErrorCodes::BAD_ARGUMENTS}; if (has_hierarchy && !hierarchy_allowed) throw Exception{ "Hierarchy not allowed in '" + prefix, ErrorCodes::BAD_ARGUMENTS}; if (has_hierarchy && hierarchical) throw Exception{ "Only one hierarchical attribute supported", ErrorCodes::BAD_ARGUMENTS}; has_hierarchy = has_hierarchy || hierarchical; attributes.emplace_back(DictionaryAttribute{ name, underlying_type, type, expression, null_value, hierarchical, injective }); } return attributes; }
Pool::Pool(const Poco::Util::AbstractConfiguration & cfg, const std::string & config_name, unsigned default_connections_, unsigned max_connections_, const char * parent_config_name_) : default_connections(default_connections_), max_connections(max_connections_) { server = cfg.getString(config_name + ".host"); if (parent_config_name_) { const std::string parent_config_name(parent_config_name_); db = cfg.getString(config_name + ".db", cfg.getString(parent_config_name + ".db", "")); user = cfg.has(config_name + ".user") ? cfg.getString(config_name + ".user") : cfg.getString(parent_config_name + ".user"); password = cfg.has(config_name + ".password") ? cfg.getString(config_name + ".password") : cfg.getString(parent_config_name + ".password"); if (!cfg.has(config_name + ".port") && !cfg.has(config_name + ".socket") && !cfg.has(parent_config_name + ".port") && !cfg.has(parent_config_name + ".socket")) throw Poco::Exception("mysqlxx::Pool configuration: expected port or socket"); port = cfg.has(config_name + ".port") ? cfg.getInt(config_name + ".port") : cfg.getInt(parent_config_name + ".port", 0); socket = cfg.has(config_name + ".socket") ? cfg.getString(config_name + ".socket") : cfg.getString(parent_config_name + ".socket", ""); ssl_ca = cfg.has(config_name + ".ssl_ca") ? cfg.getString(config_name + ".ssl_ca") : cfg.getString(parent_config_name + ".ssl_ca", ""); ssl_cert = cfg.has(config_name + ".ssl_cert") ? cfg.getString(config_name + ".ssl_cert") : cfg.getString(parent_config_name + ".ssl_cert", ""); ssl_key = cfg.has(config_name + ".ssl_key") ? cfg.getString(config_name + ".ssl_key") : cfg.getString(parent_config_name + ".ssl_key", ""); } else { db = cfg.getString(config_name + ".db", ""); user = cfg.getString(config_name + ".user"); password = cfg.getString(config_name + ".password"); if (!cfg.has(config_name + ".port") && !cfg.has(config_name + ".socket")) throw Poco::Exception("mysqlxx::Pool configuration: expected port or socket"); port = cfg.getInt(config_name + ".port", 0); socket = cfg.getString(config_name + ".socket", ""); ssl_ca = cfg.getString(config_name + ".ssl_ca", ""); ssl_cert = cfg.getString(config_name + ".ssl_cert", ""); ssl_key = cfg.getString(config_name + ".ssl_key", ""); } connect_timeout = cfg.getInt(config_name + ".connect_timeout", cfg.getInt("mysql_connect_timeout", MYSQLXX_DEFAULT_TIMEOUT)); rw_timeout = cfg.getInt(config_name + ".rw_timeout", cfg.getInt("mysql_rw_timeout", MYSQLXX_DEFAULT_RW_TIMEOUT)); }