ProxyConfig::ProxyConfig(proxy_t* proxy,
const folly::dynamic& json,
std::string configMd5Digest,
std::shared_ptr<PoolFactory> poolFactory)
: poolFactory_(std::move(poolFactory)),
configMd5Digest_(std::move(configMd5Digest)) {
McRouteHandleProvider provider(proxy, *proxy->destinationMap, *poolFactory_);
RouteHandleFactory<McrouterRouteHandleIf> factory(provider);
checkLogic(json.isObject(), "Config is not an object");
if (json.count("named_handles")) {
checkLogic(json["named_handles"].isArray(), "named_handles is not array");
for (const auto& it : json["named_handles"]) {
factory.create(it);
}
}
RouteSelectorMap routeSelectors;
auto jRoute = json.get_ptr("route");
auto jRoutes = json.get_ptr("routes");
checkLogic(!jRoute || !jRoutes,
"Invalid config: both 'route' and 'routes' are specified");
checkLogic(jRoute || jRoutes, "No route/routes in config");
if (jRoute) {
routeSelectors[proxy->getRouterOptions().default_route] =
std::make_shared<PrefixSelectorRoute>(factory, *jRoute);
} else { // jRoutes
checkLogic(jRoutes->isArray() || jRoutes->isObject(),
"Config: routes is not array/object");
if (jRoutes->isArray()) {
for (const auto& it : *jRoutes) {
checkLogic(it.isObject(), "RoutePolicy is not an object");
auto jCurRoute = it.get_ptr("route");
auto jAliases = it.get_ptr("aliases");
checkLogic(jCurRoute, "RoutePolicy: no route");
checkLogic(jAliases, "RoutePolicy: no aliases");
checkLogic(jAliases->isArray(), "RoutePolicy: aliases is not an array");
auto routeSelector =
std::make_shared<PrefixSelectorRoute>(factory, *jCurRoute);
for (const auto& alias : *jAliases) {
checkLogic(alias.isString(), "RoutePolicy: alias is not a string");
routeSelectors[alias.stringPiece()] = routeSelector;
}
}
} else { // object
for (const auto& it : jRoutes->items()) {
checkLogic(it.first.isString(), "RoutePolicy: alias is not a string");
routeSelectors[it.first.stringPiece()] =
std::make_shared<PrefixSelectorRoute>(factory, it.second);
}
}
}
asyncLogRoutes_ = provider.releaseAsyncLogRoutes();
proxyRoute_ = std::make_shared<ProxyRoute>(proxy, routeSelectors);
serviceInfo_ = std::make_shared<ServiceInfo>(proxy, *this);
}
void PoolFactory::parseQos(std::string parentName, const folly::dynamic& jQos,
uint64_t& qosClass, uint64_t& qosPath) {
if (!jQos.isObject()) {
MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
"{}: qos must be an object.", parentName);
return;
}
uint64_t prevClass = qosClass;
if (auto jClass = jQos.get_ptr("class")) {
if (jClass->isInt() && isQosClassValid(jClass->getInt())) {
qosClass = jClass->getInt();
} else {
MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
"{}: qos.class must be an integer in the range [0, 4]",
parentName);
}
}
if (auto jPath = jQos.get_ptr("path")) {
if (jPath->isInt() && isQosPathValid(jPath->getInt())) {
qosPath = jPath->getInt();
} else {
MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
"{}: qos.path must be an integer in the range [0, 3]",
parentName);
qosClass = prevClass;
}
}
}
McrouterRouteHandlePtr makeHashRoute(
const folly::dynamic& json,
std::vector<McrouterRouteHandlePtr> rh,
size_t threadId) {
std::string salt;
folly::StringPiece funcType = Ch3HashFunc::type();
if (json.isObject()) {
if (auto jsalt = json.get_ptr("salt")) {
checkLogic(jsalt->isString(), "HashRoute: salt is not a string");
salt = jsalt->getString();
}
if (auto jhashFunc = json.get_ptr("hash_func")) {
checkLogic(jhashFunc->isString(),
"HashRoute: hash_func is not a string");
funcType = jhashFunc->stringPiece();
}
}
if (funcType == Ch3HashFunc::type()) {
return makeHashRouteCh3(std::move(rh), std::move(salt));
} else if (funcType == Crc32HashFunc::type()) {
return makeHashRouteCrc32(std::move(rh), std::move(salt));
} else if (funcType == WeightedCh3HashFunc::type()) {
WeightedCh3HashFunc func{json, rh.size()};
return makeHashRouteWeightedCh3(std::move(rh), std::move(salt),
std::move(func));
} else if (funcType == ConstShardHashFunc::type()) {
return makeHashRouteConstShard(std::move(rh), std::move(salt));
} else if (funcType == "Latest") {
return makeLatestRoute(json, std::move(rh), threadId);
}
throwLogic("Unknown hash function: {}", funcType);
}
McrouterRouteHandlePtr makeFailoverWithExptimeRoute(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json) {
checkLogic(json.isObject(), "FailoverWithExptimeRoute is not an object");
auto jnormal = json.get_ptr("normal");
checkLogic(jnormal, "FailoverWithExptimeRoute: normal not found");
auto normal = factory.create(*jnormal);
int32_t failoverExptime = 60;
if (auto jexptime = json.get_ptr("failover_exptime")) {
checkLogic(jexptime->isInt(), "FailoverWithExptimeRoute: "
"failover_exptime is not an integer");
failoverExptime = jexptime->getInt();
}
std::vector<McrouterRouteHandlePtr> failover;
if (auto jfailover = json.get_ptr("failover")) {
failover = factory.createList(*jfailover);
}
auto children = getFailoverChildren(std::move(normal),
std::move(failover),
failoverExptime);
return makeFailoverRoute(json, std::move(children));
}
McrouterRouteHandlePtr makeRateLimitRoute(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json) {
checkLogic(json.isObject(), "RateLimitRoute is not an object");
auto jtarget = json.get_ptr("target");
checkLogic(jtarget, "RateLimitRoute: target not found");
auto target = factory.create(*jtarget);
auto jrates = json.get_ptr("rates");
checkLogic(jrates, "RateLimitRoute: rates not found");
return makeRateLimitRoute(std::move(target), RateLimiter(*jrates));
}
McrouterRouteHandlePtr makeLatestRoute(
const folly::dynamic& json,
std::vector<McrouterRouteHandlePtr> targets) {
size_t failoverCount = 5;
FailoverErrorsSettings failoverErrors;
if (json.isObject()) {
if (auto jfailoverCount = json.get_ptr("failover_count")) {
checkLogic(jfailoverCount->isInt(),
"LatestRoute: failover_count is not an integer");
failoverCount = jfailoverCount->getInt();
}
if (auto jFailoverErrors = json.get_ptr("failover_errors")) {
failoverErrors = FailoverErrorsSettings(*jFailoverErrors);
}
}
return makeLatestRoute(std::move(targets), failoverCount,
std::move(failoverErrors));
}
FailoverErrorsSettings::FailoverErrorsSettings(const folly::dynamic& json) {
checkLogic(json.isObject() || json.isArray(),
"Failover errors must be either an array or an object.");
if (json.isObject()) {
if (auto jsonGets = json.get_ptr("gets")) {
gets_ = FailoverErrorsSettings::List(*jsonGets);
}
if (auto jsonUpdates = json.get_ptr("updates")) {
updates_ = FailoverErrorsSettings::List(*jsonUpdates);
}
if (auto jsonDeletes = json.get_ptr("deletes")) {
deletes_ = FailoverErrorsSettings::List(*jsonDeletes);
}
} else if (json.isArray()) {
gets_ = FailoverErrorsSettings::List(json);
updates_ = FailoverErrorsSettings::List(json);
deletes_ = FailoverErrorsSettings::List(json);
}
}
PoolFactory::PoolJson PoolFactory::parsePool(const folly::dynamic& json) {
checkLogic(json.isString() || json.isObject(),
"Pool should be a string (name of pool) or an object");
if (json.isString()) {
return parseNamedPool(json.stringPiece());
}
auto jname = json.get_ptr("name");
checkLogic(jname && jname->isString(), "Pool should have string 'name'");
pools_.emplace(jname->stringPiece(), std::make_pair(json, PoolState::NEW));
return parseNamedPool(jname->stringPiece());
}
/**
* @return target and asynclogName
* Caller may call makeAsynclogRoute afterwards.
*/
std::pair<McrouterRouteHandlePtr, std::string> parseAsynclogRoute(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json) {
std::string asynclogName;
McrouterRouteHandlePtr target;
checkLogic(json.isObject() || json.isString(),
"AsynclogRoute should be object or string");
if (json.isString()) {
asynclogName = json.stringPiece().str();
target = factory.create(json);
} else { // object
auto jname = json.get_ptr("name");
checkLogic(jname && jname->isString(),
"AsynclogRoute: required string name");
auto jtarget = json.get_ptr("target");
checkLogic(jtarget, "AsynclogRoute: target not found");
asynclogName = jname->stringPiece().str();
target = factory.create(*jtarget);
}
return { std::move(target), std::move(asynclogName) };
}
std::shared_ptr<ClientPool>
PoolFactory::parsePool(const folly::dynamic& json) {
checkLogic(json.isString() || json.isObject(),
"Pool should be a string (name of pool) or an object");
if (json.isString()) {
return parsePool(json.stringPiece().str(), json);
} else {
auto name = json.get_ptr("name");
checkLogic(name && name->isString(), "Pool should have string 'name'");
return parsePool(name->stringPiece().str(), json);
}
}
PoolFactory::PoolFactory(const folly::dynamic& config, ConfigApiIf& configApi)
: configApi_(configApi) {
checkLogic(config.isObject(), "config is not an object");
if (auto jpools = config.get_ptr("pools")) {
checkLogic(jpools->isObject(), "config: 'pools' is not an object");
for (const auto& it : jpools->items()) {
pools_.emplace(
it.first.stringPiece(), std::make_pair(it.second, PoolState::NEW));
}
}
}
std::shared_ptr<ShadowSettings>
ShadowSettings::create(const folly::dynamic& json, McrouterInstance& router) {
auto result = std::shared_ptr<ShadowSettings>(new ShadowSettings());
try {
checkLogic(json.isObject(), "json is not an object");
if (auto jKeyFractionRange = json.get_ptr("key_fraction_range")) {
checkLogic(jKeyFractionRange->isArray(),
"key_fraction_range is not an array");
auto ar = folly::convertTo<std::vector<double>>(*jKeyFractionRange);
checkLogic(ar.size() == 2, "key_fraction_range size is not 2");
result->setKeyRange(ar[0], ar[1]);
}
if (auto jIndexRange = json.get_ptr("index_range")) {
checkLogic(jIndexRange->isArray(), "index_range is not an array");
auto ar = folly::convertTo<std::vector<size_t>>(*jIndexRange);
checkLogic(ar.size() == 2, "index_range size is not 2");
checkLogic(ar[0] <= ar[1], "index_range start > end");
result->startIndex_ = ar[0];
result->endIndex_ = ar[1];
}
if (auto jKeyFractionRangeRv = json.get_ptr("key_fraction_range_rv")) {
checkLogic(jKeyFractionRangeRv->isString(),
"key_fraction_range_rv is not a string");
result->keyFractionRangeRv_ = jKeyFractionRangeRv->stringPiece().str();
}
if (auto jValidateReplies = json.get_ptr("validate_replies")) {
checkLogic(jValidateReplies->isBool(),
"validate_replies is not a bool");
result->validateReplies_ = jValidateReplies->getBool();
}
} catch (const std::logic_error& e) {
MC_LOG_FAILURE(router.opts(), failure::Category::kInvalidConfig,
"ShadowSettings: {}", e.what());
return nullptr;
}
result->registerOnUpdateCallback(router);
return result;
}
McrouterRouteHandlePtr makeLatestRoute(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json) {
std::vector<McrouterRouteHandlePtr> children;
if (json.isObject()) {
if (auto jchildren = json.get_ptr("children")) {
children = factory.createList(*jchildren);
}
} else {
children = factory.createList(json);
}
return makeLatestRoute(json, std::move(children));
}
McrouterRouteHandlePtr makeFailoverRoute(
const folly::dynamic& json,
std::vector<McrouterRouteHandlePtr> children) {
FailoverErrorsSettings failoverErrors;
std::unique_ptr<FailoverRateLimiter> rateLimiter;
bool failoverTagging = false;
if (json.isObject()) {
if (auto jFailoverErrors = json.get_ptr("failover_errors")) {
failoverErrors = FailoverErrorsSettings(*jFailoverErrors);
}
if (auto jFailoverTag = json.get_ptr("failover_tag")) {
checkLogic(jFailoverTag->isBool(),
"Failover: failover_tag is not bool");
failoverTagging = jFailoverTag->getBool();
}
if (auto jFailoverLimit = json.get_ptr("failover_limit")) {
rateLimiter = folly::make_unique<FailoverRateLimiter>(*jFailoverLimit);
}
}
return makeFailoverRoute(std::move(children), std::move(failoverErrors),
std::move(rateLimiter), failoverTagging);
}
PoolFactory::PoolFactory(const folly::dynamic& config,
ConfigApi& configApi,
const McrouterOptions& opts)
: configApi_(configApi),
opts_(opts) {
checkLogic(config.isObject(), "config is not an object");
if (auto jpools = config.get_ptr("pools")) {
checkLogic(jpools->isObject(), "config: 'pools' is not an object");
for (const auto& it : jpools->items()) {
parsePool(it.first.stringPiece().str(), it.second);
}
}
}
McrouterRouteHandlePtr makeFailoverWithExptimeRoute(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json) {
checkLogic(json.isObject(), "FailoverWithExptimeRoute is not an object");
McrouterRouteHandlePtr normal;
if (auto jnormal = json.get_ptr("normal")) {
normal = factory.create(*jnormal);
}
std::vector<McrouterRouteHandlePtr> failoverTargets;
if (auto jfailover = json.get_ptr("failover")) {
failoverTargets = factory.createList(*jfailover);
}
int32_t failoverExptime = 60;
if (auto jexptime = json.get_ptr("failover_exptime")) {
checkLogic(jexptime->isInt(), "FailoverWithExptimeRoute: "
"failover_exptime is not an integer");
failoverExptime = jexptime->getInt();
}
// Check if only one format is being used
checkLogic(!(json.count("settings") && // old
(json.count("failover_errors") || json.count("failover_tag"))), // new
"Use either 'settings' (old format) or 'failover_errors' / 'failover_tag'"
);
// new format
FailoverErrorsSettings failoverErrors;
bool failoverTagging = false;
if (auto jfailoverTag = json.get_ptr("failover_tag")) {
checkLogic(jfailoverTag->isBool(),
"FailoverWithExptime: failover_tag is not bool");
failoverTagging = jfailoverTag->getBool();
}
if (auto jfailoverErrors = json.get_ptr("failover_errors")) {
failoverErrors = FailoverErrorsSettings(*jfailoverErrors);
}
// old format
if (auto jsettings = json.get_ptr("settings")) {
VLOG(1) << "FailoverWithExptime: This config format is deprecated. "
"Use 'failover_errors' instead of 'settings'.";
auto oldSettings = FailoverWithExptimeSettings(*jsettings);
failoverTagging = oldSettings.failoverTagging;
failoverErrors = oldSettings.getFailoverErrors();
}
return makeFailoverWithExptimeRoute(
std::move(normal),
std::move(failoverTargets),
failoverExptime,
std::move(failoverErrors),
failoverTagging);
}
std::vector<McrouterRouteHandlePtr> makeShadowRoutes(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json,
proxy_t& proxy,
ExtraRouteHandleProviderIf& extraProvider) {
checkLogic(json.isObject(), "ShadowRoute should be an object");
const auto jchildren = json.get_ptr("children");
checkLogic(jchildren, "ShadowRoute: children not found");
auto children = factory.createList(*jchildren);
if (json.count("shadows")) {
children = makeShadowRoutes(
factory, json, std::move(children), proxy, extraProvider);
}
return children;
}
McrouterRouteHandlePtr makeErrorRoute(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json) {
checkLogic(json.isObject() || json.isString() || json.isNull(),
"ErrorRoute: should be string or object");
std::string response;
if (json.isString()) {
response = json.stringPiece().str();
} else if (json.isObject()) {
if (auto jResponse = json.get_ptr("response")) {
checkLogic(jResponse->isString(), "ErrorRoute: response is not a string");
response = jResponse->stringPiece().str();
}
}
return makeErrorRoute(std::move(response));
}
McrouterRouteHandlePtr makeOperationSelectorRoute(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json) {
if (!json.isObject()) {
return factory.create(json);
}
McrouterRouteHandlePtr defaultPolicy;
if (json.count("default_policy")) {
defaultPolicy = factory.create(json["default_policy"]);
}
std::vector<McrouterRouteHandlePtr> operationPolicies{mc_nops};
if (auto jOpPolicies = json.get_ptr("operation_policies")) {
checkLogic(jOpPolicies->isObject(),
"OperationSelectorRoute: operation_policies is not an object");
std::map<std::string, const folly::dynamic*> orderedPolicies;
for (auto& it : jOpPolicies->items()) {
checkLogic(it.first.isString(),
"OperationSelectorRoute: operation_policies' "
"key is not a string");
auto key = it.first.stringPiece().str();
orderedPolicies.emplace(std::move(key), &it.second);
}
// order is important here: named handles may not be resolved if we parse
// policies in random order
for (const auto& it : orderedPolicies) {
auto opId = mc_op_from_string(it.first.data());
checkLogic(opId != mc_op_unknown, "Unknown mc operation: {}", it.first);
operationPolicies[opId] = factory.create(*it.second);
}
return makeOperationSelectorRoute(std::move(operationPolicies),
std::move(defaultPolicy));
}
return defaultPolicy;
}
std::vector<McrouterRouteHandlePtr> makeShadowRoutes(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json,
std::vector<McrouterRouteHandlePtr> children,
proxy_t& proxy,
ExtraRouteHandleProviderIf& extraProvider) {
folly::StringPiece shadowPolicy = "default";
if (auto jshadow_policy = json.get_ptr("shadow_policy")) {
checkLogic(jshadow_policy->isString(),
"ShadowRoute: shadow_policy is not a string");
shadowPolicy = jshadow_policy->stringPiece();
}
auto jshadows = json.get_ptr("shadows");
checkLogic(jshadows, "ShadowRoute: route doesn't contain shadows field");
if (!jshadows->isArray()) {
MC_LOG_FAILURE(proxy.router().opts(),
failure::Category::kInvalidConfig,
"ShadowRoute: shadows specified in route is not an array");
return children;
}
McrouterShadowData data;
for (auto& shadow : *jshadows) {
if (!shadow.isObject()) {
MC_LOG_FAILURE(proxy.router().opts(),
failure::Category::kInvalidConfig,
"ShadowRoute: shadow is not an object");
continue;
}
auto jtarget = shadow.get_ptr("target");
if (!jtarget) {
MC_LOG_FAILURE(proxy.router().opts(),
failure::Category::kInvalidConfig,
"ShadowRoute shadows: no target for shadow");
continue;
}
try {
auto s = ShadowSettings::create(shadow, proxy.router());
if (s) {
data.emplace_back(factory.create(*jtarget), std::move(s));
}
} catch (const std::exception& e) {
MC_LOG_FAILURE(proxy.router().opts(),
failure::Category::kInvalidConfig,
"Can not create shadow for ShadowRoute: {}",
e.what());
}
}
for (size_t i = 0; i < children.size(); ++i) {
McrouterShadowData childrenShadows;
for (const auto& shadowData : data) {
if (shadowData.second->startIndex() <= i &&
i < shadowData.second->endIndex()) {
childrenShadows.push_back(shadowData);
}
}
if (!childrenShadows.empty()) {
childrenShadows.shrink_to_fit();
children[i] = extraProvider.makeShadow(proxy,
std::move(children[i]),
std::move(childrenShadows),
shadowPolicy);
}
}
return children;
}
std::shared_ptr<ClientPool>
PoolFactory::parsePool(const std::string& name, const folly::dynamic& json) {
auto seenPoolIt = pools_.find(name);
if (seenPoolIt != pools_.end()) {
return seenPoolIt->second;
}
if (json.isString()) {
// get the pool from ConfigApi
std::string jsonStr;
checkLogic(configApi_.get(ConfigType::Pool, name, jsonStr),
"Can not read pool: {}", name);
return parsePool(name, parseJsonString(jsonStr));
} else {
// one day we may add inheriting from local pool
if (auto jinherit = json.get_ptr("inherit")) {
checkLogic(jinherit->isString(),
"Pool {}: inherit is not a string", name);
auto path = jinherit->stringPiece().str();
std::string jsonStr;
checkLogic(configApi_.get(ConfigType::Pool, path, jsonStr),
"Can not read pool from: {}", path);
auto newJson = parseJsonString(jsonStr);
for (auto& it : json.items()) {
newJson.insert(it.first, it.second);
}
newJson.erase("inherit");
return parsePool(name, newJson);
}
}
// pool_locality
std::chrono::milliseconds timeout{opts_.server_timeout_ms};
if (auto jlocality = json.get_ptr("pool_locality")) {
if (!jlocality->isString()) {
MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
"Pool {}: pool_locality is not a string", name);
} else {
auto str = jlocality->stringPiece();
if (str == "cluster") {
if (opts_.cluster_pools_timeout_ms != 0) {
timeout = std::chrono::milliseconds(opts_.cluster_pools_timeout_ms);
}
} else if (str == "region") {
if (opts_.regional_pools_timeout_ms != 0) {
timeout = std::chrono::milliseconds(opts_.regional_pools_timeout_ms);
}
} else {
MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
"Pool {}: '{}' pool locality is not supported",
name, str);
}
}
}
// region & cluster
std::string region, cluster;
if (auto jregion = json.get_ptr("region")) {
if (!jregion->isString()) {
MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
"Pool {}: pool_region is not a string", name);
} else {
region = jregion->stringPiece().str();
}
}
if (auto jcluster = json.get_ptr("cluster")) {
if (!jcluster->isString()) {
MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
"Pool {}: pool_cluster is not a string", name);
} else {
cluster = jcluster->stringPiece().str();
}
}
if (auto jtimeout = json.get_ptr("server_timeout")) {
if (!jtimeout->isInt()) {
MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
"Pool {}: server_timeout is not an int", name);
} else {
timeout = std::chrono::milliseconds(jtimeout->getInt());
}
}
if (!region.empty() && !cluster.empty()) {
auto& route = opts_.default_route;
if (region == route.getRegion() && cluster == route.getCluster()) {
if (opts_.within_cluster_timeout_ms != 0) {
timeout = std::chrono::milliseconds(opts_.within_cluster_timeout_ms);
}
} else if (region == route.getRegion()) {
if (opts_.cross_cluster_timeout_ms != 0) {
timeout = std::chrono::milliseconds(opts_.cross_cluster_timeout_ms);
}
} else {
if (opts_.cross_region_timeout_ms != 0) {
timeout = std::chrono::milliseconds(opts_.cross_region_timeout_ms);
}
}
}
auto protocol = parseProtocol(json, mc_ascii_protocol);
bool keep_routing_prefix = false;
if (auto jkeep_routing_prefix = json.get_ptr("keep_routing_prefix")) {
checkLogic(jkeep_routing_prefix->isBool(),
"Pool {}: keep_routing_prefix is not a bool");
keep_routing_prefix = jkeep_routing_prefix->getBool();
}
uint64_t qosClass = opts_.default_qos_class;
uint64_t qosPath = opts_.default_qos_path;
if (auto jqos = json.get_ptr("qos")) {
parseQos(folly::sformat("Pool {}", name), *jqos, qosClass, qosPath);
}
bool useSsl = false;
if (auto juseSsl = json.get_ptr("use_ssl")) {
checkLogic(juseSsl->isBool(), "Pool {}: use_ssl is not a bool", name);
useSsl = juseSsl->getBool();
}
// servers
auto jservers = json.get_ptr("servers");
checkLogic(jservers, "Pool {}: servers not found", name);
checkLogic(jservers->isArray(), "Pool {}: servers is not an array", name);
auto clientPool = std::make_shared<ClientPool>(name);
for (size_t i = 0; i < jservers->size(); ++i) {
const auto& server = jservers->at(i);
std::shared_ptr<AccessPoint> ap;
bool serverUseSsl = useSsl;
uint64_t serverQosClass = qosClass;
uint64_t serverQosPath = qosPath;
checkLogic(server.isString() || server.isObject(),
"Pool {}: server #{} is not a string/object", name, i);
if (server.isString()) {
// we support both host:port and host:port:protocol
ap = AccessPoint::create(server.stringPiece(), protocol);
checkLogic(ap != nullptr,
"Pool {}: invalid server {}", name, server.stringPiece());
} else { // object
auto jhostname = server.get_ptr("hostname");
checkLogic(jhostname,
"Pool {}: hostname not found for server #{}", name, i);
checkLogic(jhostname->isString(),
"Pool {}: hostname is not a string for server #{}", name, i);
if (auto jqos = server.get_ptr("qos")) {
parseQos(folly::sformat("Pool {}, server #{}", name, i),
*jqos, qosClass, qosPath);
}
if (auto juseSsl = server.get_ptr("use_ssl")) {
checkLogic(juseSsl->isBool(),
"Pool {}: use_ssl is not a bool for server #{}", name, i);
serverUseSsl = juseSsl->getBool();
}
ap = AccessPoint::create(jhostname->stringPiece(),
parseProtocol(server, protocol));
checkLogic(ap != nullptr, "Pool {}: invalid server #{}", name, i);
}
auto client = clientPool->emplaceClient(
timeout,
std::move(ap),
keep_routing_prefix,
serverUseSsl,
serverQosClass,
serverQosPath);
clients_.push_back(std::move(client));
} // servers
// weights
if (auto jweights = json.get_ptr("weights")) {
clientPool->setWeights(*jweights);
}
pools_.emplace(name, clientPool);
return clientPool;
}
McrouterRouteHandlePtr McRouteHandleProvider::makePoolRoute(
McRouteHandleFactory& factory, const folly::dynamic& json) {
checkLogic(json.isObject() || json.isString(),
"PoolRoute should be object or string");
const folly::dynamic* jpool;
if (json.isObject()) {
jpool = json.get_ptr("pool");
checkLogic(jpool, "PoolRoute: pool not found");
} else {
jpool = &json;
}
auto p = makePool(*jpool);
auto pool = std::move(p.first);
auto destinations = std::move(p.second);
if (json.isObject()) {
if (auto maxOutstandingPtr = json.get_ptr("max_outstanding")) {
checkLogic(maxOutstandingPtr->isInt(),
"PoolRoute {}: max_outstanding is not int", pool->getName());
auto maxOutstanding = maxOutstandingPtr->asInt();
if (maxOutstanding) {
for (auto& destination: destinations) {
destination = makeOutstandingLimitRoute(std::move(destination),
maxOutstanding);
}
}
}
}
if (json.isObject() && json.count("shadows")) {
folly::StringPiece shadowPolicy = "default";
if (auto jshadow_policy = json.get_ptr("shadow_policy")) {
checkLogic(jshadow_policy->isString(),
"PoolRoute: shadow_policy is not a string");
shadowPolicy = jshadow_policy->stringPiece();
}
McrouterShadowData data;
for (auto& shadow : json["shadows"]) {
checkLogic(shadow.count("target"),
"PoolRoute {} shadows: no target for shadow", pool->getName());
auto s = ShadowSettings::create(shadow, proxy_->router());
if (s) {
data.emplace_back(factory.create(shadow["target"]), std::move(s));
}
}
for (size_t i = 0; i < destinations.size(); ++i) {
McrouterShadowData destinationShadows;
for (const auto& shadowData : data) {
if (shadowData.second->startIndex() <= i &&
i < shadowData.second->endIndex()) {
destinationShadows.push_back(shadowData);
}
}
if (!destinationShadows.empty()) {
destinationShadows.shrink_to_fit();
destinations[i] = extraProvider_->makeShadow(
proxy_, std::move(destinations[i]),
std::move(destinationShadows), shadowPolicy);
}
}
}
// add weights and override whatever we have in PoolRoute::hash
folly::dynamic jhashWithWeights = folly::dynamic::object();
if (pool->getWeights()) {
jhashWithWeights = folly::dynamic::object
("hash_func", WeightedCh3HashFunc::type())
("weights", *pool->getWeights());
}
if (json.isObject()) {
if (auto jhash = json.get_ptr("hash")) {
checkLogic(jhash->isObject() || jhash->isString(),
"PoolRoute {}: hash is not object/string", pool->getName());
if (jhash->isString()) {
jhashWithWeights["hash_func"] = *jhash;
} else { // object
for (const auto& it : jhash->items()) {
jhashWithWeights[it.first] = it.second;
}
}
}
}
auto route = makeHashRoute(jhashWithWeights, std::move(destinations));
if (json.isObject()) {
if (proxy_->router().opts().destination_rate_limiting) {
if (auto jrates = json.get_ptr("rates")) {
route = makeRateLimitRoute(std::move(route), RateLimiter(*jrates));
}
}
if (auto jsplits = json.get_ptr("shard_splits")) {
route = makeShardSplitRoute(std::move(route), ShardSplitter(*jsplits));
}
}
auto asynclogName = pool->getName();
bool needAsynclog = true;
if (json.isObject()) {
if (auto jasynclog = json.get_ptr("asynclog")) {
checkLogic(jasynclog->isBool(), "PoolRoute: asynclog is not bool");
needAsynclog = jasynclog->getBool();
}
if (auto jname = json.get_ptr("name")) {
checkLogic(jname->isString(), "PoolRoute: name is not a string");
asynclogName = jname->stringPiece().str();
}
}
if (needAsynclog) {
route = createAsynclogRoute(std::move(route), asynclogName);
}
return route;
}
ProxyConfig::ProxyConfig(proxy_t& proxy,
const folly::dynamic& json,
std::string configMd5Digest,
PoolFactory& poolFactory)
: configMd5Digest_(std::move(configMd5Digest)) {
McRouteHandleProvider provider(proxy, poolFactory);
RouteHandleFactory<McrouterRouteHandleIf> factory(provider);
checkLogic(json.isObject(), "Config is not an object");
if (auto jNamedHandles = json.get_ptr("named_handles")) {
if (jNamedHandles->isArray()) {
for (const auto& it : *jNamedHandles) {
factory.create(it);
}
} else if (jNamedHandles->isObject()) {
for (const auto& it : jNamedHandles->items()) {
factory.addNamed(it.first.stringPiece(), it.second);
}
} else {
throwLogic("named_handles is {} expected array/object",
jNamedHandles->typeName());
}
}
RouteSelectorMap routeSelectors;
auto jRoute = json.get_ptr("route");
auto jRoutes = json.get_ptr("routes");
checkLogic(!jRoute || !jRoutes,
"Invalid config: both 'route' and 'routes' are specified");
if (jRoute) {
routeSelectors[proxy.getRouterOptions().default_route] =
std::make_shared<PrefixSelectorRoute>(factory, *jRoute);
} else if (jRoutes) { // jRoutes
checkLogic(jRoutes->isArray() || jRoutes->isObject(),
"Config: routes is not array/object");
if (jRoutes->isArray()) {
for (const auto& it : *jRoutes) {
checkLogic(it.isObject(), "RoutePolicy is not an object");
auto jCurRoute = it.get_ptr("route");
auto jAliases = it.get_ptr("aliases");
checkLogic(jCurRoute, "RoutePolicy: no route");
checkLogic(jAliases, "RoutePolicy: no aliases");
checkLogic(jAliases->isArray(), "RoutePolicy: aliases is not an array");
auto routeSelector =
std::make_shared<PrefixSelectorRoute>(factory, *jCurRoute);
for (const auto& alias : *jAliases) {
checkLogic(alias.isString(), "RoutePolicy: alias is not a string");
routeSelectors[alias.stringPiece()] = routeSelector;
}
}
} else { // object
for (const auto& it : jRoutes->items()) {
checkLogic(it.first.isString(), "RoutePolicy: alias is not a string");
routeSelectors[it.first.stringPiece()] =
std::make_shared<PrefixSelectorRoute>(factory, it.second);
}
}
} else {
throwLogic("No route/routes in config");
}
asyncLogRoutes_ = provider.releaseAsyncLogRoutes();
pools_ = provider.releasePools();
accessPoints_ = provider.releaseAccessPoints();
proxyRoute_ = std::make_shared<ProxyRoute>(&proxy, routeSelectors);
serviceInfo_ = std::make_shared<ServiceInfo>(&proxy, *this);
}