FailoverWithExptimeRoute::FailoverWithExptimeRoute(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json)
: failoverExptime_(60) {
checkLogic(json.isObject(), "FailoverWithExptimeRoute is not object");
std::vector<McrouterRouteHandlePtr> failoverTargets;
if (json.count("failover")) {
failoverTargets = factory.createList(json["failover"]);
}
failover_ = FailoverRoute<McrouterRouteHandleIf>(std::move(failoverTargets));
if (json.count("normal")) {
normal_ = factory.create(json["normal"]);
}
if (json.count("failover_exptime")) {
checkLogic(json["failover_exptime"].isInt(),
"failover_exptime is not integer");
failoverExptime_ = json["failover_exptime"].asInt();
}
if (json.count("settings")) {
settings_ = FailoverWithExptimeSettings(json["settings"]);
}
}
McrouterRouteHandlePtr makeFailoverWithExptimeRoute(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json) {
checkLogic(json.isObject(), "FailoverWithExptimeRoute is not an object");
McrouterRouteHandlePtr normal;
if (auto jnormal = json.get_ptr("normal")) {
normal = factory.create(*jnormal);
}
std::vector<McrouterRouteHandlePtr> failoverTargets;
if (auto jfailover = json.get_ptr("failover")) {
failoverTargets = factory.createList(*jfailover);
}
int32_t failoverExptime = 60;
if (auto jexptime = json.get_ptr("failover_exptime")) {
checkLogic(jexptime->isInt(), "FailoverWithExptimeRoute: "
"failover_exptime is not an integer");
failoverExptime = jexptime->getInt();
}
// Check if only one format is being used
checkLogic(!(json.count("settings") && // old
(json.count("failover_errors") || json.count("failover_tag"))), // new
"Use either 'settings' (old format) or 'failover_errors' / 'failover_tag'"
);
// new format
FailoverErrorsSettings failoverErrors;
bool failoverTagging = false;
if (auto jfailoverTag = json.get_ptr("failover_tag")) {
checkLogic(jfailoverTag->isBool(),
"FailoverWithExptime: failover_tag is not bool");
failoverTagging = jfailoverTag->getBool();
}
if (auto jfailoverErrors = json.get_ptr("failover_errors")) {
failoverErrors = FailoverErrorsSettings(*jfailoverErrors);
}
// old format
if (auto jsettings = json.get_ptr("settings")) {
VLOG(1) << "FailoverWithExptime: This config format is deprecated. "
"Use 'failover_errors' instead of 'settings'.";
auto oldSettings = FailoverWithExptimeSettings(*jsettings);
failoverTagging = oldSettings.failoverTagging;
failoverErrors = oldSettings.getFailoverErrors();
}
return makeFailoverWithExptimeRoute(
std::move(normal),
std::move(failoverTargets),
failoverExptime,
std::move(failoverErrors),
failoverTagging);
}
ProxyConfig::ProxyConfig(proxy_t* proxy,
const folly::dynamic& json,
std::string configMd5Digest,
std::shared_ptr<PoolFactory> poolFactory)
: poolFactory_(std::move(poolFactory)),
configMd5Digest_(std::move(configMd5Digest)) {
McRouteHandleProvider provider(proxy, *proxy->destinationMap, *poolFactory_);
RouteHandleFactory<McrouterRouteHandleIf> factory(provider);
checkLogic(json.isObject(), "Config is not an object");
if (json.count("named_handles")) {
checkLogic(json["named_handles"].isArray(), "named_handles is not array");
for (const auto& it : json["named_handles"]) {
factory.create(it);
}
}
RouteSelectorMap routeSelectors;
auto jRoute = json.get_ptr("route");
auto jRoutes = json.get_ptr("routes");
checkLogic(!jRoute || !jRoutes,
"Invalid config: both 'route' and 'routes' are specified");
checkLogic(jRoute || jRoutes, "No route/routes in config");
if (jRoute) {
routeSelectors[proxy->getRouterOptions().default_route] =
std::make_shared<PrefixSelectorRoute>(factory, *jRoute);
} else { // jRoutes
checkLogic(jRoutes->isArray() || jRoutes->isObject(),
"Config: routes is not array/object");
if (jRoutes->isArray()) {
for (const auto& it : *jRoutes) {
checkLogic(it.isObject(), "RoutePolicy is not an object");
auto jCurRoute = it.get_ptr("route");
auto jAliases = it.get_ptr("aliases");
checkLogic(jCurRoute, "RoutePolicy: no route");
checkLogic(jAliases, "RoutePolicy: no aliases");
checkLogic(jAliases->isArray(), "RoutePolicy: aliases is not an array");
auto routeSelector =
std::make_shared<PrefixSelectorRoute>(factory, *jCurRoute);
for (const auto& alias : *jAliases) {
checkLogic(alias.isString(), "RoutePolicy: alias is not a string");
routeSelectors[alias.stringPiece()] = routeSelector;
}
}
} else { // object
for (const auto& it : jRoutes->items()) {
checkLogic(it.first.isString(), "RoutePolicy: alias is not a string");
routeSelectors[it.first.stringPiece()] =
std::make_shared<PrefixSelectorRoute>(factory, it.second);
}
}
}
asyncLogRoutes_ = provider.releaseAsyncLogRoutes();
proxyRoute_ = std::make_shared<ProxyRoute>(proxy, routeSelectors);
serviceInfo_ = std::make_shared<ServiceInfo>(proxy, *this);
}
std::vector<double> WeightedChHashFuncBase::parseWeights(
const folly::dynamic& json,
size_t n) {
std::vector<double> weights;
checkLogic(
json.isObject() && json.count("weights"),
"WeightedChHashFunc: not an object or no weights");
checkLogic(
json["weights"].isArray(), "WeightedChHashFunc: weights is not array");
const auto& jWeights = json["weights"];
LOG_IF(ERROR, jWeights.size() < n)
<< "WeightedChHashFunc: CONFIG IS BROKEN!!! number of weights ("
<< jWeights.size() << ") is smaller than number of servers (" << n
<< "). Missing weights are set to 0.5";
for (size_t i = 0; i < std::min(n, jWeights.size()); ++i) {
const auto& weight = jWeights[i];
checkLogic(weight.isNumber(), "WeightedChHashFunc: weight is not number");
const auto weightNum = weight.asDouble();
checkLogic(
0 <= weightNum && weightNum <= 1.0,
"WeightedChHashFunc: weight must be in range [0, 1.0]");
weights.push_back(weightNum);
}
weights.resize(n, 0.5);
return weights;
}
ProxyConfig::ProxyConfig(proxy_t* proxy,
const folly::dynamic& json,
std::string configMd5Digest,
std::shared_ptr<PoolFactory> poolFactory)
: poolFactory_(std::move(poolFactory)),
configMd5Digest_(std::move(configMd5Digest)) {
McRouteHandleProvider provider(proxy, *proxy->destinationMap, *poolFactory_);
RouteHandleFactory<McrouterRouteHandleIf> factory(provider);
checkLogic(json.isObject(), "Config is not an object");
if (json.count("named_handles")) {
checkLogic(json["named_handles"].isArray(), "named_handles is not array");
for (const auto& it : json["named_handles"]) {
factory.create(it);
}
}
checkLogic(!json.count("route") || !json.count("routes"),
"Config ambiguous, has both route and routes");
RouteSelectorMap routeSelectors;
if (json.count("route")) {
addRouteSelector({ proxy->opts.default_route.str() },
json["route"], factory, routeSelectors);
} else if (json.count("routes")) {
checkLogic(json["routes"].isArray(), "Config: routes is not array");
for (const auto& it : json["routes"]) {
checkLogic(it.isObject(), "RoutePolicy is not object");
checkLogic(it.count("route"), "RoutePolicy has no route");
checkLogic(it.count("aliases"), "RoutePolicy has no aliases");
const auto& aliases = it["aliases"];
checkLogic(aliases.isArray(), "RoutePolicy aliases is not array");
addRouteSelector(aliases, it["route"], factory, routeSelectors);
}
} else {
throw std::logic_error("No route/routes in config");
}
asyncLogRoutes_ = provider.releaseAsyncLogRoutes();
proxyRoute_ = std::make_shared<ProxyRoute>(proxy, routeSelectors);
serviceInfo_ = std::make_shared<ServiceInfo>(proxy, *this);
}
inline void checkStringField(
const folly::dynamic& json,
folly::StringPiece name,
std::string expected) {
ASSERT_EQ(1, json.count(name));
ASSERT_TRUE(json[name].isString());
EXPECT_EQ(expected, json[name].asString());
}
inline void checkIntField(
const folly::dynamic& json,
folly::StringPiece name,
Type expected) {
ASSERT_EQ(1, json.count(name));
ASSERT_TRUE(json[name].isInt());
EXPECT_EQ(static_cast<int64_t>(expected), json[name].asInt());
}
RateLimiter::RateLimiter(const folly::dynamic& json) {
checkLogic(json.isObject(), "RateLimiter settings json is not an object");
auto now = TokenBucket::defaultClockNow();
if (json.count("gets_rate")) {
double rate = asPositiveDouble(json, "gets_rate");
double burst = asPositiveDoubleDefault(json, "gets_burst", rate);
getsTb_ = TokenBucket(rate, burst, now);
}
if (json.count("sets_rate")) {
double rate = asPositiveDouble(json, "sets_rate");
double burst = asPositiveDoubleDefault(json, "sets_burst", rate);
setsTb_ = TokenBucket(rate, burst, now);
}
if (json.count("deletes_rate")) {
double rate = asPositiveDouble(json, "deletes_rate");
double burst = asPositiveDoubleDefault(json, "deletes_burst", rate);
deletesTb_ = TokenBucket(rate, burst, now);
}
}
std::vector<McrouterRouteHandlePtr> makeShadowRoutes(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json,
proxy_t& proxy,
ExtraRouteHandleProviderIf& extraProvider) {
checkLogic(json.isObject(), "ShadowRoute should be an object");
const auto jchildren = json.get_ptr("children");
checkLogic(jchildren, "ShadowRoute: children not found");
auto children = factory.createList(*jchildren);
if (json.count("shadows")) {
children = makeShadowRoutes(
factory, json, std::move(children), proxy, extraProvider);
}
return children;
}
OperationSelectorRoute::OperationSelectorRoute(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json) {
if (!json.isObject()) {
defaultPolicy_ = factory.create(json);
return;
}
if (json.count("default_policy")) {
defaultPolicy_ = factory.create(json["default_policy"]);
}
operationPolicies_.resize(mc_nops);
if (json.count("operation_policies")) {
const auto& policies = json["operation_policies"];
checkLogic(policies.isObject(),
"OperationSelectorRoute: operation_policies is not object");
std::map<std::string, folly::dynamic> orderedPolicies;
for (const auto& it : policies.items()) {
checkLogic(it.first.isString(),
"OperationSelectorRoute: operation_policies "
"key is not a string");
auto key = it.first.asString().toStdString();
orderedPolicies.insert({ key, it.second });
}
// order is important here: named handles may not be resolved if we parse
// policies in random order
for (const auto& it : orderedPolicies) {
auto opId = mc_op_from_string(it.first.data());
checkLogic(opId != mc_op_unknown, "Unknown mc operation: {}", it.first);
operationPolicies_[opId] = factory.create(it.second);
}
}
}
WeightedCh3HashFunc::WeightedCh3HashFunc(const folly::dynamic& json, size_t n) {
checkLogic(json.isObject() && json.count("weights"),
"WeightedCh3HashFunc: not an object or no weights");
checkLogic(json["weights"].isArray(),
"WeightedCh3HashFunc: weights is not array");
const auto& jWeights = json["weights"];
LOG_IF(ERROR, jWeights.size() < n)
<< "WeightedCh3HashFunc: CONFIG IS BROKEN!!! number of weights ("
<< jWeights.size() << ") is smaller than number of servers (" << n
<< "). Missing weights are set to 0.5";
for (size_t i = 0; i < std::min(n, jWeights.size()); ++i) {
const auto& weight = jWeights[i];
checkLogic(weight.isNumber(), "WeightedCh3HashFunc: weight is not number");
weights_.push_back(weight.asDouble());
}
weights_.resize(n, 0.5);
}
McrouterRouteHandlePtr makeOperationSelectorRoute(
RouteHandleFactory<McrouterRouteHandleIf>& factory,
const folly::dynamic& json) {
if (!json.isObject()) {
return factory.create(json);
}
McrouterRouteHandlePtr defaultPolicy;
if (json.count("default_policy")) {
defaultPolicy = factory.create(json["default_policy"]);
}
std::vector<McrouterRouteHandlePtr> operationPolicies{mc_nops};
if (auto jOpPolicies = json.get_ptr("operation_policies")) {
checkLogic(jOpPolicies->isObject(),
"OperationSelectorRoute: operation_policies is not an object");
std::map<std::string, const folly::dynamic*> orderedPolicies;
for (auto& it : jOpPolicies->items()) {
checkLogic(it.first.isString(),
"OperationSelectorRoute: operation_policies' "
"key is not a string");
auto key = it.first.stringPiece().str();
orderedPolicies.emplace(std::move(key), &it.second);
}
// order is important here: named handles may not be resolved if we parse
// policies in random order
for (const auto& it : orderedPolicies) {
auto opId = mc_op_from_string(it.first.data());
checkLogic(opId != mc_op_unknown, "Unknown mc operation: {}", it.first);
operationPolicies[opId] = factory.create(*it.second);
}
return makeOperationSelectorRoute(std::move(operationPolicies),
std::move(defaultPolicy));
}
return defaultPolicy;
}
SwitchStateFields
SwitchStateFields::fromFollyDynamic(const folly::dynamic& swJson) {
SwitchStateFields switchState;
switchState.interfaces = InterfaceMap::fromFollyDynamic(
swJson[kInterfaces]);
switchState.ports = PortMap::fromFollyDynamic(swJson[kPorts]);
switchState.vlans = VlanMap::fromFollyDynamic(swJson[kVlans]);
switchState.routeTables = RouteTableMap::fromFollyDynamic(
swJson[kRouteTables]);
switchState.acls = AclMap::fromFollyDynamic(swJson[kAcls]);
if (swJson.count(kSflowCollectors) > 0) {
switchState.sFlowCollectors = SflowCollectorMap::fromFollyDynamic(
swJson[kSflowCollectors]);
}
switchState.defaultVlan = VlanID(swJson[kDefaultVlan].asInt());
if (swJson.find(kControlPlane) != swJson.items().end()) {
switchState.controlPlane = ControlPlane::fromFollyDynamic(
swJson[kControlPlane]);
}
//TODO verify that created state here is internally consistent t4155406
return switchState;
}
McrouterRouteHandlePtr McRouteHandleProvider::makePoolRoute(
McRouteHandleFactory& factory, const folly::dynamic& json) {
checkLogic(json.isObject() || json.isString(),
"PoolRoute should be object or string");
const folly::dynamic* jpool;
if (json.isObject()) {
jpool = json.get_ptr("pool");
checkLogic(jpool, "PoolRoute: pool not found");
} else {
jpool = &json;
}
auto p = makePool(*jpool);
auto pool = std::move(p.first);
auto destinations = std::move(p.second);
if (json.isObject()) {
if (auto maxOutstandingPtr = json.get_ptr("max_outstanding")) {
checkLogic(maxOutstandingPtr->isInt(),
"PoolRoute {}: max_outstanding is not int", pool->getName());
auto maxOutstanding = maxOutstandingPtr->asInt();
if (maxOutstanding) {
for (auto& destination: destinations) {
destination = makeOutstandingLimitRoute(std::move(destination),
maxOutstanding);
}
}
}
}
if (json.isObject() && json.count("shadows")) {
folly::StringPiece shadowPolicy = "default";
if (auto jshadow_policy = json.get_ptr("shadow_policy")) {
checkLogic(jshadow_policy->isString(),
"PoolRoute: shadow_policy is not a string");
shadowPolicy = jshadow_policy->stringPiece();
}
McrouterShadowData data;
for (auto& shadow : json["shadows"]) {
checkLogic(shadow.count("target"),
"PoolRoute {} shadows: no target for shadow", pool->getName());
auto s = ShadowSettings::create(shadow, proxy_->router());
if (s) {
data.emplace_back(factory.create(shadow["target"]), std::move(s));
}
}
for (size_t i = 0; i < destinations.size(); ++i) {
McrouterShadowData destinationShadows;
for (const auto& shadowData : data) {
if (shadowData.second->startIndex() <= i &&
i < shadowData.second->endIndex()) {
destinationShadows.push_back(shadowData);
}
}
if (!destinationShadows.empty()) {
destinationShadows.shrink_to_fit();
destinations[i] = extraProvider_->makeShadow(
proxy_, std::move(destinations[i]),
std::move(destinationShadows), shadowPolicy);
}
}
}
// add weights and override whatever we have in PoolRoute::hash
folly::dynamic jhashWithWeights = folly::dynamic::object();
if (pool->getWeights()) {
jhashWithWeights = folly::dynamic::object
("hash_func", WeightedCh3HashFunc::type())
("weights", *pool->getWeights());
}
if (json.isObject()) {
if (auto jhash = json.get_ptr("hash")) {
checkLogic(jhash->isObject() || jhash->isString(),
"PoolRoute {}: hash is not object/string", pool->getName());
if (jhash->isString()) {
jhashWithWeights["hash_func"] = *jhash;
} else { // object
for (const auto& it : jhash->items()) {
jhashWithWeights[it.first] = it.second;
}
}
}
}
auto route = makeHashRoute(jhashWithWeights, std::move(destinations));
if (json.isObject()) {
if (proxy_->router().opts().destination_rate_limiting) {
if (auto jrates = json.get_ptr("rates")) {
route = makeRateLimitRoute(std::move(route), RateLimiter(*jrates));
}
}
if (auto jsplits = json.get_ptr("shard_splits")) {
route = makeShardSplitRoute(std::move(route), ShardSplitter(*jsplits));
}
}
auto asynclogName = pool->getName();
bool needAsynclog = true;
if (json.isObject()) {
if (auto jasynclog = json.get_ptr("asynclog")) {
checkLogic(jasynclog->isBool(), "PoolRoute: asynclog is not bool");
needAsynclog = jasynclog->getBool();
}
if (auto jname = json.get_ptr("name")) {
checkLogic(jname->isString(), "PoolRoute: name is not a string");
asynclogName = jname->stringPiece().str();
}
}
if (needAsynclog) {
route = createAsynclogRoute(std::move(route), asynclogName);
}
return route;
}