Example #1
0
ProxyConfig::ProxyConfig(proxy_t* proxy,
                         const folly::dynamic& json,
                         std::string configMd5Digest,
                         std::shared_ptr<PoolFactory> poolFactory)
  : poolFactory_(std::move(poolFactory)),
    configMd5Digest_(std::move(configMd5Digest)) {

  McRouteHandleProvider provider(proxy, *proxy->destinationMap, *poolFactory_);
  RouteHandleFactory<McrouterRouteHandleIf> factory(provider);

  checkLogic(json.isObject(), "Config is not an object");

  if (json.count("named_handles")) {
    checkLogic(json["named_handles"].isArray(), "named_handles is not array");
    for (const auto& it : json["named_handles"]) {
      factory.create(it);
    }
  }

  RouteSelectorMap routeSelectors;

  auto jRoute = json.get_ptr("route");
  auto jRoutes = json.get_ptr("routes");
  checkLogic(!jRoute || !jRoutes,
             "Invalid config: both 'route' and 'routes' are specified");
  checkLogic(jRoute || jRoutes, "No route/routes in config");
  if (jRoute) {
    routeSelectors[proxy->getRouterOptions().default_route] =
        std::make_shared<PrefixSelectorRoute>(factory, *jRoute);
  } else { // jRoutes
    checkLogic(jRoutes->isArray() || jRoutes->isObject(),
               "Config: routes is not array/object");
    if (jRoutes->isArray()) {
      for (const auto& it : *jRoutes) {
        checkLogic(it.isObject(), "RoutePolicy is not an object");
        auto jCurRoute = it.get_ptr("route");
        auto jAliases = it.get_ptr("aliases");
        checkLogic(jCurRoute, "RoutePolicy: no route");
        checkLogic(jAliases, "RoutePolicy: no aliases");
        checkLogic(jAliases->isArray(), "RoutePolicy: aliases is not an array");
        auto routeSelector =
            std::make_shared<PrefixSelectorRoute>(factory, *jCurRoute);
        for (const auto& alias : *jAliases) {
          checkLogic(alias.isString(), "RoutePolicy: alias is not a string");
          routeSelectors[alias.stringPiece()] = routeSelector;
        }
      }
    } else { // object
      for (const auto& it : jRoutes->items()) {
        checkLogic(it.first.isString(), "RoutePolicy: alias is not a string");
        routeSelectors[it.first.stringPiece()] =
            std::make_shared<PrefixSelectorRoute>(factory, it.second);
      }
    }
  }

  asyncLogRoutes_ = provider.releaseAsyncLogRoutes();
  proxyRoute_ = std::make_shared<ProxyRoute>(proxy, routeSelectors);
  serviceInfo_ = std::make_shared<ServiceInfo>(proxy, *this);
}
Example #2
0
void PoolFactory::parseQos(std::string parentName, const folly::dynamic& jQos,
                           uint64_t& qosClass, uint64_t& qosPath) {
  if (!jQos.isObject()) {
    MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
                   "{}: qos must be an object.", parentName);
    return;
  }

  uint64_t prevClass = qosClass;
  if (auto jClass = jQos.get_ptr("class")) {
    if (jClass->isInt() && isQosClassValid(jClass->getInt())) {
      qosClass = jClass->getInt();
    } else {
      MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
                     "{}: qos.class must be an integer in the range [0, 4]",
                     parentName);
    }
  }
  if (auto jPath = jQos.get_ptr("path")) {
    if (jPath->isInt() && isQosPathValid(jPath->getInt())) {
      qosPath = jPath->getInt();
    } else {
      MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
                     "{}: qos.path must be an integer in the range [0, 3]",
                     parentName);
      qosClass = prevClass;
    }
  }
}
Example #3
0
McrouterRouteHandlePtr makeHashRoute(
  const folly::dynamic& json,
  std::vector<McrouterRouteHandlePtr> rh,
  size_t threadId) {

  std::string salt;
  folly::StringPiece funcType = Ch3HashFunc::type();
  if (json.isObject()) {
    if (auto jsalt = json.get_ptr("salt")) {
      checkLogic(jsalt->isString(), "HashRoute: salt is not a string");
      salt = jsalt->getString();
    }
    if (auto jhashFunc = json.get_ptr("hash_func")) {
      checkLogic(jhashFunc->isString(),
                 "HashRoute: hash_func is not a string");
      funcType = jhashFunc->stringPiece();
    }
  }

  if (funcType == Ch3HashFunc::type()) {
    return makeHashRouteCh3(std::move(rh), std::move(salt));
  } else if (funcType == Crc32HashFunc::type()) {
    return makeHashRouteCrc32(std::move(rh), std::move(salt));
  } else if (funcType == WeightedCh3HashFunc::type()) {
    WeightedCh3HashFunc func{json, rh.size()};
    return makeHashRouteWeightedCh3(std::move(rh), std::move(salt),
                                    std::move(func));
  } else if (funcType == ConstShardHashFunc::type()) {
    return makeHashRouteConstShard(std::move(rh), std::move(salt));
  } else if (funcType == "Latest") {
    return makeLatestRoute(json, std::move(rh), threadId);
  }
  throwLogic("Unknown hash function: {}", funcType);
}
McrouterRouteHandlePtr makeFailoverWithExptimeRoute(
    RouteHandleFactory<McrouterRouteHandleIf>& factory,
    const folly::dynamic& json) {
  checkLogic(json.isObject(), "FailoverWithExptimeRoute is not an object");
  auto jnormal = json.get_ptr("normal");
  checkLogic(jnormal, "FailoverWithExptimeRoute: normal not found");
  auto normal = factory.create(*jnormal);

  int32_t failoverExptime = 60;
  if (auto jexptime = json.get_ptr("failover_exptime")) {
    checkLogic(jexptime->isInt(), "FailoverWithExptimeRoute: "
                                  "failover_exptime is not an integer");
    failoverExptime = jexptime->getInt();
  }

  std::vector<McrouterRouteHandlePtr> failover;
  if (auto jfailover = json.get_ptr("failover")) {
    failover = factory.createList(*jfailover);
  }

  auto children = getFailoverChildren(std::move(normal),
                                      std::move(failover),
                                      failoverExptime);
  return makeFailoverRoute(json, std::move(children));
}
McrouterRouteHandlePtr makeRateLimitRoute(
    RouteHandleFactory<McrouterRouteHandleIf>& factory,
    const folly::dynamic& json) {
  checkLogic(json.isObject(), "RateLimitRoute is not an object");
  auto jtarget = json.get_ptr("target");
  checkLogic(jtarget, "RateLimitRoute: target not found");
  auto target = factory.create(*jtarget);
  auto jrates = json.get_ptr("rates");
  checkLogic(jrates, "RateLimitRoute: rates not found");
  return makeRateLimitRoute(std::move(target), RateLimiter(*jrates));
}
Example #6
0
McrouterRouteHandlePtr makeLatestRoute(
  const folly::dynamic& json,
  std::vector<McrouterRouteHandlePtr> targets) {

  size_t failoverCount = 5;
  FailoverErrorsSettings failoverErrors;
  if (json.isObject()) {
    if (auto jfailoverCount = json.get_ptr("failover_count")) {
      checkLogic(jfailoverCount->isInt(),
                 "LatestRoute: failover_count is not an integer");
      failoverCount = jfailoverCount->getInt();
    }
    if (auto jFailoverErrors = json.get_ptr("failover_errors")) {
      failoverErrors = FailoverErrorsSettings(*jFailoverErrors);
    }
  }
  return makeLatestRoute(std::move(targets), failoverCount,
                         std::move(failoverErrors));
}
FailoverErrorsSettings::FailoverErrorsSettings(const folly::dynamic& json) {
  checkLogic(json.isObject() || json.isArray(),
      "Failover errors must be either an array or an object.");

  if (json.isObject()) {
    if (auto jsonGets = json.get_ptr("gets")) {
      gets_ = FailoverErrorsSettings::List(*jsonGets);
    }
    if (auto jsonUpdates = json.get_ptr("updates")) {
      updates_ = FailoverErrorsSettings::List(*jsonUpdates);
    }
    if (auto jsonDeletes = json.get_ptr("deletes")) {
      deletes_ = FailoverErrorsSettings::List(*jsonDeletes);
    }
  } else if (json.isArray()) {
    gets_ = FailoverErrorsSettings::List(json);
    updates_ = FailoverErrorsSettings::List(json);
    deletes_ = FailoverErrorsSettings::List(json);
  }
}
Example #8
0
PoolFactory::PoolJson PoolFactory::parsePool(const folly::dynamic& json) {
  checkLogic(json.isString() || json.isObject(),
             "Pool should be a string (name of pool) or an object");
  if (json.isString()) {
    return parseNamedPool(json.stringPiece());
  }
  auto jname = json.get_ptr("name");
  checkLogic(jname && jname->isString(), "Pool should have string 'name'");
  pools_.emplace(jname->stringPiece(), std::make_pair(json, PoolState::NEW));
  return parseNamedPool(jname->stringPiece());
}
/**
 * @return target and asynclogName
 *         Caller may call makeAsynclogRoute afterwards.
 */
std::pair<McrouterRouteHandlePtr, std::string> parseAsynclogRoute(
    RouteHandleFactory<McrouterRouteHandleIf>& factory,
    const folly::dynamic& json) {
  std::string asynclogName;
  McrouterRouteHandlePtr target;
  checkLogic(json.isObject() || json.isString(),
             "AsynclogRoute should be object or string");
  if (json.isString()) {
    asynclogName = json.stringPiece().str();
    target = factory.create(json);
  } else { // object
    auto jname = json.get_ptr("name");
    checkLogic(jname && jname->isString(),
               "AsynclogRoute: required string name");
    auto jtarget = json.get_ptr("target");
    checkLogic(jtarget, "AsynclogRoute: target not found");
    asynclogName = jname->stringPiece().str();
    target = factory.create(*jtarget);
  }
  return { std::move(target), std::move(asynclogName) };
}
Example #10
0
std::shared_ptr<ClientPool>
PoolFactory::parsePool(const folly::dynamic& json) {
  checkLogic(json.isString() || json.isObject(),
             "Pool should be a string (name of pool) or an object");
  if (json.isString()) {
    return parsePool(json.stringPiece().str(), json);
  } else {
    auto name = json.get_ptr("name");
    checkLogic(name && name->isString(), "Pool should have string 'name'");
    return parsePool(name->stringPiece().str(), json);
  }
}
Example #11
0
PoolFactory::PoolFactory(const folly::dynamic& config, ConfigApiIf& configApi)
    : configApi_(configApi) {
  checkLogic(config.isObject(), "config is not an object");
  if (auto jpools = config.get_ptr("pools")) {
    checkLogic(jpools->isObject(), "config: 'pools' is not an object");

    for (const auto& it : jpools->items()) {
      pools_.emplace(
          it.first.stringPiece(), std::make_pair(it.second, PoolState::NEW));
    }
  }
}
Example #12
0
std::shared_ptr<ShadowSettings>
ShadowSettings::create(const folly::dynamic& json, McrouterInstance& router) {
  auto result = std::shared_ptr<ShadowSettings>(new ShadowSettings());
  try {
    checkLogic(json.isObject(), "json is not an object");
    if (auto jKeyFractionRange = json.get_ptr("key_fraction_range")) {
      checkLogic(jKeyFractionRange->isArray(),
                 "key_fraction_range is not an array");
      auto ar = folly::convertTo<std::vector<double>>(*jKeyFractionRange);
      checkLogic(ar.size() == 2, "key_fraction_range size is not 2");
      result->setKeyRange(ar[0], ar[1]);
    }
    if (auto jIndexRange = json.get_ptr("index_range")) {
      checkLogic(jIndexRange->isArray(), "index_range is not an array");
      auto ar = folly::convertTo<std::vector<size_t>>(*jIndexRange);
      checkLogic(ar.size() == 2, "index_range size is not 2");
      checkLogic(ar[0] <= ar[1], "index_range start > end");
      result->startIndex_ = ar[0];
      result->endIndex_ = ar[1];
    }
    if (auto jKeyFractionRangeRv = json.get_ptr("key_fraction_range_rv")) {
      checkLogic(jKeyFractionRangeRv->isString(),
                 "key_fraction_range_rv is not a string");
      result->keyFractionRangeRv_ = jKeyFractionRangeRv->stringPiece().str();
    }
    if (auto jValidateReplies = json.get_ptr("validate_replies")) {
      checkLogic(jValidateReplies->isBool(),
                 "validate_replies is not a bool");
      result->validateReplies_ = jValidateReplies->getBool();
    }
  } catch (const std::logic_error& e) {
    MC_LOG_FAILURE(router.opts(), failure::Category::kInvalidConfig,
                   "ShadowSettings: {}", e.what());
    return nullptr;
  }

  result->registerOnUpdateCallback(router);

  return result;
}
Example #13
0
McrouterRouteHandlePtr makeLatestRoute(
    RouteHandleFactory<McrouterRouteHandleIf>& factory,
    const folly::dynamic& json) {
  std::vector<McrouterRouteHandlePtr> children;
  if (json.isObject()) {
    if (auto jchildren = json.get_ptr("children")) {
      children = factory.createList(*jchildren);
    }
  } else {
    children = factory.createList(json);
  }
  return makeLatestRoute(json, std::move(children));
}
McrouterRouteHandlePtr makeFailoverRoute(
    const folly::dynamic& json,
    std::vector<McrouterRouteHandlePtr> children) {

  FailoverErrorsSettings failoverErrors;
  std::unique_ptr<FailoverRateLimiter> rateLimiter;
  bool failoverTagging = false;
  if (json.isObject()) {
    if (auto jFailoverErrors = json.get_ptr("failover_errors")) {
      failoverErrors = FailoverErrorsSettings(*jFailoverErrors);
    }
    if (auto jFailoverTag = json.get_ptr("failover_tag")) {
      checkLogic(jFailoverTag->isBool(),
                 "Failover: failover_tag is not bool");
      failoverTagging = jFailoverTag->getBool();
    }
    if (auto jFailoverLimit = json.get_ptr("failover_limit")) {
      rateLimiter = folly::make_unique<FailoverRateLimiter>(*jFailoverLimit);
    }
  }
  return makeFailoverRoute(std::move(children), std::move(failoverErrors),
                           std::move(rateLimiter), failoverTagging);
}
Example #15
0
PoolFactory::PoolFactory(const folly::dynamic& config,
                         ConfigApi& configApi,
                         const McrouterOptions& opts)
  : configApi_(configApi),
    opts_(opts) {

  checkLogic(config.isObject(), "config is not an object");
  if (auto jpools = config.get_ptr("pools")) {
    checkLogic(jpools->isObject(), "config: 'pools' is not an object");

    for (const auto& it : jpools->items()) {
      parsePool(it.first.stringPiece().str(), it.second);
    }
  }
}
Example #16
0
McrouterRouteHandlePtr makeFailoverWithExptimeRoute(
    RouteHandleFactory<McrouterRouteHandleIf>& factory,
    const folly::dynamic& json) {
  checkLogic(json.isObject(), "FailoverWithExptimeRoute is not an object");

  McrouterRouteHandlePtr normal;
  if (auto jnormal = json.get_ptr("normal")) {
    normal = factory.create(*jnormal);
  }

  std::vector<McrouterRouteHandlePtr> failoverTargets;
  if (auto jfailover = json.get_ptr("failover")) {
    failoverTargets = factory.createList(*jfailover);
  }

  int32_t failoverExptime = 60;
  if (auto jexptime = json.get_ptr("failover_exptime")) {
    checkLogic(jexptime->isInt(), "FailoverWithExptimeRoute: "
                                  "failover_exptime is not an integer");
    failoverExptime = jexptime->getInt();
  }

  // Check if only one format is being used
  checkLogic(!(json.count("settings") && // old
        (json.count("failover_errors") || json.count("failover_tag"))), // new
      "Use either 'settings' (old format) or 'failover_errors' / 'failover_tag'"
    );

  // new format
  FailoverErrorsSettings failoverErrors;
  bool failoverTagging = false;
  if (auto jfailoverTag = json.get_ptr("failover_tag")) {
    checkLogic(jfailoverTag->isBool(),
               "FailoverWithExptime: failover_tag is not bool");
    failoverTagging = jfailoverTag->getBool();
  }
  if (auto jfailoverErrors = json.get_ptr("failover_errors")) {
    failoverErrors = FailoverErrorsSettings(*jfailoverErrors);
  }

  // old format
  if (auto jsettings = json.get_ptr("settings")) {
    VLOG(1) << "FailoverWithExptime: This config format is deprecated. "
               "Use 'failover_errors' instead of 'settings'.";
    auto oldSettings = FailoverWithExptimeSettings(*jsettings);
    failoverTagging = oldSettings.failoverTagging;
    failoverErrors = oldSettings.getFailoverErrors();
  }

  return makeFailoverWithExptimeRoute(
    std::move(normal),
    std::move(failoverTargets),
    failoverExptime,
    std::move(failoverErrors),
    failoverTagging);
}
Example #17
0
std::vector<McrouterRouteHandlePtr> makeShadowRoutes(
    RouteHandleFactory<McrouterRouteHandleIf>& factory,
    const folly::dynamic& json,
    proxy_t& proxy,
    ExtraRouteHandleProviderIf& extraProvider) {
  checkLogic(json.isObject(), "ShadowRoute should be an object");
  const auto jchildren = json.get_ptr("children");
  checkLogic(jchildren, "ShadowRoute: children not found");
  auto children = factory.createList(*jchildren);
  if (json.count("shadows")) {
    children = makeShadowRoutes(
        factory, json, std::move(children), proxy, extraProvider);
  }
  return children;
}
Example #18
0
McrouterRouteHandlePtr makeErrorRoute(
    RouteHandleFactory<McrouterRouteHandleIf>& factory,
    const folly::dynamic& json) {
  checkLogic(json.isObject() || json.isString() || json.isNull(),
             "ErrorRoute: should be string or object");
  std::string response;
  if (json.isString()) {
    response = json.stringPiece().str();
  } else if (json.isObject()) {
    if (auto jResponse = json.get_ptr("response")) {
      checkLogic(jResponse->isString(), "ErrorRoute: response is not a string");
      response = jResponse->stringPiece().str();
    }
  }
  return makeErrorRoute(std::move(response));
}
McrouterRouteHandlePtr makeOperationSelectorRoute(
    RouteHandleFactory<McrouterRouteHandleIf>& factory,
    const folly::dynamic& json) {
  if (!json.isObject()) {
    return factory.create(json);
  }

  McrouterRouteHandlePtr defaultPolicy;
  if (json.count("default_policy")) {
    defaultPolicy = factory.create(json["default_policy"]);
  }

  std::vector<McrouterRouteHandlePtr> operationPolicies{mc_nops};
  if (auto jOpPolicies = json.get_ptr("operation_policies")) {
    checkLogic(jOpPolicies->isObject(),
               "OperationSelectorRoute: operation_policies is not an object");

    std::map<std::string, const folly::dynamic*> orderedPolicies;
    for (auto& it : jOpPolicies->items()) {
      checkLogic(it.first.isString(),
                 "OperationSelectorRoute: operation_policies' "
                 "key is not a string");
      auto key = it.first.stringPiece().str();
      orderedPolicies.emplace(std::move(key), &it.second);
    }

    // order is important here: named handles may not be resolved if we parse
    // policies in random order
    for (const auto& it : orderedPolicies) {
      auto opId = mc_op_from_string(it.first.data());
      checkLogic(opId != mc_op_unknown, "Unknown mc operation: {}", it.first);
      operationPolicies[opId] = factory.create(*it.second);
    }

    return makeOperationSelectorRoute(std::move(operationPolicies),
                                      std::move(defaultPolicy));
  }

  return defaultPolicy;
}
Example #20
0
std::vector<McrouterRouteHandlePtr> makeShadowRoutes(
    RouteHandleFactory<McrouterRouteHandleIf>& factory,
    const folly::dynamic& json,
    std::vector<McrouterRouteHandlePtr> children,
    proxy_t& proxy,
    ExtraRouteHandleProviderIf& extraProvider) {
  folly::StringPiece shadowPolicy = "default";
  if (auto jshadow_policy = json.get_ptr("shadow_policy")) {
    checkLogic(jshadow_policy->isString(),
               "ShadowRoute: shadow_policy is not a string");
    shadowPolicy = jshadow_policy->stringPiece();
  }

  auto jshadows = json.get_ptr("shadows");
  checkLogic(jshadows, "ShadowRoute: route doesn't contain shadows field");

  if (!jshadows->isArray()) {
    MC_LOG_FAILURE(proxy.router().opts(),
                   failure::Category::kInvalidConfig,
                   "ShadowRoute: shadows specified in route is not an array");
    return children;
  }

  McrouterShadowData data;
  for (auto& shadow : *jshadows) {
    if (!shadow.isObject()) {
      MC_LOG_FAILURE(proxy.router().opts(),
                     failure::Category::kInvalidConfig,
                     "ShadowRoute: shadow is not an object");
      continue;
    }
    auto jtarget = shadow.get_ptr("target");
    if (!jtarget) {
      MC_LOG_FAILURE(proxy.router().opts(),
                     failure::Category::kInvalidConfig,
                     "ShadowRoute shadows: no target for shadow");
      continue;
    }
    try {
      auto s = ShadowSettings::create(shadow, proxy.router());
      if (s) {
        data.emplace_back(factory.create(*jtarget), std::move(s));
      }
    } catch (const std::exception& e) {
      MC_LOG_FAILURE(proxy.router().opts(),
                     failure::Category::kInvalidConfig,
                     "Can not create shadow for ShadowRoute: {}",
                     e.what());
    }
  }
  for (size_t i = 0; i < children.size(); ++i) {
    McrouterShadowData childrenShadows;
    for (const auto& shadowData : data) {
      if (shadowData.second->startIndex() <= i &&
          i < shadowData.second->endIndex()) {
        childrenShadows.push_back(shadowData);
      }
    }
    if (!childrenShadows.empty()) {
      childrenShadows.shrink_to_fit();
      children[i] = extraProvider.makeShadow(proxy,
                                                  std::move(children[i]),
                                                  std::move(childrenShadows),
                                                  shadowPolicy);
    }
  }
  return children;
}
Example #21
0
std::shared_ptr<ClientPool>
PoolFactory::parsePool(const std::string& name, const folly::dynamic& json) {
  auto seenPoolIt = pools_.find(name);
  if (seenPoolIt != pools_.end()) {
    return seenPoolIt->second;
  }

  if (json.isString()) {
    // get the pool from ConfigApi
    std::string jsonStr;
    checkLogic(configApi_.get(ConfigType::Pool, name, jsonStr),
               "Can not read pool: {}", name);
    return parsePool(name, parseJsonString(jsonStr));
  } else {
    // one day we may add inheriting from local pool
    if (auto jinherit = json.get_ptr("inherit")) {
      checkLogic(jinherit->isString(),
                 "Pool {}: inherit is not a string", name);
      auto path = jinherit->stringPiece().str();
      std::string jsonStr;
      checkLogic(configApi_.get(ConfigType::Pool, path, jsonStr),
                 "Can not read pool from: {}", path);
      auto newJson = parseJsonString(jsonStr);
      for (auto& it : json.items()) {
        newJson.insert(it.first, it.second);
      }
      newJson.erase("inherit");
      return parsePool(name, newJson);
    }
  }

  // pool_locality
  std::chrono::milliseconds timeout{opts_.server_timeout_ms};
  if (auto jlocality = json.get_ptr("pool_locality")) {
    if (!jlocality->isString()) {
      MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
                     "Pool {}: pool_locality is not a string", name);
    } else {
      auto str = jlocality->stringPiece();
      if (str == "cluster") {
        if (opts_.cluster_pools_timeout_ms != 0) {
          timeout = std::chrono::milliseconds(opts_.cluster_pools_timeout_ms);
        }
      } else if (str == "region") {
        if (opts_.regional_pools_timeout_ms != 0) {
          timeout = std::chrono::milliseconds(opts_.regional_pools_timeout_ms);
        }
      } else {
        MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
                       "Pool {}: '{}' pool locality is not supported",
                       name, str);
      }
    }
  }

  // region & cluster
  std::string region, cluster;
  if (auto jregion = json.get_ptr("region")) {
    if (!jregion->isString()) {
      MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
                     "Pool {}: pool_region is not a string", name);
    } else {
      region = jregion->stringPiece().str();
    }
  }
  if (auto jcluster = json.get_ptr("cluster")) {
    if (!jcluster->isString()) {
      MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
                     "Pool {}: pool_cluster is not a string", name);
    } else {
      cluster = jcluster->stringPiece().str();
    }
  }

  if (auto jtimeout = json.get_ptr("server_timeout")) {
    if (!jtimeout->isInt()) {
      MC_LOG_FAILURE(opts_, memcache::failure::Category::kInvalidConfig,
                     "Pool {}: server_timeout is not an int", name);
    } else {
      timeout = std::chrono::milliseconds(jtimeout->getInt());
    }
  }

  if (!region.empty() && !cluster.empty()) {
    auto& route = opts_.default_route;
    if (region == route.getRegion() && cluster == route.getCluster()) {
      if (opts_.within_cluster_timeout_ms != 0) {
        timeout = std::chrono::milliseconds(opts_.within_cluster_timeout_ms);
      }
    } else if (region == route.getRegion()) {
      if (opts_.cross_cluster_timeout_ms != 0) {
        timeout = std::chrono::milliseconds(opts_.cross_cluster_timeout_ms);
      }
    } else {
      if (opts_.cross_region_timeout_ms != 0) {
        timeout = std::chrono::milliseconds(opts_.cross_region_timeout_ms);
      }
    }
  }

  auto protocol = parseProtocol(json, mc_ascii_protocol);

  bool keep_routing_prefix = false;
  if (auto jkeep_routing_prefix = json.get_ptr("keep_routing_prefix")) {
    checkLogic(jkeep_routing_prefix->isBool(),
               "Pool {}: keep_routing_prefix is not a bool");
    keep_routing_prefix = jkeep_routing_prefix->getBool();
  }

  uint64_t qosClass = opts_.default_qos_class;
  uint64_t qosPath = opts_.default_qos_path;
  if (auto jqos = json.get_ptr("qos")) {
    parseQos(folly::sformat("Pool {}", name), *jqos, qosClass, qosPath);
  }

  bool useSsl = false;
  if (auto juseSsl = json.get_ptr("use_ssl")) {
    checkLogic(juseSsl->isBool(), "Pool {}: use_ssl is not a bool", name);
    useSsl = juseSsl->getBool();
  }

  // servers
  auto jservers = json.get_ptr("servers");
  checkLogic(jservers, "Pool {}: servers not found", name);
  checkLogic(jservers->isArray(), "Pool {}: servers is not an array", name);
  auto clientPool = std::make_shared<ClientPool>(name);
  for (size_t i = 0; i < jservers->size(); ++i) {
    const auto& server = jservers->at(i);
    std::shared_ptr<AccessPoint> ap;
    bool serverUseSsl = useSsl;
    uint64_t serverQosClass = qosClass;
    uint64_t serverQosPath = qosPath;
    checkLogic(server.isString() || server.isObject(),
               "Pool {}: server #{} is not a string/object", name, i);
    if (server.isString()) {
      // we support both host:port and host:port:protocol
      ap = AccessPoint::create(server.stringPiece(), protocol);
      checkLogic(ap != nullptr,
                 "Pool {}: invalid server {}", name, server.stringPiece());
    } else { // object
      auto jhostname = server.get_ptr("hostname");
      checkLogic(jhostname,
                 "Pool {}: hostname not found for server #{}", name, i);
      checkLogic(jhostname->isString(),
                 "Pool {}: hostname is not a string for server #{}", name, i);

      if (auto jqos = server.get_ptr("qos")) {
        parseQos(folly::sformat("Pool {}, server #{}", name, i),
                 *jqos, qosClass, qosPath);
      }

      if (auto juseSsl = server.get_ptr("use_ssl")) {
        checkLogic(juseSsl->isBool(),
                   "Pool {}: use_ssl is not a bool for server #{}", name, i);
        serverUseSsl = juseSsl->getBool();
      }

      ap = AccessPoint::create(jhostname->stringPiece(),
                               parseProtocol(server, protocol));
      checkLogic(ap != nullptr, "Pool {}: invalid server #{}", name, i);
    }

    auto client = clientPool->emplaceClient(
      timeout,
      std::move(ap),
      keep_routing_prefix,
      serverUseSsl,
      serverQosClass,
      serverQosPath);

    clients_.push_back(std::move(client));
  } // servers

  // weights
  if (auto jweights = json.get_ptr("weights")) {
    clientPool->setWeights(*jweights);
  }

  pools_.emplace(name, clientPool);
  return clientPool;
}
Example #22
0
McrouterRouteHandlePtr McRouteHandleProvider::makePoolRoute(
  McRouteHandleFactory& factory, const folly::dynamic& json) {

  checkLogic(json.isObject() || json.isString(),
             "PoolRoute should be object or string");
  const folly::dynamic* jpool;
  if (json.isObject()) {
    jpool = json.get_ptr("pool");
    checkLogic(jpool, "PoolRoute: pool not found");
  } else {
    jpool = &json;
  }
  auto p = makePool(*jpool);
  auto pool = std::move(p.first);
  auto destinations = std::move(p.second);

  if (json.isObject()) {
    if (auto maxOutstandingPtr = json.get_ptr("max_outstanding")) {
      checkLogic(maxOutstandingPtr->isInt(),
                 "PoolRoute {}: max_outstanding is not int", pool->getName());
      auto maxOutstanding = maxOutstandingPtr->asInt();
      if (maxOutstanding) {
        for (auto& destination: destinations) {
          destination = makeOutstandingLimitRoute(std::move(destination),
                                                  maxOutstanding);
        }
      }
    }
  }

  if (json.isObject() && json.count("shadows")) {
    folly::StringPiece shadowPolicy = "default";
    if (auto jshadow_policy = json.get_ptr("shadow_policy")) {
      checkLogic(jshadow_policy->isString(),
                 "PoolRoute: shadow_policy is not a string");
      shadowPolicy = jshadow_policy->stringPiece();
    }

    McrouterShadowData data;
    for (auto& shadow : json["shadows"]) {
      checkLogic(shadow.count("target"),
                 "PoolRoute {} shadows: no target for shadow", pool->getName());
      auto s = ShadowSettings::create(shadow, proxy_->router());
      if (s) {
        data.emplace_back(factory.create(shadow["target"]), std::move(s));
      }
    }

    for (size_t i = 0; i < destinations.size(); ++i) {
      McrouterShadowData destinationShadows;
      for (const auto& shadowData : data) {
        if (shadowData.second->startIndex() <= i &&
            i < shadowData.second->endIndex()) {
          destinationShadows.push_back(shadowData);
        }
      }
      if (!destinationShadows.empty()) {
        destinationShadows.shrink_to_fit();
        destinations[i] = extraProvider_->makeShadow(
          proxy_, std::move(destinations[i]),
          std::move(destinationShadows), shadowPolicy);
      }
    }
  }

  // add weights and override whatever we have in PoolRoute::hash
  folly::dynamic jhashWithWeights = folly::dynamic::object();
  if (pool->getWeights()) {
    jhashWithWeights = folly::dynamic::object
      ("hash_func", WeightedCh3HashFunc::type())
      ("weights", *pool->getWeights());
  }

  if (json.isObject()) {
    if (auto jhash = json.get_ptr("hash")) {
      checkLogic(jhash->isObject() || jhash->isString(),
                 "PoolRoute {}: hash is not object/string", pool->getName());
      if (jhash->isString()) {
        jhashWithWeights["hash_func"] = *jhash;
      } else { // object
        for (const auto& it : jhash->items()) {
          jhashWithWeights[it.first] = it.second;
        }
      }
    }
  }
  auto route = makeHashRoute(jhashWithWeights, std::move(destinations));

  if (json.isObject()) {
    if (proxy_->router().opts().destination_rate_limiting) {
      if (auto jrates = json.get_ptr("rates")) {
        route = makeRateLimitRoute(std::move(route), RateLimiter(*jrates));
      }
    }

    if (auto jsplits = json.get_ptr("shard_splits")) {
      route = makeShardSplitRoute(std::move(route), ShardSplitter(*jsplits));
    }
  }

  auto asynclogName = pool->getName();
  bool needAsynclog = true;
  if (json.isObject()) {
    if (auto jasynclog = json.get_ptr("asynclog")) {
      checkLogic(jasynclog->isBool(), "PoolRoute: asynclog is not bool");
      needAsynclog = jasynclog->getBool();
    }
    if (auto jname = json.get_ptr("name")) {
      checkLogic(jname->isString(), "PoolRoute: name is not a string");
      asynclogName = jname->stringPiece().str();
    }
  }
  if (needAsynclog) {
    route = createAsynclogRoute(std::move(route), asynclogName);
  }

  return route;
}
Example #23
0
ProxyConfig::ProxyConfig(proxy_t& proxy,
                         const folly::dynamic& json,
                         std::string configMd5Digest,
                         PoolFactory& poolFactory)
  : configMd5Digest_(std::move(configMd5Digest)) {

  McRouteHandleProvider provider(proxy, poolFactory);
  RouteHandleFactory<McrouterRouteHandleIf> factory(provider);

  checkLogic(json.isObject(), "Config is not an object");

  if (auto jNamedHandles = json.get_ptr("named_handles")) {
    if (jNamedHandles->isArray()) {
      for (const auto& it : *jNamedHandles) {
        factory.create(it);
      }
    } else if (jNamedHandles->isObject()) {
      for (const auto& it : jNamedHandles->items()) {
        factory.addNamed(it.first.stringPiece(), it.second);
      }
    } else {
      throwLogic("named_handles is {} expected array/object",
                 jNamedHandles->typeName());
    }
  }

  RouteSelectorMap routeSelectors;

  auto jRoute = json.get_ptr("route");
  auto jRoutes = json.get_ptr("routes");
  checkLogic(!jRoute || !jRoutes,
             "Invalid config: both 'route' and 'routes' are specified");
  if (jRoute) {
    routeSelectors[proxy.getRouterOptions().default_route] =
        std::make_shared<PrefixSelectorRoute>(factory, *jRoute);
  } else if (jRoutes) { // jRoutes
    checkLogic(jRoutes->isArray() || jRoutes->isObject(),
               "Config: routes is not array/object");
    if (jRoutes->isArray()) {
      for (const auto& it : *jRoutes) {
        checkLogic(it.isObject(), "RoutePolicy is not an object");
        auto jCurRoute = it.get_ptr("route");
        auto jAliases = it.get_ptr("aliases");
        checkLogic(jCurRoute, "RoutePolicy: no route");
        checkLogic(jAliases, "RoutePolicy: no aliases");
        checkLogic(jAliases->isArray(), "RoutePolicy: aliases is not an array");
        auto routeSelector =
            std::make_shared<PrefixSelectorRoute>(factory, *jCurRoute);
        for (const auto& alias : *jAliases) {
          checkLogic(alias.isString(), "RoutePolicy: alias is not a string");
          routeSelectors[alias.stringPiece()] = routeSelector;
        }
      }
    } else { // object
      for (const auto& it : jRoutes->items()) {
        checkLogic(it.first.isString(), "RoutePolicy: alias is not a string");
        routeSelectors[it.first.stringPiece()] =
            std::make_shared<PrefixSelectorRoute>(factory, it.second);
      }
    }
  } else {
    throwLogic("No route/routes in config");
  }

  asyncLogRoutes_ = provider.releaseAsyncLogRoutes();
  pools_ = provider.releasePools();
  accessPoints_ = provider.releaseAccessPoints();
  proxyRoute_ = std::make_shared<ProxyRoute>(&proxy, routeSelectors);
  serviceInfo_ = std::make_shared<ServiceInfo>(&proxy, *this);
}