TEST_F(PacksTests, test_sharding) {
auto shard1 = getMachineShard("localhost.localdomain");
auto shard2 = getMachineShard("not.localhost.localdomain");
// Expect some static caching.
EXPECT_EQ(shard1, shard2);
// Bypass the caching.
shard2 = getMachineShard("not.localhost.localdomain", true);
EXPECT_NE(shard1, shard2);
}
void Pack::initialize(const std::string& name,
const std::string& source,
const pt::ptree& tree) {
name_ = name;
source_ = source;
// Check the shard limitation, shards falling below this value are included.
if (tree.count("shard") > 0) {
shard_ = tree.get<size_t>("shard", 0);
}
// Check for a platform restriction.
platform_.clear();
if (tree.count("platform") > 0) {
platform_ = tree.get<std::string>("platform", "");
}
// Check for a version restriction.
version_.clear();
if (tree.count("version") > 0) {
version_ = tree.get<std::string>("version", "");
}
// Apply the shard, platform, and version checking.
// It is important to set each value such that the packs meta-table can report
// each of the restrictions.
if ((shard_ > 0 && shard_ < getMachineShard()) || !checkPlatform() ||
!checkVersion()) {
return;
}
discovery_queries_.clear();
if (tree.count("discovery") > 0) {
for (const auto& item : tree.get_child("discovery")) {
discovery_queries_.push_back(item.second.get_value<std::string>());
}
}
// Initialize a discovery cache at time 0.
discovery_cache_ = std::make_pair<size_t, bool>(0, false);
valid_ = true;
// If the splay percent is less than 1 reset to a sane estimate.
if (FLAGS_schedule_splay_percent <= 1) {
FLAGS_schedule_splay_percent = 10;
}
schedule_.clear();
if (tree.count("queries") == 0) {
// This pack contained no queries.
return;
}
// Iterate the queries (or schedule) and check platform/version/sanity.
for (const auto& q : tree.get_child("queries")) {
if (q.second.count("shard") > 0) {
auto shard = q.second.get<size_t>("shard", 0);
if (shard > 0 && shard < getMachineShard()) {
continue;
}
}
if (q.second.count("platform")) {
if (!checkPlatform(q.second.get<std::string>("platform", ""))) {
continue;
}
}
if (q.second.count("version")) {
if (!checkVersion(q.second.get<std::string>("version", ""))) {
continue;
}
}
ScheduledQuery query;
query.query = q.second.get<std::string>("query", "");
query.interval = q.second.get("interval", FLAGS_schedule_default_interval);
if (query.interval <= 0 || query.query.empty() || query.interval > 592200) {
// Invalid pack query.
VLOG(1) << "Query has invalid interval: " << q.first << ": "
<< query.interval;
continue;
}
query.splayed_interval = restoreSplayedValue(q.first, query.interval);
query.options["snapshot"] = q.second.get<bool>("snapshot", false);
query.options["removed"] = q.second.get<bool>("removed", true);
schedule_[q.first] = query;
}
}
void Pack::initialize(const std::string& name,
const std::string& source,
const rj::Value& obj) {
name_ = name;
source_ = source;
// Check the shard limitation, shards falling below this value are included.
if (obj.HasMember("shard")) {
shard_ = JSON::valueToSize(obj["shard"]);
}
// Check for a platform restriction.
platform_.clear();
if (obj.HasMember("platform") && obj["platform"].IsString()) {
platform_ = obj["platform"].GetString();
}
// Check for a version restriction.
version_.clear();
if (obj.HasMember("version") && obj["version"].IsString()) {
version_ = obj["version"].GetString();
}
// Apply the shard, platform, and version checking.
// It is important to set each value such that the packs meta-table can report
// each of the restrictions.
if ((shard_ > 0 && shard_ < getMachineShard()) || !checkPlatform() ||
!checkVersion()) {
return;
}
discovery_queries_.clear();
if (obj.HasMember("discovery") && obj["discovery"].IsArray()) {
for (const auto& item : obj["discovery"].GetArray()) {
discovery_queries_.push_back(item.GetString());
}
}
// Initialize a discovery cache at time 0.
discovery_cache_ = std::make_pair<size_t, bool>(0, false);
valid_ = true;
// If the splay percent is less than 1 reset to a sane estimate.
if (FLAGS_schedule_splay_percent <= 1) {
FLAGS_schedule_splay_percent = 10;
}
schedule_.clear();
if (!obj.HasMember("queries") || !obj["queries"].IsObject()) {
// This pack contained no queries.
return;
}
// Iterate the queries (or schedule) and check platform/version/sanity.
for (const auto& q : obj["queries"].GetObject()) {
if (q.value.HasMember("shard")) {
auto shard = JSON::valueToSize(q.value["shard"]);
if (shard > 0 && shard < getMachineShard()) {
continue;
}
}
if (q.value.HasMember("platform") && q.value["platform"].IsString()) {
if (!checkPlatform(q.value["platform"].GetString())) {
continue;
}
}
if (q.value.HasMember("version") && q.value["version"].IsString()) {
if (!checkVersion(q.value["version"].GetString())) {
continue;
}
}
if (!q.value.HasMember("query") || !q.value["query"].IsString()) {
continue;
}
ScheduledQuery query;
query.query = q.value["query"].GetString();
if (!q.value.HasMember("interval")) {
query.interval = FLAGS_schedule_default_interval;
} else {
query.interval = JSON::valueToSize(q.value["interval"]);
}
if (query.interval <= 0 || query.query.empty() ||
query.interval > kMaxQueryInterval) {
// Invalid pack query.
LOG(WARNING) << "Query has invalid interval: " << q.name.GetString()
<< ": " << query.interval;
continue;
}
query.splayed_interval =
restoreSplayedValue(q.name.GetString(), query.interval);
if (!q.value.HasMember("snapshot")) {
query.options["snapshot"] = false;
} else {
query.options["snapshot"] = JSON::valueToBool(q.value["snapshot"]);
}
if (!q.value.HasMember("removed")) {
query.options["removed"] = true;
} else {
query.options["removed"] = JSON::valueToBool(q.value["removed"]);
}
query.options["blacklist"] = (q.value.HasMember("blacklist"))
? q.value["blacklist"].GetBool()
: true;
schedule_.emplace(std::make_pair(q.name.GetString(), std::move(query)));
}
}
