void Pack::initialize(const std::string& name,
const std::string& source,
const pt::ptree& tree) {
name_ = name;
source_ = source;
discovery_queries_.clear();
if (tree.count("discovery") > 0) {
for (const auto& item : tree.get_child("discovery")) {
discovery_queries_.push_back(item.second.get_value<std::string>());
}
}
discovery_cache_ = std::make_pair<int, bool>(0, false);
stats_ = {0, 0, 0};
platform_.clear();
if (tree.count("platform") > 0) {
platform_ = tree.get<std::string>("platform");
}
version_.clear();
if (tree.count("version") > 0) {
version_ = tree.get<std::string>("version");
}
schedule_.clear();
if (tree.count("queries") > 0) {
for (const auto& q : tree.get_child("queries")) {
if (q.second.count("platform")) {
if (!checkPlatform(q.second.get<std::string>("platform"))) {
continue;
}
}
if (q.second.count("version")) {
if (!checkVersion(q.second.get<std::string>("version"))) {
continue;
}
}
ScheduledQuery query;
query.interval =
q.second.get<int>("interval", FLAGS_schedule_default_interval);
query.splayed_interval =
splayValue(query.interval, FLAGS_schedule_splay_percent);
query.query = q.second.get<std::string>("query");
query.options["snapshot"] = q.second.get<bool>("snapshot", false);
query.options["removed"] = q.second.get<bool>("removed", true);
schedule_[q.first] = query;
}
}
}
QueryData parseALFExceptionsTree(const pt::ptree& tree) {
QueryData results;
if (tree.count("exceptions") == 0) {
return {};
}
auto exceptions_tree = tree.get_child("exceptions");
for (const auto& it : exceptions_tree) {
Row r;
r["path"] = it.second.get("path", "");
r["state"] = INTEGER(it.second.get("state", -1));
results.push_back(r);
}
auto applications_tree = tree.get_child("applications");
for (const auto& it : applications_tree) {
Row r;
if (it.second.get("alias", "").length() > 0) {
std::string path;
auto alias_data = it.second.get<std::string>("alias", "");
if (pathFromPlistAliasData(alias_data, path).ok()) {
r["path"] = path;
r["state"] = INTEGER(it.second.get("state", -1));
results.push_back(r);
}
}
}
return results;
}
void Config::applyParsers(const std::string& source,
const pt::ptree& tree,
bool pack) {
// Iterate each parser.
for (const auto& plugin : Registry::all("config_parser")) {
std::shared_ptr<ConfigParserPlugin> parser = nullptr;
try {
parser = std::dynamic_pointer_cast<ConfigParserPlugin>(plugin.second);
} catch (const std::bad_cast& /* e */) {
LOG(ERROR) << "Error casting config parser plugin: " << plugin.first;
}
if (parser == nullptr || parser.get() == nullptr) {
continue;
}
// For each key requested by the parser, add a property tree reference.
std::map<std::string, pt::ptree> parser_config;
for (const auto& key : parser->keys()) {
if (tree.count(key) > 0) {
parser_config[key] = tree.get_child(key);
} else {
parser_config[key] = pt::ptree();
}
}
// The config parser plugin will receive a copy of each property tree for
// each top-level-config key. The parser may choose to update the config's
// internal state
parser->update(source, parser_config);
}
}
Status getConfig(boost::property_tree::ptree& output) {
// Make request to endpoint with secrets.
auto r = Request<HTTPTransport, JSONSerializer>(FLAGS_config_enrollment_uri);
boost::property_tree::ptree params;
PluginResponse response;
Registry::call("enrollment", "get_key", {{"enroll", "0"}}, response);
params.put<std::string>("enrollment_key", response[0]["key"]);
params.put<std::string>("app_id", FLAGS_enrollment_app_id);
auto status = r.call(params);
if (!status.ok()) {
return status;
}
// The call succeeded, store the enrolled key.
status = r.getResponse(output);
if (!status.ok()) {
return status;
}
// Receive config or key rejection
if (output.count("enrollment_invalid") > 0 &&
output.get<std::string>("enrollment_invalid", "") == "1") {
return status;
}
return Status(0, "OK");
}
void genMatches(const pt::ptree& entry, std::vector<Row>& results) {
if (entry.count("Matches") == 0) {
return;
}
bool optional = (entry.get("MatchType", "") == "MatchAny");
for (const auto& match : entry.get_child("Matches")) {
if (match.second.count("Matches") > 0) {
genMatches(match.second, results);
continue;
}
Row r;
r["optional"] = (optional) ? "1" : "0";
r["identity"] = match.second.get("Identity", "");
if (match.second.count("MatchFile") == 0) {
// There is no file in this match entry, odd.
continue;
}
// This can contain any of Foundation/Classes/NSURL_Class keys.
auto fileinfo = match.second.get_child("MatchFile");
if (fileinfo.count("LSDownloadContentTypeKey") > 0) {
r["filetype"] = fileinfo.get<std::string>("LSDownloadContentTypeKey");
} else {
r["filetype"] = fileinfo.get("NSURLTypeIdentifierKey", "");
}
r["uses_pattern"] = (match.second.count("Pattern") > 0) ? "1" : "0";
r["filename"] = fileinfo.get("NSURLNameKey", "");
results.push_back(r);
}
}
Status deserializeDiffResults(const pt::ptree& tree, DiffResults& dr) {
if (tree.count("added") > 0) {
auto status = deserializeQueryData(tree.get_child("added"), dr.added);
if (!status.ok()) {
return status;
}
}
if (tree.count("removed") > 0) {
auto status = deserializeQueryData(tree.get_child("removed"), dr.removed);
if (!status.ok()) {
return status;
}
}
return Status(0, "OK");
}
Status deserializeQueryLogItem(const pt::ptree& tree, QueryLogItem& item) {
if (tree.count("diffResults") > 0) {
auto status =
deserializeDiffResults(tree.get_child("diffResults"), item.results);
if (!status.ok()) {
return status;
}
} else if (tree.count("snapshot") > 0) {
auto status =
deserializeQueryData(tree.get_child("snapshot"), item.snapshot_results);
if (!status.ok()) {
return status;
}
}
getLegacyFieldsAndDecorations(tree, item);
return Status(0, "OK");
}
Option::Option(const boost::property_tree::ptree& tree)
: m_name("")
, m_value("")
, m_description("")
{
m_name = tree.get<std::string>("Name");
m_value = tree.get<std::string>("Value");
m_description = tree.count("Description") ? tree.get<std::string>("Description") : "";
return;
}
Status deserializeQueryLogItem(const pt::ptree& tree, QueryLogItem& item) {
if (tree.count("diffResults") > 0) {
auto status =
deserializeDiffResults(tree.get_child("diffResults"), item.results);
if (!status.ok()) {
return status;
}
} else if (tree.count("snapshot") > 0) {
auto status =
deserializeQueryData(tree.get_child("snapshot"), item.snapshot_results);
if (!status.ok()) {
return status;
}
}
item.name = tree.get<std::string>("name", "");
item.identifier = tree.get<std::string>("hostIdentifier", "");
item.calendar_time = tree.get<std::string>("calendarTime", "");
item.time = tree.get<int>("unixTime", 0);
return Status(0, "OK");
}
void DecoratorsConfigParserPlugin::updateDecorations(
const std::string& source, const pt::ptree& decorators) {
WriteLock lock(DecoratorsConfigParserPlugin::kDecorationsMutex);
// Assign load decorators.
auto& load_key = kDecorationPointKeys.at(DECORATE_LOAD);
if (decorators.count(load_key) > 0) {
for (const auto& item : decorators.get_child(load_key)) {
load_[source].push_back(item.second.data());
}
}
// Assign always decorators.
auto& always_key = kDecorationPointKeys.at(DECORATE_ALWAYS);
if (decorators.count(always_key) > 0) {
for (const auto& item : decorators.get_child(always_key)) {
always_[source].push_back(item.second.data());
}
}
// Check if intervals are defined.
auto& interval_key = kDecorationPointKeys.at(DECORATE_INTERVAL);
if (decorators.count(interval_key) > 0) {
auto& interval = decorators.get_child(interval_key);
for (const auto& item : interval) {
size_t rate = std::stoll(item.first);
if (rate % 60 != 0) {
LOG(WARNING) << "Invalid decorator interval rate " << rate
<< " in config source: " << source;
continue;
}
// This is a valid interval, update the set of intervals to include
// this value. When intervals are checked this set is scanned, if a
// match is found, then the associated config data is executed.
for (const auto& interval_query : item.second) {
intervals_[source][rate].push_back(interval_query.second.data());
}
}
}
}
inline void getLegacyFieldsAndDecorations(const pt::ptree& tree,
QueryLogItem& item) {
if (tree.count("decorations") > 0) {
auto& decorations = tree.get_child("decorations");
for (const auto& name : decorations) {
item.decorations[name.first] = name.second.data();
}
}
item.name = tree.get<std::string>("name", "");
item.identifier = tree.get<std::string>("hostIdentifier", "");
item.calendar_time = tree.get<std::string>("calendarTime", "");
item.time = tree.get<int>("unixTime", 0);
}
QueryData parseALFExplicitAuthsTree(const pt::ptree& tree) {
QueryData results;
if (tree.count("explicitauths") == 0) {
return {};
}
auto auths_tree = tree.get_child("explicitauths");
for (const auto& it : auths_tree) {
Row r;
r["process"] = it.second.get("id", "");
results.push_back(r);
}
return results;
}
Status parsePack(const std::string& name, const pt::ptree& data) {
if (data.count("queries") == 0) {
return Status(0, "Pack contains no queries");
}
// Check the pack-global minimum SDK version and platform.
auto version = data.get("version", "");
if (version.size() > 0 && !versionChecker(version, kSDKVersion)) {
return Status(0, "Minimum SDK version not met");
}
auto platform = data.get("platform", "");
if (platform.size() > 0 && !platformChecker(platform, kSDKPlatform)) {
return Status(0, "Platform version mismatch");
}
// For each query in the pack's queries, check their version/platform.
for (const auto& query : data.get_child("queries")) {
auto query_string = query.second.get("query", "");
if (Config::checkScheduledQuery(query_string)) {
VLOG(1) << "Query pack " << name
<< " contains a duplicated query: " << query.first;
continue;
}
// Check the specific query's required version.
version = query.second.get("version", "");
if (version.size() > 0 && !versionChecker(version, kSDKVersion)) {
continue;
}
// Check the specific query's required platform.
platform = query.second.get("platform", "");
if (platform.size() > 0 && !platformChecker(platform, kSDKPlatform)) {
continue;
}
// Hope there is a supplied/non-0 query interval to apply this query pack
// query to the osquery schedule.
auto query_interval = query.second.get("interval", 0);
if (query_interval > 0) {
auto query_name = "pack_" + name + "_" + query.first;
Config::addScheduledQuery(query_name, query_string, query_interval);
}
}
return Status(0, "OK");
}
QueryData parseALFServicesTree(const pt::ptree& tree) {
QueryData results;
if (tree.count("firewall") == 0) {
return {};
}
auto& firewall_tree = tree.get_child("firewall");
for (const auto& it : firewall_tree) {
Row r;
r["service"] = it.first;
r["process"] = it.second.get("proc", "");
r["state"] = INTEGER(it.second.get("state", -1));
results.push_back(r);
}
return results;
}
std::map<std::string, T> read_dictionary_section(const boost::property_tree::ptree& data,
std::string key) {
std::map<std::string, T> ret;
// no section found
if (data.count(key) == 0) {
return ret;
}
const boost::property_tree::ptree& section = data.get_child(key);
// loop through the children of the column_names section
for(auto val: section) {
ret.insert(std::make_pair(val.first,
val.second.get_value<T>()));
}
return ret;
}
Option::Option(const boost::property_tree::ptree& tree)
: m_name("")
, m_value("")
, m_description("")
{
using namespace boost::property_tree;
m_name = tree.get<std::string>("Name");
m_value = tree.get<std::string>("Value");
m_description = tree.count("Description") ? tree.get<std::string>("Description") : "";
boost::property_tree::ptree opts;
ptree const& options = tree.get_child("Options", opts);
if (options.size())
m_options = options::OptionsPtr(new Options(options));
return;
}
static std::string single_child(const boost::property_tree::ptree& tree, const std::string& key, uint32_t eid)
{
switch (tree.count(key)) {
case 0:
{
std::ostringstream o;
o << "Key '" << key << "' not found for EID " << eid;
throw GenericException(o.str());
}
case 1:
break;
default:
{
std::ostringstream o;
o << "Duplicate key '" << key << "' for EID" << eid;
throw GenericException(o.str());
}
}
return tree.find(key)->second.get_value<std::string>();
}
void
mergePropertyTrees (boost::property_tree::ptree &ptMerged,
const boost::property_tree::ptree &ptSecond, int level )
{
// Value or object or array
if (level > 0 && ptSecond.empty() ) {
// Copy value
ptMerged = ptSecond;
} else if (level > 0 && ptSecond.count (std::string() ) == ptSecond.size() ) {
// Copy array
ptMerged = ptSecond;
} else {
auto it = ptSecond.begin();
for (; it != ptSecond.end(); ++it) {
boost::property_tree::ptree child = ptMerged.get_child (it->first.data(),
boost::property_tree::ptree() );
mergePropertyTrees (child, it->second, level + 1);
ptMerged.erase (it->first.data() );
ptMerged.add_child (it->first.data(), child);
}
}
}
harp::psf_gauss_sim::psf_gauss_sim ( boost::property_tree::ptree const & props ) : psf ( "gauss_sim", props ) {
// check to see if we are getting the wavelength solution from a spec
if ( props.count ( psf_gauss_sim_key_lambda_spec ) ) {
lambda_spec_props_ = props.get_child ( psf_gauss_sim_key_lambda_spec );
lambda_spec_type_ = props.get < string > ( psf_gauss_sim_key_lambda_spec_type );
plugin_registry & reg = plugin_registry::get();
spec_p child_spec ( reg.create_spec ( lambda_spec_type_, lambda_spec_props_ ) );
nlambda_ = child_spec->n_lambda();
child_spec->lambda ( lambda_ );
first_lambda_ = lambda_[0];
last_lambda_ = lambda_[ nlambda_ - 1 ];
} else {
// in this case, the props must specify the number of lambda points and spacing
last_lambda_ = props.get < double > ( psf_gauss_sim_key_lambda_stop, 9808.0 );
first_lambda_ = props.get < double > ( psf_gauss_sim_key_lambda_start, 7460.0 );
nlambda_ = props.get < size_t > ( psf_gauss_sim_key_lambda_n, 4697 );
lambda_.resize ( nlambda_ );
double incr = ( last_lambda_ - first_lambda_ ) / (double)( nlambda_ - 1 );
for ( size_t i = 0; i < nlambda_; ++i ) {
lambda_[i] = incr * (double)i;
}
}
// fiber spacing
n_bundle_ = props.get < size_t > ( psf_gauss_sim_key_nbundle, 20 );
bundle_size_ = props.get < size_t > ( psf_gauss_sim_key_bundle_size, 25 );
nspec_ = n_bundle_ * bundle_size_;
pix_margin_ = props.get < double > ( psf_gauss_sim_key_margin, 10.0 );
pix_gap_ = props.get < double > ( psf_gauss_sim_key_gap, 7.0 );
pix_bundle_ = 2.0 * pix_margin_ + (double)(bundle_size_ - 1) * pix_gap_ + (double)bundle_size_;
// response fwhm
psf_fwhm_ = props.get < double > ( psf_gauss_sim_key_fwhm, 2.2 );
// response correlation length in pixels
pixcorr_ = props.get < size_t > ( psf_gauss_sim_key_corr, 10 );
cols_ = (size_t)( pix_bundle_ * (double)n_bundle_ + 1.0 );
pix_offset_ = 0.5 * ( (double)cols_ - ( pix_bundle_ * (double)n_bundle_ ) );
rows_ = nlambda_ + 2 * pixcorr_;
// HDU list for writing
hdus_[ psf_gauss_sim_hdu_x ] = 1;
hdus_[ psf_gauss_sim_hdu_y ] = 2;
hdus_[ psf_gauss_sim_hdu_lambda ] = 3;
hdus_[ psf_gauss_sim_hdu_amp ] = 4;
hdus_[ psf_gauss_sim_hdu_maj ] = 5;
hdus_[ psf_gauss_sim_hdu_min ] = 6;
hdus_[ psf_gauss_sim_hdu_ang ] = 7;
// generate response parameters
nglobal_ = nspec_ * nlambda_;
npix_ = rows_ * cols_;
resp_.resize ( nglobal_ );
double xpix;
double ypix;
for ( size_t spec = 0; spec < nspec_; ++spec ) {
for ( size_t lambda = 0; lambda < nlambda_; ++lambda ) {
size_t bin = spec * nlambda_ + lambda;
spec2pix ( spec, lambda, xpix, ypix );
resp_[ bin ].x = xpix;
resp_[ bin ].y = ypix;
resp_[ bin ].amp = 1.0;
resp_[ bin ].lambda = lambda_[ lambda ];
resp_[ bin ].maj = psf_fwhm_ / 2.0;
resp_[ bin ].min = psf_fwhm_ / 2.0;
resp_[ bin ].ang = 0.0;
}
}
}
void Pack::initialize(const std::string& name,
const std::string& source,
const pt::ptree& tree) {
name_ = name;
source_ = source;
discovery_queries_.clear();
if (tree.count("discovery") > 0) {
for (const auto& item : tree.get_child("discovery")) {
discovery_queries_.push_back(item.second.get_value<std::string>());
}
}
discovery_cache_ = std::make_pair<int, bool>(0, false);
stats_ = {0, 0, 0};
platform_.clear();
if (tree.count("platform") > 0) {
platform_ = tree.get<std::string>("platform", "");
}
version_.clear();
if (tree.count("version") > 0) {
version_ = tree.get<std::string>("version", "");
}
schedule_.clear();
if (tree.count("queries") == 0) {
// This pack contained no queries.
return;
}
// If the splay percent is less than 1 reset to a sane estimate.
if (FLAGS_schedule_splay_percent <= 1) {
FLAGS_schedule_splay_percent = 10;
}
// Iterate the queries (or schedule) and check platform/version/sanity.
for (const auto& q : tree.get_child("queries")) {
if (q.second.count("platform")) {
if (!checkPlatform(q.second.get<std::string>("platform", ""))) {
continue;
}
}
if (q.second.count("version")) {
if (!checkVersion(q.second.get<std::string>("version", ""))) {
continue;
}
}
ScheduledQuery query;
query.query = q.second.get<std::string>("query", "");
query.interval = q.second.get("interval", FLAGS_schedule_default_interval);
if (query.interval <= 0 || query.query.empty()) {
// Invalid pack query.
continue;
}
query.splayed_interval = restoreSplayedValue(q.first, query.interval);
query.options["snapshot"] = q.second.get<bool>("snapshot", false);
query.options["removed"] = q.second.get<bool>("removed", true);
schedule_[q.first] = query;
}
}
void Pack::initialize(const std::string& name,
const std::string& source,
const pt::ptree& tree) {
name_ = name;
source_ = source;
// Check the shard limitation, shards falling below this value are included.
if (tree.count("shard") > 0) {
shard_ = tree.get<size_t>("shard", 0);
}
// Check for a platform restriction.
platform_.clear();
if (tree.count("platform") > 0) {
platform_ = tree.get<std::string>("platform", "");
}
// Check for a version restriction.
version_.clear();
if (tree.count("version") > 0) {
version_ = tree.get<std::string>("version", "");
}
// Apply the shard, platform, and version checking.
// It is important to set each value such that the packs meta-table can report
// each of the restrictions.
if ((shard_ > 0 && shard_ < getMachineShard()) || !checkPlatform() ||
!checkVersion()) {
return;
}
discovery_queries_.clear();
if (tree.count("discovery") > 0) {
for (const auto& item : tree.get_child("discovery")) {
discovery_queries_.push_back(item.second.get_value<std::string>());
}
}
// Initialize a discovery cache at time 0.
discovery_cache_ = std::make_pair<size_t, bool>(0, false);
valid_ = true;
// If the splay percent is less than 1 reset to a sane estimate.
if (FLAGS_schedule_splay_percent <= 1) {
FLAGS_schedule_splay_percent = 10;
}
schedule_.clear();
if (tree.count("queries") == 0) {
// This pack contained no queries.
return;
}
// Iterate the queries (or schedule) and check platform/version/sanity.
for (const auto& q : tree.get_child("queries")) {
if (q.second.count("shard") > 0) {
auto shard = q.second.get<size_t>("shard", 0);
if (shard > 0 && shard < getMachineShard()) {
continue;
}
}
if (q.second.count("platform")) {
if (!checkPlatform(q.second.get<std::string>("platform", ""))) {
continue;
}
}
if (q.second.count("version")) {
if (!checkVersion(q.second.get<std::string>("version", ""))) {
continue;
}
}
ScheduledQuery query;
query.query = q.second.get<std::string>("query", "");
query.interval = q.second.get("interval", FLAGS_schedule_default_interval);
if (query.interval <= 0 || query.query.empty() || query.interval > 592200) {
// Invalid pack query.
VLOG(1) << "Query has invalid interval: " << q.first << ": "
<< query.interval;
continue;
}
query.splayed_interval = restoreSplayedValue(q.first, query.interval);
query.options["snapshot"] = q.second.get<bool>("snapshot", false);
query.options["removed"] = q.second.get<bool>("removed", true);
schedule_[q.first] = query;
}
}