void EventSubscriberPlugin::expireRecords(const std::string& list_type,
const std::string& index,
bool all) {
auto record_key = "records." + dbNamespace();
auto data_key = "data." + dbNamespace();
// If the expirations is not removing all records, rewrite the persisting.
std::vector<std::string> persisting_records;
// Request all records within this list-size + bin offset.
auto expired_records = getRecords({list_type + "." + index});
for (const auto& record : expired_records) {
if (all || record.second <= expire_time_) {
deleteDatabaseValue(kEvents, data_key + "." + record.first);
} else {
persisting_records.push_back(record.first + ":" +
std::to_string(record.second));
}
}
// Either drop or overwrite the record list.
if (all) {
deleteDatabaseValue(kEvents, record_key + "." + list_type + "." + index);
} else if (persisting_records.size() < expired_records.size()) {
auto new_records = boost::algorithm::join(persisting_records, ",");
setDatabaseValue(
kEvents, record_key + "." + list_type + "." + index, new_records);
}
}
void TLSLogForwarderRunner::check() {
// Instead of using the 'help' database API, prefer to interact with the
// DBHandle directly for additional performance.
auto handle = DBHandle::getInstance();
// Get a list of all the buffered log items, with a max of 1024 lines.
std::vector<std::string> indexes;
auto status = handle->Scan(kLogs, indexes, kTLSMaxLogLines);
// For each index, accumulate the log line into the result or status set.
std::vector<std::string> results, statuses;
iterate(indexes,
([&handle, &results, &statuses](std::string& index) {
std::string value;
auto& target = ((index.at(0) == 'r') ? results : statuses);
if (handle->Get(kLogs, index, value)) {
// Enforce a max log line size for TLS logging.
if (value.size() > FLAGS_logger_tls_max) {
LOG(WARNING) << "Line exceeds TLS logger max: " << value.size();
} else {
target.push_back(std::move(value));
}
}
}));
// If any results/statuses were found in the flushed buffer, send.
if (results.size() > 0) {
status = send(results, "result");
if (!status.ok()) {
VLOG(1) << "Could not send results to logger URI: " << uri_ << " ("
<< status.getMessage() << ")";
} else {
// Clear the results logs once they were sent.
iterate(indexes,
([&results](std::string& index) {
if (index.at(0) != 'r') {
return;
}
deleteDatabaseValue(kLogs, index);
}));
}
}
if (statuses.size() > 0) {
status = send(statuses, "status");
if (!status.ok()) {
VLOG(1) << "Could not send status logs to logger URI: " << uri_ << " ("
<< status.getMessage() << ")";
} else {
// Clear the status logs once they were sent.
iterate(indexes,
([&results](std::string& index) {
if (index.at(0) != 's') {
return;
}
deleteDatabaseValue(kLogs, index);
}));
}
}
}
void Config::purge() {
// The first use of purge is removing expired query results.
std::vector<std::string> saved_queries;
scanDatabaseKeys(kQueries, saved_queries);
const auto& schedule = this->schedule_;
auto queryExists = [&schedule](const std::string& query_name) {
for (const auto& pack : schedule->packs_) {
const auto& pack_queries = pack->getSchedule();
if (pack_queries.count(query_name)) {
return true;
}
}
return false;
};
RecursiveLock lock(config_schedule_mutex_);
// Iterate over each result set in the database.
for (const auto& saved_query : saved_queries) {
if (queryExists(saved_query)) {
continue;
}
std::string content;
getDatabaseValue(kPersistentSettings, "timestamp." + saved_query, content);
if (content.empty()) {
// No timestamp is set for this query, perhaps this is the first time
// query results expiration is applied.
setDatabaseValue(kPersistentSettings,
"timestamp." + saved_query,
std::to_string(getUnixTime()));
continue;
}
// Parse the timestamp and compare.
size_t last_executed = 0;
try {
last_executed = boost::lexical_cast<size_t>(content);
} catch (const boost::bad_lexical_cast& /* e */) {
// Erase the timestamp as is it potentially corrupt.
deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query);
continue;
}
if (last_executed < getUnixTime() - 592200) {
// Query has not run in the last week, expire results and interval.
deleteDatabaseValue(kQueries, saved_query);
deleteDatabaseValue(kPersistentSettings, "interval." + saved_query);
deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query);
VLOG(1) << "Expiring results for scheduled query: " << saved_query;
}
}
}
/// Remove these ATC tables from the registry and database
Status ATCConfigParserPlugin::removeATCTables(
const std::set<std::string>& detach_tables) {
auto registry_table = RegistryFactory::get().registry("table");
std::set<std::string> failed_tables;
for (const auto& table : detach_tables) {
if (registry_table->exists(table)) {
std::string value;
if (getDatabaseValue(
kPersistentSettings, kDatabaseKeyPrefix + table, value)
.ok()) {
registry_table->remove(table);
PluginResponse resp;
Registry::call(
"sql", "sql", {{"action", "detatch"}, {"table", table}}, resp);
LOG(INFO) << "Removed ATC table: " << table;
} else {
failed_tables.insert(table);
}
}
deleteDatabaseValue(kPersistentSettings, kDatabaseKeyPrefix + table);
}
if (failed_tables.empty()) {
return Status();
}
return Status(
1, "Attempted to remove non ATC tables: " + join(failed_tables, ", "));
}
TEST_F(TLSLoggerTests, test_database) {
// Start a server.
TLSServerRunner::start();
TLSServerRunner::setClientConfig();
auto forwarder = std::make_shared<TLSLogForwarder>();
std::string expected = "{\"new_json\": true}";
forwarder->logString(expected);
StatusLogLine status;
status.message = "{\"status\": \"bar\"}";
forwarder->logStatus({status});
// Stop the server.
TLSServerRunner::unsetClientConfig();
TLSServerRunner::stop();
std::vector<std::string> indexes;
scanDatabaseKeys(kLogs, indexes);
EXPECT_EQ(2U, indexes.size());
// Iterate using an unordered search, and search for the expected string
// that was just logged.
bool found_string = false;
for (const auto& index : indexes) {
std::string value;
getDatabaseValue(kLogs, index, value);
found_string = (found_string || value == expected);
deleteDatabaseValue(kLogs, index);
}
EXPECT_TRUE(found_string);
}
static void DATABASE_store(benchmark::State& state) {
while (state.KeepRunning()) {
setDatabaseValue(kPersistentSettings, "benchmark", "1");
}
// All benchmarks will share a single database handle.
deleteDatabaseValue(kPersistentSettings, "benchmark");
}
void EventSubscriberPlugin::expireCheck(bool cleanup) {
auto data_key = "data." + dbNamespace();
auto eid_key = "eid." + dbNamespace();
// Min key will be the last surviving key.
size_t min_key = 0;
{
auto limit = getEventsMax();
std::vector<std::string> keys;
scanDatabaseKeys(kEvents, keys, data_key);
if (keys.size() <= limit) {
return;
}
// There is an overflow of events buffered for this subscriber.
LOG(WARNING) << "Expiring events for subscriber: " << getName()
<< " (limit " << limit << ")";
VLOG(1) << "Subscriber events " << getName() << " exceeded limit " << limit
<< " by: " << keys.size() - limit;
// Inspect the N-FLAGS_events_max -th event's value and expire before the
// time within the content.
std::string last_key;
getDatabaseValue(kEvents, eid_key, last_key);
// The EID is the next-index.
// EID - events_max is the most last-recent event to keep.
min_key = boost::lexical_cast<size_t>(last_key) - getEventsMax();
if (cleanup) {
// Scan each of the keys in keys, if their ID portion is < min_key.
// Nix them, this requires lots of conversions, use with care.
for (const auto& key : keys) {
if (std::stoul(key.substr(key.rfind('.') + 1)) < min_key) {
deleteDatabaseValue(kEvents, key);
}
}
}
}
// Convert the key index into a time using the content.
// The last-recent event is fetched and the corresponding time is used as
// the expiration time for the subscriber.
std::string content;
getDatabaseValue(kEvents, data_key + "." + std::to_string(min_key), content);
// Decode the value into a row structure to extract the time.
Row r;
if (!deserializeRowJSON(content, r) || r.count("time") == 0) {
return;
}
// The last time will become the implicit expiration time.
size_t last_time = boost::lexical_cast<size_t>(r.at("time"));
if (last_time > 0) {
expire_time_ = last_time;
}
// Finally, attempt an index query to trigger expirations.
// In this case the result set is not used.
getIndexes(expire_time_, 0);
}
Status BufferedLogForwarder::deleteValueWithCount(const std::string& domain,
const std::string& key) {
Status status = deleteDatabaseValue(domain, key);
if (status.ok()) {
buffer_count_--;
}
return status;
}
Status ViewsConfigParserPlugin::update(const std::string& source,
const ParserConfig& config) {
auto cv = config.find("views");
if (cv == config.end()) {
return Status(1);
}
auto obj = data_.getObject();
data_.copyFrom(cv->second.doc(), obj);
data_.add("views", obj);
const auto& views = data_.doc()["views"];
// We use a restricted scope below to change the data structure from
// an array to a set. This lets us do deletes much more efficiently
std::vector<std::string> created_views;
std::set<std::string> erase_views;
{
std::vector<std::string> old_views_vec;
scanDatabaseKeys(kQueries, old_views_vec, kConfigViews);
for (const auto& view : old_views_vec) {
erase_views.insert(view.substr(kConfigViews.size()));
}
}
QueryData r;
for (const auto& view : views.GetObject()) {
std::string name = view.name.GetString();
std::string query = view.value.GetString();
if (query.empty()) {
continue;
}
std::string old_query = "";
getDatabaseValue(kQueries, kConfigViews + name, old_query);
erase_views.erase(name);
if (old_query == query) {
continue;
}
// View has been updated
osquery::query("DROP VIEW " + name, r);
auto s = osquery::query("CREATE VIEW " + name + " AS " + query, r);
if (s.ok()) {
setDatabaseValue(kQueries, kConfigViews + name, query);
} else {
LOG(INFO) << "Error creating view (" << name << "): " << s.getMessage();
}
}
// Any views left are views that don't exist in the new configuration file
// so we tear them down and remove them from the database.
for (const auto& old_view : erase_views) {
osquery::query("DROP VIEW " + old_view, r);
deleteDatabaseValue(kQueries, kConfigViews + old_view);
}
return Status(0, "OK");
}
static void DATABASE_store_append(benchmark::State& state) {
// Serialize the example result set into a string.
std::string content;
auto qd = getExampleQueryData(20, 100);
serializeQueryDataJSON(qd, content);
size_t k = 0;
while (state.KeepRunning()) {
setDatabaseValue(kPersistentSettings, "key" + std::to_string(k), content);
deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(k));
k++;
}
// All benchmarks will share a single database handle.
for (size_t i = 0; i < k; ++i) {
deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(i));
}
}
Status ATCConfigParserPlugin::setUp() {
VLOG(1) << "Removing stale ATC entries";
std::vector<std::string> keys;
scanDatabaseKeys(kPersistentSettings, keys, kDatabaseKeyPrefix);
for (const auto& key : keys) {
auto s = deleteDatabaseValue(kPersistentSettings, key);
if (!s.ok()) {
LOG(INFO) << "Could not clear ATC key " << key << "from database";
}
}
return Status();
}
static void DATABASE_store_large(benchmark::State& state) {
// Serialize the example result set into a string.
std::string content;
auto qd = getExampleQueryData(20, 100);
serializeQueryDataJSON(qd, content);
while (state.KeepRunning()) {
setDatabaseValue(kPersistentSettings, "benchmark", content);
}
// All benchmarks will share a single database handle.
deleteDatabaseValue(kPersistentSettings, "benchmark");
}
DistributedQueryRequest Distributed::popRequest() {
// Read all pending queries.
std::vector<std::string> queries;
scanDatabaseKeys(kQueries, queries, kDistributedQueryPrefix);
// Set the last-most-recent query as the request, and delete it.
DistributedQueryRequest request;
const auto& next = queries.front();
request.id = next.substr(kDistributedQueryPrefix.size());
getDatabaseValue(kQueries, next, request.query);
deleteDatabaseValue(kQueries, next);
return request;
}
TEST_F(DatabaseTests, test_delete_values) {
setDatabaseValue(kLogs, "k", "0");
std::string value;
getDatabaseValue(kLogs, "k", value);
EXPECT_FALSE(value.empty());
auto s = deleteDatabaseValue(kLogs, "k");
EXPECT_TRUE(s.ok());
// Make sure the key has been deleted.
value.clear();
s = getDatabaseValue(kLogs, "k", value);
EXPECT_FALSE(s.ok());
EXPECT_TRUE(value.empty());
}
Status init() override {
// Before starting our subscription, purge any residual db entries as it's
// unlikely we'll finish re-assmebling them
std::vector<std::string> keys;
scanDatabaseKeys(kEvents, keys, kScriptBlockPrefix);
for (const auto& k : keys) {
auto s = deleteDatabaseValue(kEvents, k);
if (!s.ok()) {
VLOG(1) << "Failed to delete stale script block from the database "
<< k;
}
}
auto wc = createSubscriptionContext();
wc->sources.insert(kPowershellEventsChannel);
subscribe(&PowershellEventSubscriber::Callback, wc);
return Status();
}
Status PowershellEventSubscriber::Callback(const ECRef& ec, const SCRef& sc) {
// For script block logging we only care about events with script blocks
auto eid = ec->eventRecord.get("Event.System.EventID", -1);
if (eid != kScriptBlockLoggingEid) {
return Status();
}
Row results;
for (const auto& node : ec->eventRecord.get_child("Event", pt::ptree())) {
if (node.first == "System" || node.first == "<xmlattr>") {
continue;
}
// #4357: This should make use of RapidJSON
parseTree(node.second, results);
}
FILETIME etime;
GetSystemTimeAsFileTime(&etime);
results["time"] = BIGINT(filetimeToUnixtime(etime));
results["datetime"] =
ec->eventRecord.get("Event.System.TimeCreated.<xmlattr>.SystemTime", "");
// If there's only one script block no reassembly is needed
if (results["MessageTotal"] == "1") {
addScriptResult(results);
return Status();
}
// Add the script content to the DB for later reassembly
auto s = setDatabaseValue(kEvents,
kScriptBlockPrefix + results["ScriptBlockId"] +
"." + results["MessageNumber"],
results["ScriptBlockText"]);
if (!s.ok()) {
LOG(WARNING) << "Failed to add new Powershell block to database for script "
<< results["ScriptBlockId"];
}
// If we expect more blocks bail out early
if (results["MessageNumber"] != results["MessageTotal"]) {
return Status();
}
// Otherwise all script blocks should be accounted for so reconstruct
std::vector<std::string> keys;
s = scanDatabaseKeys(
kEvents, keys, kScriptBlockPrefix + results["ScriptBlockId"]);
if (!s.ok()) {
LOG(WARNING) << "Failed to look up powershell script blocks for "
<< results["ScriptBlockId"];
return Status(1);
}
std::string powershell_script{""};
for (const auto& key : keys) {
std::string val{""};
s = getDatabaseValue(kEvents, key, val);
if (!s.ok()) {
LOG(WARNING) << "Failed to retrieve script block " << key;
continue;
}
powershell_script += val;
s = deleteDatabaseValue(kEvents, key);
if (!s.ok()) {
LOG(WARNING) << "Failed to delete script block key from db " << key;
}
}
results["ScriptBlockText"] = powershell_script;
addScriptResult(results);
return Status();
}
Status ATCConfigParserPlugin::update(const std::string& source,
const ParserConfig& config) {
auto cv = config.find(kParserKey);
if (cv == config.end()) {
return Status(1, "No configuration for ATC (Auto Table Construction)");
}
auto obj = data_.getObject();
data_.copyFrom(cv->second.doc(), obj);
data_.add(kParserKey, obj);
const auto& ac_tables = data_.doc()[kParserKey];
auto tables = RegistryFactory::get().registry("table");
auto registered = registeredATCTables();
for (const auto& ac_table : ac_tables.GetObject()) {
std::string table_name{ac_table.name.GetString()};
auto params = ac_table.value.GetObject();
std::string query{params.HasMember("query") && params["query"].IsString()
? params["query"].GetString()
: ""};
std::string path{params.HasMember("path") && params["path"].IsString()
? params["path"].GetString()
: ""};
std::string platform{params.HasMember("platform") &&
params["platform"].IsString()
? params["platform"].GetString()
: ""};
if (query.empty() || path.empty()) {
LOG(WARNING) << "ATC Table: " << table_name << " is misconfigured";
continue;
}
if (!checkPlatform(platform)) {
VLOG(1) << "Skipping ATC table: " << table_name
<< " because platform doesn't match";
continue;
}
TableColumns columns;
std::string columns_value;
columns_value.reserve(256);
for (const auto& column : params["columns"].GetArray()) {
columns.push_back(make_tuple(
std::string(column.GetString()), TEXT_TYPE, ColumnOptions::DEFAULT));
columns_value += std::string(column.GetString()) + ",";
}
registered.erase(table_name);
std::string table_settings{table_name + query + columns_value + path};
std::string old_setting;
auto s = getDatabaseValue(
kPersistentSettings, kDatabaseKeyPrefix + table_name, old_setting);
// The ATC table hasn't changed so we skip ahead
if (table_settings == old_setting) {
continue;
}
// Remove the old table to replace with the new one
s = removeATCTables({table_name});
if (!s.ok()) {
LOG(WARNING) << "ATC table overrides core table; Refusing registration";
continue;
}
s = setDatabaseValue(
kPersistentSettings, kDatabaseKeyPrefix + table_name, table_settings);
if (!s.ok()) {
LOG(WARNING) << "Could not write to database";
continue;
}
s = tables->add(
table_name, std::make_shared<ATCPlugin>(path, columns, query), true);
if (!s.ok()) {
LOG(WARNING) << s.getMessage();
deleteDatabaseValue(kPersistentSettings, kDatabaseKeyPrefix + table_name);
continue;
}
PluginResponse resp;
Registry::call(
"sql", "sql", {{"action", "attach"}, {"table", table_name}}, resp);
LOG(INFO) << "Registered ATC table: " << table_name;
}
if (registered.size() > 0) {
VLOG(1)
<< "Removing any ATC tables that were removed in this configuration "
"change";
removeATCTables(registered);
}
return Status();
}
void SetUp() {
deleteDatabaseValue(kPersistentSettings, "nodeKey");
deleteDatabaseValue(kPersistentSettings, "nodeKeyTime");
}
