void EventSubscriberPlugin::expireCheck(bool cleanup) {
auto data_key = "data." + dbNamespace();
auto eid_key = "eid." + dbNamespace();
// Min key will be the last surviving key.
size_t min_key = 0;
{
auto limit = getEventsMax();
std::vector<std::string> keys;
scanDatabaseKeys(kEvents, keys, data_key);
if (keys.size() <= limit) {
return;
}
// There is an overflow of events buffered for this subscriber.
LOG(WARNING) << "Expiring events for subscriber: " << getName()
<< " (limit " << limit << ")";
VLOG(1) << "Subscriber events " << getName() << " exceeded limit " << limit
<< " by: " << keys.size() - limit;
// Inspect the N-FLAGS_events_max -th event's value and expire before the
// time within the content.
std::string last_key;
getDatabaseValue(kEvents, eid_key, last_key);
// The EID is the next-index.
// EID - events_max is the most last-recent event to keep.
min_key = boost::lexical_cast<size_t>(last_key) - getEventsMax();
if (cleanup) {
// Scan each of the keys in keys, if their ID portion is < min_key.
// Nix them, this requires lots of conversions, use with care.
for (const auto& key : keys) {
if (std::stoul(key.substr(key.rfind('.') + 1)) < min_key) {
deleteDatabaseValue(kEvents, key);
}
}
}
}
// Convert the key index into a time using the content.
// The last-recent event is fetched and the corresponding time is used as
// the expiration time for the subscriber.
std::string content;
getDatabaseValue(kEvents, data_key + "." + std::to_string(min_key), content);
// Decode the value into a row structure to extract the time.
Row r;
if (!deserializeRowJSON(content, r) || r.count("time") == 0) {
return;
}
// The last time will become the implicit expiration time.
size_t last_time = boost::lexical_cast<size_t>(r.at("time"));
if (last_time > 0) {
expire_time_ = last_time;
}
// Finally, attempt an index query to trigger expirations.
// In this case the result set is not used.
getIndexes(expire_time_, 0);
}
Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime time) {
Status status;
std::string time_value = boost::lexical_cast<std::string>(time);
// The record is identified by the event type then module name.
std::string index_key = "indexes." + dbNamespace();
std::string record_key = "records." + dbNamespace();
// The list key includes the list type (bin size) and the list ID (bin).
std::string list_key;
std::string list_id;
for (const auto& time_list : kEventTimeLists) {
// The list_id is the MOST-Specific key ID, the bin for this list.
// If the event time was 13 and the time_list is 5 seconds, lid = 2.
list_id = boost::lexical_cast<std::string>(time / time_list);
// The list name identifies the 'type' of list.
list_key = boost::lexical_cast<std::string>(time_list);
// list_key = list_key + "." + list_id;
{
WriteLock lock(event_record_lock_);
// Append the record (eid, unix_time) to the list bin.
std::string record_value;
status = getDatabaseValue(
kEvents, record_key + "." + list_key + "." + list_id, record_value);
if (record_value.length() == 0) {
// This is a new list_id for list_key, append the ID to the indirect
// lookup for this list_key.
std::string index_value;
status =
getDatabaseValue(kEvents, index_key + "." + list_key, index_value);
if (index_value.length() == 0) {
// A new index.
index_value = list_id;
} else {
index_value += "," + list_id;
}
status =
setDatabaseValue(kEvents, index_key + "." + list_key, index_value);
record_value = eid + ":" + time_value;
} else {
// Tokenize a record using ',' and the EID/time using ':'.
record_value += "," + eid + ":" + time_value;
}
status = setDatabaseValue(
kEvents, record_key + "." + list_key + "." + list_id, record_value);
if (!status.ok()) {
LOG(ERROR) << "Could not put Event Record key: " << record_key;
}
}
}
return Status(0, "OK");
}
std::vector<EventRecord> EventSubscriberPlugin::getRecords(
const std::set<std::string>& indexes) {
auto record_key = "records." + dbNamespace();
std::vector<EventRecord> records;
for (const auto& index : indexes) {
std::vector<std::string> bin_records;
{
std::string record_value;
getDatabaseValue(kEvents, record_key + "." + index, record_value);
if (record_value.empty()) {
// There are actually no events in this bin, interesting error case.
continue;
}
// Each list is tokenized into a record=event_id:time.
boost::split(bin_records, record_value, boost::is_any_of(",:"));
}
auto bin_it = bin_records.begin();
// Iterate over every 2 items: EID:TIME.
for (; bin_it != bin_records.end(); bin_it++) {
const auto& eid = *bin_it;
EventTime time = timeFromRecord(*(++bin_it));
records.push_back(std::make_pair(eid, time));
}
}
return records;
}
TEST_F(TLSLoggerTests, test_database) {
// Start a server.
TLSServerRunner::start();
TLSServerRunner::setClientConfig();
auto forwarder = std::make_shared<TLSLogForwarder>();
std::string expected = "{\"new_json\": true}";
forwarder->logString(expected);
StatusLogLine status;
status.message = "{\"status\": \"bar\"}";
forwarder->logStatus({status});
// Stop the server.
TLSServerRunner::unsetClientConfig();
TLSServerRunner::stop();
std::vector<std::string> indexes;
scanDatabaseKeys(kLogs, indexes);
EXPECT_EQ(2U, indexes.size());
// Iterate using an unordered search, and search for the expected string
// that was just logged.
bool found_string = false;
for (const auto& index : indexes) {
std::string value;
getDatabaseValue(kLogs, index, value);
found_string = (found_string || value == expected);
deleteDatabaseValue(kLogs, index);
}
EXPECT_TRUE(found_string);
}
size_t restoreSplayedValue(const std::string& name, size_t interval) {
// Attempt to restore a previously-calculated splay.
std::string content;
getDatabaseValue(kPersistentSettings, "interval." + name, content);
if (!content.empty()) {
// This query name existed before, check the last requested interval.
auto details = osquery::split(content, ":");
if (details.size() == 2) {
long last_interval, last_splay;
if (safeStrtol(details[0], 10, last_interval) &&
safeStrtol(details[1], 10, last_splay)) {
if (last_interval == static_cast<long>(interval) && last_splay > 0) {
// This is a matching interval, use the previous splay.
return static_cast<size_t>(last_splay);
}
}
}
}
// If the splayed interval was not restored from the database.
auto splay = splayValue(interval, FLAGS_schedule_splay_percent);
content = std::to_string(interval) + ":" + std::to_string(splay);
setDatabaseValue(kPersistentSettings, "interval." + name, content);
return splay;
}
/// Remove these ATC tables from the registry and database
Status ATCConfigParserPlugin::removeATCTables(
const std::set<std::string>& detach_tables) {
auto registry_table = RegistryFactory::get().registry("table");
std::set<std::string> failed_tables;
for (const auto& table : detach_tables) {
if (registry_table->exists(table)) {
std::string value;
if (getDatabaseValue(
kPersistentSettings, kDatabaseKeyPrefix + table, value)
.ok()) {
registry_table->remove(table);
PluginResponse resp;
Registry::call(
"sql", "sql", {{"action", "detatch"}, {"table", table}}, resp);
LOG(INFO) << "Removed ATC table: " << table;
} else {
failed_tables.insert(table);
}
}
deleteDatabaseValue(kPersistentSettings, kDatabaseKeyPrefix + table);
}
if (failed_tables.empty()) {
return Status();
}
return Status(
1, "Attempted to remove non ATC tables: " + join(failed_tables, ", "));
}
TEST_F(DatabaseTests, test_get_value) {
std::string expected = "{}";
setDatabaseValue(kLogs, "i", expected);
std::string value;
auto s = getDatabaseValue(kLogs, "i", value);
EXPECT_TRUE(s.ok());
EXPECT_EQ(value, expected);
// Unknown keys return failed, but will return empty data.
value.clear();
s = getDatabaseValue(kLogs, "does_not_exist", value);
EXPECT_FALSE(s.ok());
EXPECT_TRUE(value.empty());
}
TEST_F(DatabaseTests, test_delete_values) {
setDatabaseValue(kLogs, "k", "0");
std::string value;
getDatabaseValue(kLogs, "k", value);
EXPECT_FALSE(value.empty());
auto s = deleteDatabaseValue(kLogs, "k");
EXPECT_TRUE(s.ok());
// Make sure the key has been deleted.
value.clear();
s = getDatabaseValue(kLogs, "k", value);
EXPECT_FALSE(s.ok());
EXPECT_TRUE(value.empty());
}
EventID EventSubscriberPlugin::getEventID() {
Status status;
// First get an event ID from the meta key.
std::string eid_key = "eid." + dbNamespace();
std::string last_eid_value;
std::string eid_value;
{
WriteLock lock(event_id_lock_);
status = getDatabaseValue(kEvents, eid_key, last_eid_value);
if (!status.ok() || last_eid_value.empty()) {
last_eid_value = "0";
}
last_eid_ = boost::lexical_cast<size_t>(last_eid_value) + 1;
eid_value = boost::lexical_cast<std::string>(last_eid_);
status = setDatabaseValue(kEvents, eid_key, eid_value);
}
if (!status.ok()) {
return "0";
}
return eid_value;
}
Status ViewsConfigParserPlugin::update(const std::string& source,
const ParserConfig& config) {
auto cv = config.find("views");
if (cv == config.end()) {
return Status(1);
}
auto obj = data_.getObject();
data_.copyFrom(cv->second.doc(), obj);
data_.add("views", obj);
const auto& views = data_.doc()["views"];
// We use a restricted scope below to change the data structure from
// an array to a set. This lets us do deletes much more efficiently
std::vector<std::string> created_views;
std::set<std::string> erase_views;
{
std::vector<std::string> old_views_vec;
scanDatabaseKeys(kQueries, old_views_vec, kConfigViews);
for (const auto& view : old_views_vec) {
erase_views.insert(view.substr(kConfigViews.size()));
}
}
QueryData r;
for (const auto& view : views.GetObject()) {
std::string name = view.name.GetString();
std::string query = view.value.GetString();
if (query.empty()) {
continue;
}
std::string old_query = "";
getDatabaseValue(kQueries, kConfigViews + name, old_query);
erase_views.erase(name);
if (old_query == query) {
continue;
}
// View has been updated
osquery::query("DROP VIEW " + name, r);
auto s = osquery::query("CREATE VIEW " + name + " AS " + query, r);
if (s.ok()) {
setDatabaseValue(kQueries, kConfigViews + name, query);
} else {
LOG(INFO) << "Error creating view (" << name << "): " << s.getMessage();
}
}
// Any views left are views that don't exist in the new configuration file
// so we tear them down and remove them from the database.
for (const auto& old_view : erase_views) {
osquery::query("DROP VIEW " + old_view, r);
deleteDatabaseValue(kQueries, kConfigViews + old_view);
}
return Status(0, "OK");
}
static void DATABASE_get(benchmark::State& state) {
setDatabaseValue(kPersistentSettings, "benchmark", "1");
while (state.KeepRunning()) {
std::string value;
getDatabaseValue(kPersistentSettings, "benchmark", value);
}
// All benchmarks will share a single database handle.
deleteDatabaseValue(kPersistentSettings, "benchmark");
}
QueryData TablePlugin::getCache() const {
VLOG(1) << "Retrieving results from cache for table: " << getName();
// Lookup results from database and deserialize.
std::string content;
getDatabaseValue(kQueries, "cache." + getName(), content);
QueryData results;
deserializeQueryDataJSON(content, results);
return results;
}
uint64_t Query::getPreviousEpoch() const {
uint64_t epoch = 0;
std::string raw;
auto status = getDatabaseValue(kQueries, name_ + "epoch", raw);
if (status.ok()) {
epoch = std::stoull(raw);
}
return epoch;
}
Status getHostUUID(std::string& ident) {
// Lookup the host identifier (UUID) previously generated and stored.
auto status = getDatabaseValue(kPersistentSettings, "hostIdentifier", ident);
if (ident.size() == 0) {
// There was no UUID stored in the database, generate one and store it.
ident = osquery::generateHostUUID();
VLOG(1) << "Using UUID " << ident << " as host identifier";
return setDatabaseValue(kPersistentSettings, "hostIdentifier", ident);
}
return status;
}
void DatabasePluginTests::testReset() {
RegistryFactory::get().setActive("database", getName());
setDatabaseValue(kLogs, "reset", "1");
resetDatabase();
if ("ephemeral" != getName()) {
// The ephemeral plugin is special and does not persist after reset.
std::string value;
EXPECT_TRUE(getDatabaseValue(kLogs, "reset", value));
EXPECT_EQ(value, "1");
}
}
void TLSLogForwarderRunner::check() {
// Get a list of all the buffered log items, with a max of 1024 lines.
std::vector<std::string> indexes;
auto status = scanDatabaseKeys(kLogs, indexes, kTLSMaxLogLines);
// For each index, accumulate the log line into the result or status set.
std::vector<std::string> results, statuses;
iterate(indexes, ([&results, &statuses](std::string& index) {
std::string value;
auto& target = ((index.at(0) == 'r') ? results : statuses);
if (getDatabaseValue(kLogs, index, value)) {
// Enforce a max log line size for TLS logging.
if (value.size() > FLAGS_logger_tls_max) {
LOG(WARNING) << "Line exceeds TLS logger max: " << value.size();
} else {
target.push_back(std::move(value));
}
}
}));
// If any results/statuses were found in the flushed buffer, send.
if (results.size() > 0) {
status = send(results, "result");
if (!status.ok()) {
VLOG(1) << "Could not send results to logger URI: " << uri_ << " ("
<< status.getMessage() << ")";
} else {
// Clear the results logs once they were sent.
iterate(indexes, ([&results](std::string& index) {
if (index.at(0) != 'r') {
return;
}
deleteDatabaseValue(kLogs, index);
}));
}
}
if (statuses.size() > 0) {
status = send(statuses, "status");
if (!status.ok()) {
VLOG(1) << "Could not send status logs to logger URI: " << uri_ << " ("
<< status.getMessage() << ")";
} else {
// Clear the status logs once they were sent.
iterate(indexes, ([&results](std::string& index) {
if (index.at(0) != 's') {
return;
}
deleteDatabaseValue(kLogs, index);
}));
}
}
}
void Config::purge() {
// The first use of purge is removing expired query results.
std::vector<std::string> saved_queries;
scanDatabaseKeys(kQueries, saved_queries);
const auto& schedule = this->schedule_;
auto queryExists = [&schedule](const std::string& query_name) {
for (const auto& pack : schedule->packs_) {
const auto& pack_queries = pack->getSchedule();
if (pack_queries.count(query_name)) {
return true;
}
}
return false;
};
RecursiveLock lock(config_schedule_mutex_);
// Iterate over each result set in the database.
for (const auto& saved_query : saved_queries) {
if (queryExists(saved_query)) {
continue;
}
std::string content;
getDatabaseValue(kPersistentSettings, "timestamp." + saved_query, content);
if (content.empty()) {
// No timestamp is set for this query, perhaps this is the first time
// query results expiration is applied.
setDatabaseValue(kPersistentSettings,
"timestamp." + saved_query,
std::to_string(getUnixTime()));
continue;
}
// Parse the timestamp and compare.
size_t last_executed = 0;
try {
last_executed = boost::lexical_cast<size_t>(content);
} catch (const boost::bad_lexical_cast& /* e */) {
// Erase the timestamp as is it potentially corrupt.
deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query);
continue;
}
if (last_executed < getUnixTime() - 592200) {
// Query has not run in the last week, expire results and interval.
deleteDatabaseValue(kQueries, saved_query);
deleteDatabaseValue(kPersistentSettings, "interval." + saved_query);
deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query);
VLOG(1) << "Expiring results for scheduled query: " << saved_query;
}
}
}
void EventSubscriberPlugin::expireCheck() {
auto data_key = "data." + dbNamespace();
auto eid_key = "eid." + dbNamespace();
std::vector<std::string> keys;
scanDatabaseKeys(kEvents, keys, data_key);
if (keys.size() <= FLAGS_events_max) {
return;
}
// There is an overflow of events buffered for this subscriber.
LOG(WARNING) << "Expiring events for subscriber: " << getName() << " limit ("
<< FLAGS_events_max << ") exceeded: " << keys.size();
// Inspect the N-FLAGS_events_max -th event's value and expire before the
// time within the content.
std::string last_key;
getDatabaseValue(kEvents, eid_key, last_key);
// The EID is the next-index.
size_t max_key = boost::lexical_cast<size_t>(last_key) - FLAGS_events_max - 1;
// Convert the key index into a time using the content.
std::string content;
getDatabaseValue(kEvents, data_key + "." + std::to_string(max_key), content);
// Decode the value into a row structure to extract the time.
Row r;
if (!deserializeRowJSON(content, r) || r.count("time") == 0) {
return;
}
// The last time will become the implicit expiration time.
size_t last_time = boost::lexical_cast<size_t>(r.at("time"));
if (last_time > 0) {
expire_time_ = last_time;
}
// Finally, attempt an index query to trigger expirations.
// In this case the result set is not used.
getIndexes(expire_time_ - 1, -1);
}
TEST_F(SchedulerTests, test_config_results_purge) {
// Set a query time for now (time is only important relative to a week ago).
auto query_time = osquery::getUnixTime();
setDatabaseValue(kPersistentSettings, "timestamp.test_query",
std::to_string(query_time));
// Store a meaningless saved query interval splay.
setDatabaseValue(kPersistentSettings, "interval.test_query", "11");
// Store meaningless query differential results.
setDatabaseValue(kQueries, "test_query", "{}");
// We do not need "THE" config instance.
// We only need to trigger a 'purge' event, this occurs when configuration
// content is updated by a plugin or on load.
Config::getInstance().purge();
// Nothing should have been purged.
{
std::string content;
getDatabaseValue(kPersistentSettings, "timestamp.test_query", content);
EXPECT_FALSE(content.empty());
}
{
std::string content;
getDatabaseValue(kPersistentSettings, "interval.test_query", content);
EXPECT_FALSE(content.empty());
}
{
std::string content;
getDatabaseValue(kQueries, "test_query", content);
EXPECT_FALSE(content.empty());
}
// Update the timestamp to have run a week and a day ago.
query_time -= (84600 * (7 + 1));
setDatabaseValue(kPersistentSettings, "timestamp.test_query",
std::to_string(query_time));
// Trigger another purge.
Config::getInstance().purge();
// Now ALL 'test_query' related storage will have been purged.
{
std::string content;
getDatabaseValue(kPersistentSettings, "timestamp.test_query", content);
EXPECT_TRUE(content.empty());
}
{
std::string content;
getDatabaseValue(kPersistentSettings, "interval.test_query", content);
EXPECT_TRUE(content.empty());
}
{
std::string content;
getDatabaseValue(kQueries, "test_query", content);
EXPECT_TRUE(content.empty());
}
}
uint64_t Query::getQueryCounter(bool new_query) const {
uint64_t counter = 0;
if (new_query) {
return counter;
}
std::string raw;
auto status = getDatabaseValue(kQueries, name_ + "counter", raw);
if (status.ok()) {
counter = std::stoull(raw) + 1;
}
return counter;
}
DistributedQueryRequest Distributed::popRequest() {
// Read all pending queries.
std::vector<std::string> queries;
scanDatabaseKeys(kQueries, queries, kDistributedQueryPrefix);
// Set the last-most-recent query as the request, and delete it.
DistributedQueryRequest request;
const auto& next = queries.front();
request.id = next.substr(kDistributedQueryPrefix.size());
getDatabaseValue(kQueries, next, request.query);
deleteDatabaseValue(kQueries, next);
return request;
}
Status Query::getPreviousQueryResults(QueryDataSet& results) const {
std::string raw;
auto status = getDatabaseValue(kQueries, name_, raw);
if (!status.ok()) {
return status;
}
status = deserializeQueryDataJSON(raw, results);
if (!status.ok()) {
return status;
}
return Status(0, "OK");
}
void BufferedLogForwarder::check() {
// Get a list of all the buffered log items, with a max of 1024 lines.
std::vector<std::string> indexes;
auto status = scanDatabaseKeys(kLogs, indexes, index_name_, max_log_lines_);
// For each index, accumulate the log line into the result or status set.
std::vector<std::string> results, statuses;
iterate(indexes, ([&results, &statuses, this](std::string& index) {
std::string value;
auto& target = isResultIndex(index) ? results : statuses;
if (getDatabaseValue(kLogs, index, value)) {
target.push_back(std::move(value));
}
}));
// If any results/statuses were found in the flushed buffer, send.
if (results.size() > 0) {
status = send(results, "result");
if (!status.ok()) {
VLOG(1) << "Error sending results to logger: " << status.getMessage();
} else {
// Clear the results logs once they were sent.
iterate(indexes, ([this](std::string& index) {
if (!isResultIndex(index)) {
return;
}
deleteValueWithCount(kLogs, index);
}));
}
}
if (statuses.size() > 0) {
status = send(statuses, "status");
if (!status.ok()) {
VLOG(1) << "Error sending status to logger: " << status.getMessage();
} else {
// Clear the status logs once they were sent.
iterate(indexes, ([this](std::string& index) {
if (!isStatusIndex(index)) {
return;
}
deleteValueWithCount(kLogs, index);
}));
}
}
// Purge any logs exceeding the max after our send attempt
if (FLAGS_buffered_log_max > 0) {
purge();
}
}
TEST_F(EnrollTests, test_enroll_key_caching) {
// Cause a fetch of the node key.
auto node_key = getNodeKey("test_simple");
// Now fetch the time the node key was last cached from the database.
std::string key_time;
auto status = getDatabaseValue(kPersistentSettings, "nodeKeyTime", key_time);
EXPECT_TRUE(status.ok());
// A subsequent call to getNodeKey will return the same node key.
// But, our simple enroll plugin is not enforcing any secret check and is
// always returning the same node key.
auto node_key2 = getNodeKey("test_simple");
// In most scenarios subsequent calls to EnrollPlugin::enroll and the backing
// enrollment service will generate and return different node keys.
EXPECT_EQ(node_key2, node_key);
// To work around our contrived example we make sure the node time was not
// updated, meaning no call to EnrollPlugin::enroll occurred.
std::string key_time2;
getDatabaseValue(kPersistentSettings, "nodeKeyTime", key_time2);
EXPECT_EQ(key_time2, key_time);
}
TEST_F(EventsDatabaseTests, test_gentable) {
auto sub = std::make_shared<DBFakeEventSubscriber>();
// Lie about the tool type to enable optimizations.
auto default_type = kToolType;
kToolType = OSQUERY_TOOL_DAEMON;
ASSERT_EQ(sub->optimize_time_, 0U);
ASSERT_EQ(sub->expire_time_, 0U);
sub->testAdd(getUnixTime() - 1);
sub->testAdd(getUnixTime());
sub->testAdd(getUnixTime() + 1);
// Test the expire workflow by creating a short expiration time.
FLAGS_events_expiry = 10;
std::vector<std::string> keys;
scanDatabaseKeys("events", keys);
EXPECT_GT(keys.size(), 10U);
// Perform a "select" equivalent.
QueryContext context;
auto results = sub->genTable(context);
// Expect all non-expired results: 11, +
EXPECT_EQ(results.size(), 9U);
// The expiration time is now - events_expiry.
EXPECT_GT(sub->expire_time_, getUnixTime() - (FLAGS_events_expiry * 2));
EXPECT_LT(sub->expire_time_, getUnixTime());
// The optimize time will be changed too.
ASSERT_GT(sub->optimize_time_, 0U);
// Restore the tool type.
kToolType = default_type;
results = sub->genTable(context);
EXPECT_EQ(results.size(), 3U);
results = sub->genTable(context);
EXPECT_EQ(results.size(), 3U);
// The optimize time should have been written to the database.
// It should be the same as the current (relative) optimize time.
std::string content;
getDatabaseValue("events", "optimize.DBFakePublisher.DBFakeSubscriber",
content);
EXPECT_EQ(std::to_string(sub->optimize_time_), content);
keys.clear();
scanDatabaseKeys("events", keys);
EXPECT_LT(keys.size(), 30U);
}
TEST_F(SchedulerTests, test_monitor) {
std::string name = "pack_test_test_query";
// This query has never run so it will not have a timestamp.
std::string timestamp;
getDatabaseValue(kPersistentSettings, "timestamp." + name, timestamp);
ASSERT_TRUE(timestamp.empty());
// Fill in a scheduled query and execute it via the query monitor wrapper.
ScheduledQuery query;
query.interval = 10;
query.splayed_interval = 11;
query.query = "select * from time";
auto results = monitor(name, query);
EXPECT_EQ(results.rows().size(), 1U);
// Ask the config instance for the monitored performance.
QueryPerformance perf;
Config::getInstance().getPerformanceStats(
name, ([&perf](const QueryPerformance& r) { perf = r; }));
// Make sure it was recorded query ran.
// There is no pack for this query within the config, that is fine as these
// performance stats are tracked independently.
EXPECT_EQ(perf.executions, 1U);
EXPECT_GT(perf.output_size, 0U);
// A bit more testing, potentially redundant, check the database results.
// Since we are only monitoring, no 'actual' results are stored.
std::string content;
getDatabaseValue(kQueries, name, content);
EXPECT_TRUE(content.empty());
// Finally, make sure there is a recorded timestamp for the execution.
// We are not concerned with the APPROX value, only that it was recorded.
getDatabaseValue(kPersistentSettings, "timestamp." + name, timestamp);
EXPECT_FALSE(timestamp.empty());
}
QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
QueryData results;
// Get the records for this time range.
auto indexes = getIndexes(start, stop);
auto records = getRecords(indexes);
std::string events_key = "data." + dbNamespace();
std::vector<std::string> mapped_records;
for (const auto& record : records) {
if (record.second >= start && (record.second <= stop || stop == 0)) {
mapped_records.push_back(events_key + "." + record.first);
}
}
if (FLAGS_events_optimize && !records.empty()) {
// If records were returned save the ordered-last as the optimization EID.
unsigned long int eidr = 0;
if (safeStrtoul(records.back().first, 10, eidr)) {
optimize_eid_ = static_cast<size_t>(eidr);
auto index_key = "optimize_id." + dbNamespace();
setDatabaseValue(kEvents, index_key, records.back().first);
}
}
// Select mapped_records using event_ids as keys.
std::string data_value;
for (const auto& record : mapped_records) {
Row r;
auto status = getDatabaseValue(kEvents, record, data_value);
if (data_value.length() == 0) {
// There is no record here, interesting error case.
continue;
}
status = deserializeRowJSON(data_value, r);
data_value.clear();
if (status.ok()) {
results.push_back(std::move(r));
}
}
if (getEventsExpiry() > 0) {
// Set the expire time to NOW - "configured lifetime".
// Index retrieval will apply the constraints checking and auto-expire.
expire_time_ = getUnixTime() - getEventsExpiry();
}
return results;
}
std::vector<std::string> EventSubscriberPlugin::getIndexes(EventTime start,
EventTime stop) {
auto index_key = "indexes." + dbNamespace();
std::vector<std::string> indexes;
EventTime l_start = (start > 0) ? start / 60 : 0;
EventTime r_stop = (stop > 0) ? stop / 60 + 1 : 0;
std::string content;
getDatabaseValue(kEvents, index_key + ".60", content);
if (content.empty()) {
return indexes;
}
std::vector<std::string> bins, expirations;
boost::split(bins, content, boost::is_any_of(","));
for (const auto& bin : bins) {
auto step = timeFromRecord(bin);
auto step_start = step * 60;
auto step_stop = (step + 1) * 60;
if (step_stop < expire_time_) {
expirations.push_back(bin);
} else if (step_start < expire_time_) {
expireRecords("60", bin, false);
}
if (step >= l_start && (r_stop == 0 || step < r_stop)) {
indexes.push_back("60." + bin);
}
}
// Rewrite the index lists and delete each expired item.
if (!expirations.empty()) {
expireIndexes("60", bins, expirations);
}
// Return indexes in binning order.
std::sort(indexes.begin(),
indexes.end(),
[](const std::string& left, const std::string& right) {
auto n1 = timeFromRecord(left.substr(left.find(".") + 1));
auto n2 = timeFromRecord(right.substr(right.find(".") + 1));
return n1 < n2;
});
// Update the new time that events expire to now - expiry.
return indexes;
}
void dumpDatabase() {
for (const auto& domain : kDomains) {
std::vector<std::string> keys;
if (!scanDatabaseKeys(domain, keys)) {
continue;
}
for (const auto& key : keys) {
std::string value;
if (!getDatabaseValue(domain, key, value)) {
continue;
}
fprintf(
stdout, "%s[%s]: %s\n", domain.c_str(), key.c_str(), value.c_str());
}
}
}
Schedule::Schedule() {
if (Registry::external()) {
// Extensions should not restore or save schedule details.
return;
}
// Parse the schedule's query blacklist from backing storage.
restoreScheduleBlacklist(blacklist_);
// Check if any queries were executing when the tool last stopped.
getDatabaseValue(kPersistentSettings, kExecutingQuery, failed_query_);
if (!failed_query_.empty()) {
LOG(WARNING) << "Scheduled query may have failed: " << failed_query_;
setDatabaseValue(kPersistentSettings, kExecutingQuery, "");
// Add this query name to the blacklist and save the blacklist.
blacklist_[failed_query_] = getUnixTime() + 86400;
saveScheduleBlacklist(blacklist_);
}
}
