void EventSubscriberPlugin::expireRecords(const std::string& list_type, const std::string& index, bool all) { auto record_key = "records." + dbNamespace(); auto data_key = "data." + dbNamespace(); // If the expirations is not removing all records, rewrite the persisting. std::vector<std::string> persisting_records; // Request all records within this list-size + bin offset. auto expired_records = getRecords({list_type + "." + index}); for (const auto& record : expired_records) { if (all || record.second <= expire_time_) { deleteDatabaseValue(kEvents, data_key + "." + record.first); } else { persisting_records.push_back(record.first + ":" + std::to_string(record.second)); } } // Either drop or overwrite the record list. if (all) { deleteDatabaseValue(kEvents, record_key + "." + list_type + "." + index); } else if (persisting_records.size() < expired_records.size()) { auto new_records = boost::algorithm::join(persisting_records, ","); setDatabaseValue( kEvents, record_key + "." + list_type + "." + index, new_records); } }
void TLSLogForwarderRunner::check() { // Instead of using the 'help' database API, prefer to interact with the // DBHandle directly for additional performance. auto handle = DBHandle::getInstance(); // Get a list of all the buffered log items, with a max of 1024 lines. std::vector<std::string> indexes; auto status = handle->Scan(kLogs, indexes, kTLSMaxLogLines); // For each index, accumulate the log line into the result or status set. std::vector<std::string> results, statuses; iterate(indexes, ([&handle, &results, &statuses](std::string& index) { std::string value; auto& target = ((index.at(0) == 'r') ? results : statuses); if (handle->Get(kLogs, index, value)) { // Enforce a max log line size for TLS logging. if (value.size() > FLAGS_logger_tls_max) { LOG(WARNING) << "Line exceeds TLS logger max: " << value.size(); } else { target.push_back(std::move(value)); } } })); // If any results/statuses were found in the flushed buffer, send. if (results.size() > 0) { status = send(results, "result"); if (!status.ok()) { VLOG(1) << "Could not send results to logger URI: " << uri_ << " (" << status.getMessage() << ")"; } else { // Clear the results logs once they were sent. iterate(indexes, ([&results](std::string& index) { if (index.at(0) != 'r') { return; } deleteDatabaseValue(kLogs, index); })); } } if (statuses.size() > 0) { status = send(statuses, "status"); if (!status.ok()) { VLOG(1) << "Could not send status logs to logger URI: " << uri_ << " (" << status.getMessage() << ")"; } else { // Clear the status logs once they were sent. iterate(indexes, ([&results](std::string& index) { if (index.at(0) != 's') { return; } deleteDatabaseValue(kLogs, index); })); } } }
void Config::purge() { // The first use of purge is removing expired query results. std::vector<std::string> saved_queries; scanDatabaseKeys(kQueries, saved_queries); const auto& schedule = this->schedule_; auto queryExists = [&schedule](const std::string& query_name) { for (const auto& pack : schedule->packs_) { const auto& pack_queries = pack->getSchedule(); if (pack_queries.count(query_name)) { return true; } } return false; }; RecursiveLock lock(config_schedule_mutex_); // Iterate over each result set in the database. for (const auto& saved_query : saved_queries) { if (queryExists(saved_query)) { continue; } std::string content; getDatabaseValue(kPersistentSettings, "timestamp." + saved_query, content); if (content.empty()) { // No timestamp is set for this query, perhaps this is the first time // query results expiration is applied. setDatabaseValue(kPersistentSettings, "timestamp." + saved_query, std::to_string(getUnixTime())); continue; } // Parse the timestamp and compare. size_t last_executed = 0; try { last_executed = boost::lexical_cast<size_t>(content); } catch (const boost::bad_lexical_cast& /* e */) { // Erase the timestamp as is it potentially corrupt. deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query); continue; } if (last_executed < getUnixTime() - 592200) { // Query has not run in the last week, expire results and interval. deleteDatabaseValue(kQueries, saved_query); deleteDatabaseValue(kPersistentSettings, "interval." + saved_query); deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query); VLOG(1) << "Expiring results for scheduled query: " << saved_query; } } }
/// Remove these ATC tables from the registry and database Status ATCConfigParserPlugin::removeATCTables( const std::set<std::string>& detach_tables) { auto registry_table = RegistryFactory::get().registry("table"); std::set<std::string> failed_tables; for (const auto& table : detach_tables) { if (registry_table->exists(table)) { std::string value; if (getDatabaseValue( kPersistentSettings, kDatabaseKeyPrefix + table, value) .ok()) { registry_table->remove(table); PluginResponse resp; Registry::call( "sql", "sql", {{"action", "detatch"}, {"table", table}}, resp); LOG(INFO) << "Removed ATC table: " << table; } else { failed_tables.insert(table); } } deleteDatabaseValue(kPersistentSettings, kDatabaseKeyPrefix + table); } if (failed_tables.empty()) { return Status(); } return Status( 1, "Attempted to remove non ATC tables: " + join(failed_tables, ", ")); }
TEST_F(TLSLoggerTests, test_database) { // Start a server. TLSServerRunner::start(); TLSServerRunner::setClientConfig(); auto forwarder = std::make_shared<TLSLogForwarder>(); std::string expected = "{\"new_json\": true}"; forwarder->logString(expected); StatusLogLine status; status.message = "{\"status\": \"bar\"}"; forwarder->logStatus({status}); // Stop the server. TLSServerRunner::unsetClientConfig(); TLSServerRunner::stop(); std::vector<std::string> indexes; scanDatabaseKeys(kLogs, indexes); EXPECT_EQ(2U, indexes.size()); // Iterate using an unordered search, and search for the expected string // that was just logged. bool found_string = false; for (const auto& index : indexes) { std::string value; getDatabaseValue(kLogs, index, value); found_string = (found_string || value == expected); deleteDatabaseValue(kLogs, index); } EXPECT_TRUE(found_string); }
static void DATABASE_store(benchmark::State& state) { while (state.KeepRunning()) { setDatabaseValue(kPersistentSettings, "benchmark", "1"); } // All benchmarks will share a single database handle. deleteDatabaseValue(kPersistentSettings, "benchmark"); }
void EventSubscriberPlugin::expireCheck(bool cleanup) { auto data_key = "data." + dbNamespace(); auto eid_key = "eid." + dbNamespace(); // Min key will be the last surviving key. size_t min_key = 0; { auto limit = getEventsMax(); std::vector<std::string> keys; scanDatabaseKeys(kEvents, keys, data_key); if (keys.size() <= limit) { return; } // There is an overflow of events buffered for this subscriber. LOG(WARNING) << "Expiring events for subscriber: " << getName() << " (limit " << limit << ")"; VLOG(1) << "Subscriber events " << getName() << " exceeded limit " << limit << " by: " << keys.size() - limit; // Inspect the N-FLAGS_events_max -th event's value and expire before the // time within the content. std::string last_key; getDatabaseValue(kEvents, eid_key, last_key); // The EID is the next-index. // EID - events_max is the most last-recent event to keep. min_key = boost::lexical_cast<size_t>(last_key) - getEventsMax(); if (cleanup) { // Scan each of the keys in keys, if their ID portion is < min_key. // Nix them, this requires lots of conversions, use with care. for (const auto& key : keys) { if (std::stoul(key.substr(key.rfind('.') + 1)) < min_key) { deleteDatabaseValue(kEvents, key); } } } } // Convert the key index into a time using the content. // The last-recent event is fetched and the corresponding time is used as // the expiration time for the subscriber. std::string content; getDatabaseValue(kEvents, data_key + "." + std::to_string(min_key), content); // Decode the value into a row structure to extract the time. Row r; if (!deserializeRowJSON(content, r) || r.count("time") == 0) { return; } // The last time will become the implicit expiration time. size_t last_time = boost::lexical_cast<size_t>(r.at("time")); if (last_time > 0) { expire_time_ = last_time; } // Finally, attempt an index query to trigger expirations. // In this case the result set is not used. getIndexes(expire_time_, 0); }
Status BufferedLogForwarder::deleteValueWithCount(const std::string& domain, const std::string& key) { Status status = deleteDatabaseValue(domain, key); if (status.ok()) { buffer_count_--; } return status; }
Status ViewsConfigParserPlugin::update(const std::string& source, const ParserConfig& config) { auto cv = config.find("views"); if (cv == config.end()) { return Status(1); } auto obj = data_.getObject(); data_.copyFrom(cv->second.doc(), obj); data_.add("views", obj); const auto& views = data_.doc()["views"]; // We use a restricted scope below to change the data structure from // an array to a set. This lets us do deletes much more efficiently std::vector<std::string> created_views; std::set<std::string> erase_views; { std::vector<std::string> old_views_vec; scanDatabaseKeys(kQueries, old_views_vec, kConfigViews); for (const auto& view : old_views_vec) { erase_views.insert(view.substr(kConfigViews.size())); } } QueryData r; for (const auto& view : views.GetObject()) { std::string name = view.name.GetString(); std::string query = view.value.GetString(); if (query.empty()) { continue; } std::string old_query = ""; getDatabaseValue(kQueries, kConfigViews + name, old_query); erase_views.erase(name); if (old_query == query) { continue; } // View has been updated osquery::query("DROP VIEW " + name, r); auto s = osquery::query("CREATE VIEW " + name + " AS " + query, r); if (s.ok()) { setDatabaseValue(kQueries, kConfigViews + name, query); } else { LOG(INFO) << "Error creating view (" << name << "): " << s.getMessage(); } } // Any views left are views that don't exist in the new configuration file // so we tear them down and remove them from the database. for (const auto& old_view : erase_views) { osquery::query("DROP VIEW " + old_view, r); deleteDatabaseValue(kQueries, kConfigViews + old_view); } return Status(0, "OK"); }
static void DATABASE_store_append(benchmark::State& state) { // Serialize the example result set into a string. std::string content; auto qd = getExampleQueryData(20, 100); serializeQueryDataJSON(qd, content); size_t k = 0; while (state.KeepRunning()) { setDatabaseValue(kPersistentSettings, "key" + std::to_string(k), content); deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(k)); k++; } // All benchmarks will share a single database handle. for (size_t i = 0; i < k; ++i) { deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(i)); } }
Status ATCConfigParserPlugin::setUp() { VLOG(1) << "Removing stale ATC entries"; std::vector<std::string> keys; scanDatabaseKeys(kPersistentSettings, keys, kDatabaseKeyPrefix); for (const auto& key : keys) { auto s = deleteDatabaseValue(kPersistentSettings, key); if (!s.ok()) { LOG(INFO) << "Could not clear ATC key " << key << "from database"; } } return Status(); }
static void DATABASE_store_large(benchmark::State& state) { // Serialize the example result set into a string. std::string content; auto qd = getExampleQueryData(20, 100); serializeQueryDataJSON(qd, content); while (state.KeepRunning()) { setDatabaseValue(kPersistentSettings, "benchmark", content); } // All benchmarks will share a single database handle. deleteDatabaseValue(kPersistentSettings, "benchmark"); }
DistributedQueryRequest Distributed::popRequest() { // Read all pending queries. std::vector<std::string> queries; scanDatabaseKeys(kQueries, queries, kDistributedQueryPrefix); // Set the last-most-recent query as the request, and delete it. DistributedQueryRequest request; const auto& next = queries.front(); request.id = next.substr(kDistributedQueryPrefix.size()); getDatabaseValue(kQueries, next, request.query); deleteDatabaseValue(kQueries, next); return request; }
TEST_F(DatabaseTests, test_delete_values) { setDatabaseValue(kLogs, "k", "0"); std::string value; getDatabaseValue(kLogs, "k", value); EXPECT_FALSE(value.empty()); auto s = deleteDatabaseValue(kLogs, "k"); EXPECT_TRUE(s.ok()); // Make sure the key has been deleted. value.clear(); s = getDatabaseValue(kLogs, "k", value); EXPECT_FALSE(s.ok()); EXPECT_TRUE(value.empty()); }
Status init() override { // Before starting our subscription, purge any residual db entries as it's // unlikely we'll finish re-assmebling them std::vector<std::string> keys; scanDatabaseKeys(kEvents, keys, kScriptBlockPrefix); for (const auto& k : keys) { auto s = deleteDatabaseValue(kEvents, k); if (!s.ok()) { VLOG(1) << "Failed to delete stale script block from the database " << k; } } auto wc = createSubscriptionContext(); wc->sources.insert(kPowershellEventsChannel); subscribe(&PowershellEventSubscriber::Callback, wc); return Status(); }
Status PowershellEventSubscriber::Callback(const ECRef& ec, const SCRef& sc) { // For script block logging we only care about events with script blocks auto eid = ec->eventRecord.get("Event.System.EventID", -1); if (eid != kScriptBlockLoggingEid) { return Status(); } Row results; for (const auto& node : ec->eventRecord.get_child("Event", pt::ptree())) { if (node.first == "System" || node.first == "<xmlattr>") { continue; } // #4357: This should make use of RapidJSON parseTree(node.second, results); } FILETIME etime; GetSystemTimeAsFileTime(&etime); results["time"] = BIGINT(filetimeToUnixtime(etime)); results["datetime"] = ec->eventRecord.get("Event.System.TimeCreated.<xmlattr>.SystemTime", ""); // If there's only one script block no reassembly is needed if (results["MessageTotal"] == "1") { addScriptResult(results); return Status(); } // Add the script content to the DB for later reassembly auto s = setDatabaseValue(kEvents, kScriptBlockPrefix + results["ScriptBlockId"] + "." + results["MessageNumber"], results["ScriptBlockText"]); if (!s.ok()) { LOG(WARNING) << "Failed to add new Powershell block to database for script " << results["ScriptBlockId"]; } // If we expect more blocks bail out early if (results["MessageNumber"] != results["MessageTotal"]) { return Status(); } // Otherwise all script blocks should be accounted for so reconstruct std::vector<std::string> keys; s = scanDatabaseKeys( kEvents, keys, kScriptBlockPrefix + results["ScriptBlockId"]); if (!s.ok()) { LOG(WARNING) << "Failed to look up powershell script blocks for " << results["ScriptBlockId"]; return Status(1); } std::string powershell_script{""}; for (const auto& key : keys) { std::string val{""}; s = getDatabaseValue(kEvents, key, val); if (!s.ok()) { LOG(WARNING) << "Failed to retrieve script block " << key; continue; } powershell_script += val; s = deleteDatabaseValue(kEvents, key); if (!s.ok()) { LOG(WARNING) << "Failed to delete script block key from db " << key; } } results["ScriptBlockText"] = powershell_script; addScriptResult(results); return Status(); }
Status ATCConfigParserPlugin::update(const std::string& source, const ParserConfig& config) { auto cv = config.find(kParserKey); if (cv == config.end()) { return Status(1, "No configuration for ATC (Auto Table Construction)"); } auto obj = data_.getObject(); data_.copyFrom(cv->second.doc(), obj); data_.add(kParserKey, obj); const auto& ac_tables = data_.doc()[kParserKey]; auto tables = RegistryFactory::get().registry("table"); auto registered = registeredATCTables(); for (const auto& ac_table : ac_tables.GetObject()) { std::string table_name{ac_table.name.GetString()}; auto params = ac_table.value.GetObject(); std::string query{params.HasMember("query") && params["query"].IsString() ? params["query"].GetString() : ""}; std::string path{params.HasMember("path") && params["path"].IsString() ? params["path"].GetString() : ""}; std::string platform{params.HasMember("platform") && params["platform"].IsString() ? params["platform"].GetString() : ""}; if (query.empty() || path.empty()) { LOG(WARNING) << "ATC Table: " << table_name << " is misconfigured"; continue; } if (!checkPlatform(platform)) { VLOG(1) << "Skipping ATC table: " << table_name << " because platform doesn't match"; continue; } TableColumns columns; std::string columns_value; columns_value.reserve(256); for (const auto& column : params["columns"].GetArray()) { columns.push_back(make_tuple( std::string(column.GetString()), TEXT_TYPE, ColumnOptions::DEFAULT)); columns_value += std::string(column.GetString()) + ","; } registered.erase(table_name); std::string table_settings{table_name + query + columns_value + path}; std::string old_setting; auto s = getDatabaseValue( kPersistentSettings, kDatabaseKeyPrefix + table_name, old_setting); // The ATC table hasn't changed so we skip ahead if (table_settings == old_setting) { continue; } // Remove the old table to replace with the new one s = removeATCTables({table_name}); if (!s.ok()) { LOG(WARNING) << "ATC table overrides core table; Refusing registration"; continue; } s = setDatabaseValue( kPersistentSettings, kDatabaseKeyPrefix + table_name, table_settings); if (!s.ok()) { LOG(WARNING) << "Could not write to database"; continue; } s = tables->add( table_name, std::make_shared<ATCPlugin>(path, columns, query), true); if (!s.ok()) { LOG(WARNING) << s.getMessage(); deleteDatabaseValue(kPersistentSettings, kDatabaseKeyPrefix + table_name); continue; } PluginResponse resp; Registry::call( "sql", "sql", {{"action", "attach"}, {"table", table_name}}, resp); LOG(INFO) << "Registered ATC table: " << table_name; } if (registered.size() > 0) { VLOG(1) << "Removing any ATC tables that were removed in this configuration " "change"; removeATCTables(registered); } return Status(); }
void SetUp() { deleteDatabaseValue(kPersistentSettings, "nodeKey"); deleteDatabaseValue(kPersistentSettings, "nodeKeyTime"); }