Status INotifyEventPublisher::restartMonitoring() {
if (last_restart_ != 0 && getUnixTime() - last_restart_ < 10) {
return Status(1, "Overflow");
}
last_restart_ = getUnixTime();
VLOG(1) << "inotify was overflown, attempting to restart handle";
// Create a copy of the descriptors, then remove each.
auto descriptors = descriptors_;
for (const auto& desc : descriptors) {
removeMonitor(desc, true);
}
{
// Then remove all path/descriptor mappings.
WriteLock lock(mutex_);
path_descriptors_.clear();
descriptor_paths_.clear();
}
// Reconfigure ourself, the subscribers will not reconfigure.
configure();
return Status(0, "OK");
}
TEST_F(PlistBenchmark, bench_parse_plist_content) {
// using LOG(ERROR) as a quick hack so that gtest displays the log line even
// when the test passes
LOG(ERROR) << "Starting: " << getAsciiTime();
LOG(ERROR) << "Performing " << FLAGS_iterations << " iterations";
int time = getUnixTime();
for (int i = 0; i < FLAGS_iterations; ++i) {
std::string content = getPlistContent();
pt::ptree tree;
auto s = parsePlistContent(content, tree);
EXPECT_TRUE(s.ok());
EXPECT_EQ(s.toString(), "OK");
EXPECT_EQ(tree.get<bool>("Disabled"), true);
EXPECT_THROW(tree.get<bool>("foobar"), pt::ptree_bad_path);
EXPECT_EQ(tree.get<std::string>("Label"), "com.apple.FileSyncAgent.sshd");
std::vector<std::string> program_arguments = {
"/System/Library/CoreServices/FileSyncAgent.app/Contents/Resources/"
"FileSyncAgent_sshd-keygen-wrapper",
"-i",
"-f",
"/System/Library/CoreServices/FileSyncAgent.app/Contents/Resources/"
"FileSyncAgent_sshd_config",
};
pt::ptree program_arguments_tree = tree.get_child("ProgramArguments");
std::vector<std::string> program_arguments_parsed;
for (const auto& argument : program_arguments_tree) {
program_arguments_parsed.push_back(argument.second.get<std::string>(""));
}
EXPECT_EQ(program_arguments_parsed, program_arguments);
}
LOG(ERROR) << "Ending: " << getAsciiTime();
LOG(ERROR) << "Benchmark executed in " << (getUnixTime() - time)
<< " seconds";
}
void WatcherRunner::createWorker() {
{
WatcherLocker locker;
if (Watcher::getState(Watcher::getWorker()).last_respawn_time >
getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) {
LOG(WARNING) << "osqueryd worker respawning too quickly: "
<< Watcher::workerRestartCount() << " times";
Watcher::workerRestarted();
// The configured automatic delay.
size_t delay = getWorkerLimit(RESPAWN_DELAY) * 1000;
// Exponential back off for quickly-respawning clients.
delay += pow(2, Watcher::workerRestartCount()) * 1000;
pauseMilli(delay);
}
}
// Get the path of the current process.
auto qd =
SQL::selectAllFrom("processes",
"pid",
EQUALS,
INTEGER(PlatformProcess::getCurrentProcess()->pid()));
if (qd.size() != 1 || qd[0].count("path") == 0 || qd[0]["path"].size() == 0) {
LOG(ERROR) << "osquery watcher cannot determine process path for worker";
Initializer::requestShutdown(EXIT_FAILURE);
return;
}
// Set an environment signaling to potential plugin-dependent workers to wait
// for extensions to broadcast.
if (Watcher::hasManagedExtensions()) {
setEnvVar("OSQUERY_EXTENSIONS", "true");
}
// Get the complete path of the osquery process binary.
boost::system::error_code ec;
auto exec_path = fs::system_complete(fs::path(qd[0]["path"]), ec);
if (!safePermissions(
exec_path.parent_path().string(), exec_path.string(), true)) {
// osqueryd binary has become unsafe.
LOG(ERROR) << RLOG(1382)
<< "osqueryd has unsafe permissions: " << exec_path.string();
Initializer::requestShutdown(EXIT_FAILURE);
return;
}
auto worker = PlatformProcess::launchWorker(exec_path.string(), argc_, argv_);
if (worker == nullptr) {
// Unrecoverable error, cannot create a worker process.
LOG(ERROR) << "osqueryd could not create a worker process";
Initializer::shutdown(EXIT_FAILURE);
return;
}
Watcher::setWorker(worker);
Watcher::resetWorkerCounters(getUnixTime());
VLOG(1) << "osqueryd watcher (" << PlatformProcess::getCurrentProcess()->pid()
<< ") executing worker (" << worker->pid() << ")";
}
void FileLib::getFileStatus(
const WIN32_FIND_DATAW &data, FileStatus *status) {
status->accessTime_ = getUnixTime(data.ftLastAccessTime);
status->attributes_ = data.dwFileAttributes;
status->creationTime_ = getUnixTime(data.ftCreationTime);
status->hardLinkCount_ = 1;
status->modificationTime_ = getUnixTime(data.ftLastWriteTime);
ULARGE_INTEGER size;
size.HighPart = data.nFileSizeHigh;
size.LowPart = data.nFileSizeLow;
status->size_ = size.QuadPart;
}
void Config::purge() {
// The first use of purge is removing expired query results.
std::vector<std::string> saved_queries;
scanDatabaseKeys(kQueries, saved_queries);
const auto& schedule = this->schedule_;
auto queryExists = [&schedule](const std::string& query_name) {
for (const auto& pack : schedule->packs_) {
const auto& pack_queries = pack->getSchedule();
if (pack_queries.count(query_name)) {
return true;
}
}
return false;
};
RecursiveLock lock(config_schedule_mutex_);
// Iterate over each result set in the database.
for (const auto& saved_query : saved_queries) {
if (queryExists(saved_query)) {
continue;
}
std::string content;
getDatabaseValue(kPersistentSettings, "timestamp." + saved_query, content);
if (content.empty()) {
// No timestamp is set for this query, perhaps this is the first time
// query results expiration is applied.
setDatabaseValue(kPersistentSettings,
"timestamp." + saved_query,
std::to_string(getUnixTime()));
continue;
}
// Parse the timestamp and compare.
size_t last_executed = 0;
try {
last_executed = boost::lexical_cast<size_t>(content);
} catch (const boost::bad_lexical_cast& /* e */) {
// Erase the timestamp as is it potentially corrupt.
deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query);
continue;
}
if (last_executed < getUnixTime() - 592200) {
// Query has not run in the last week, expire results and interval.
deleteDatabaseValue(kQueries, saved_query);
deleteDatabaseValue(kPersistentSettings, "interval." + saved_query);
deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query);
VLOG(1) << "Expiring results for scheduled query: " << saved_query;
}
}
}
void FileLib::getFileStatus(
const BY_HANDLE_FILE_INFORMATION &info, FileStatus *status) {
status->accessTime_ = getUnixTime(info.ftLastAccessTime);
status->attributes_ = info.dwFileAttributes;
status->creationTime_ = getUnixTime(info.ftCreationTime);
status->hardLinkCount_ = info.nNumberOfLinks;
status->modificationTime_ = getUnixTime(info.ftLastWriteTime);
ULARGE_INTEGER size;
size.HighPart = info.nFileSizeHigh;
size.LowPart = info.nFileSizeLow;
status->size_ = size.QuadPart;
}
TEST_F(EventsDatabaseTests, test_gentable) {
auto sub = std::make_shared<DBFakeEventSubscriber>();
// Lie about the tool type to enable optimizations.
auto default_type = kToolType;
kToolType = OSQUERY_TOOL_DAEMON;
ASSERT_EQ(sub->optimize_time_, 0U);
ASSERT_EQ(sub->expire_time_, 0U);
sub->testAdd(getUnixTime() - 1);
sub->testAdd(getUnixTime());
sub->testAdd(getUnixTime() + 1);
// Test the expire workflow by creating a short expiration time.
FLAGS_events_expiry = 10;
std::vector<std::string> keys;
scanDatabaseKeys("events", keys);
EXPECT_GT(keys.size(), 10U);
// Perform a "select" equivalent.
QueryContext context;
auto results = sub->genTable(context);
// Expect all non-expired results: 11, +
EXPECT_EQ(results.size(), 9U);
// The expiration time is now - events_expiry.
EXPECT_GT(sub->expire_time_, getUnixTime() - (FLAGS_events_expiry * 2));
EXPECT_LT(sub->expire_time_, getUnixTime());
// The optimize time will be changed too.
ASSERT_GT(sub->optimize_time_, 0U);
// Restore the tool type.
kToolType = default_type;
results = sub->genTable(context);
EXPECT_EQ(results.size(), 3U);
results = sub->genTable(context);
EXPECT_EQ(results.size(), 3U);
// The optimize time should have been written to the database.
// It should be the same as the current (relative) optimize time.
std::string content;
getDatabaseValue("events", "optimize.DBFakePublisher.DBFakeSubscriber",
content);
EXPECT_EQ(std::to_string(sub->optimize_time_), content);
keys.clear();
scanDatabaseKeys("events", keys);
EXPECT_LT(keys.size(), 30U);
}
Status INotifyEventPublisher::restartMonitoring(){
if (last_restart_ != 0 && getUnixTime() - last_restart_ < 10) {
return Status(1, "Overflow");
}
last_restart_ = getUnixTime();
VLOG(1) << "Got an overflow, trying to restart...";
for(const auto& desc : descriptors_){
removeMonitor(desc, 1);
}
path_descriptors_.clear();
descriptor_paths_.clear();
configure();
return Status(0, "OK");
}
void FileLib::getFileStatus(const struct stat &stBuf, FileStatus *status) {
status->accessTime_ = getUnixTime(stBuf.st_atime);
status->blockCount_ = stBuf.st_blocks;
status->blockSize_ = stBuf.st_blksize;
status->changeTime_ = getUnixTime(stBuf.st_ctime);
status->device_ = stBuf.st_dev;
status->gid_ = stBuf.st_gid;
status->hardLinkCount_ = stBuf.st_nlink;
status->iNode_ = stBuf.st_ino;
status->mode_ = stBuf.st_mode;
status->modificationTime_ = getUnixTime(stBuf.st_mtime);
status->rDevice_ = stBuf.st_rdev;
status->size_ = stBuf.st_size;
status->uid_ = stBuf.st_uid;
}
QueryData EventSubscriberPlugin::genTable(QueryContext& context) {
// Stop is an unsigned (-1), our end of time equivalent.
EventTime start = 0, stop = -1;
if (context.constraints["time"].getAll().size() > 0) {
// Use the 'time' constraint to optimize backing-store lookups.
for (const auto& constraint : context.constraints["time"].getAll()) {
EventTime expr = timeFromRecord(constraint.expr);
if (constraint.op == EQUALS) {
stop = start = expr;
break;
} else if (constraint.op == GREATER_THAN) {
start = std::max(start, expr + 1);
} else if (constraint.op == GREATER_THAN_OR_EQUALS) {
start = std::max(start, expr);
} else if (constraint.op == LESS_THAN) {
stop = std::min(stop, expr - 1);
} else if (constraint.op == LESS_THAN_OR_EQUALS) {
stop = std::min(stop, expr);
}
}
} else if (kToolType == OSQUERY_TOOL_DAEMON && FLAGS_events_optimize) {
// If the daemon is querying a subscriber without a 'time' constraint and
// allows optimization, only emit events since the last query.
start = optimize_time_;
optimize_time_ = getUnixTime() - 1;
// Store the optimize time such that it can be restored if the daemon is
// restarted.
auto index_key = "optimize." + dbNamespace();
setDatabaseValue(kEvents, index_key, std::to_string(optimize_time_));
}
return get(start, stop);
}
void EventPublisherPlugin::fire(const EventContextRef& ec, EventTime time) {
EventContextID ec_id;
if (isEnding()) {
// Cannot emit/fire while ending
return;
}
{
boost::lock_guard<boost::mutex> lock(ec_id_lock_);
ec_id = next_ec_id_++;
}
// Fill in EventContext ID and time if needed.
if (ec != nullptr) {
ec->id = ec_id;
if (ec->time == 0) {
if (time == 0) {
time = getUnixTime();
}
// Todo: add a check to assure normalized (seconds) time.
ec->time = time;
}
}
for (const auto& subscription : subscriptions_) {
auto es = EventFactory::getEventSubscriber(subscription->subscriber_name);
if (es->state() == SUBSCRIBER_RUNNING) {
fireCallback(subscription, ec);
}
}
}
Status carvePaths(const std::set<std::string>& paths) {
auto guid = generateNewUUID();
pt::ptree tree;
tree.put("carve_guid", guid);
tree.put("time", getUnixTime());
tree.put("status", "STARTING");
tree.put("sha256", "");
tree.put("size", -1);
if (paths.size() > 1) {
tree.put("path", boost::algorithm::join(paths, ","));
} else {
tree.put("path", *(paths.begin()));
}
std::ostringstream os;
pt::write_json(os, tree, false);
auto s = setDatabaseValue(kCarveDbDomain, kCarverDBPrefix + guid, os.str());
if (!s.ok()) {
return s;
} else {
auto requestId = Distributed::getCurrentRequestId();
Dispatcher::addService(std::make_shared<Carver>(paths, guid, requestId));
}
return s;
}
Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
std::shared_ptr<DBHandle> db = nullptr;
try {
db = DBHandle::getInstance();
} catch (const std::runtime_error& e) {
return Status(1, e.what());
}
// Get and increment the EID for this module.
EventID eid = getEventID();
// Without encouraging a missing event time, do not support a 0-time.
r["time"] = std::to_string((event_time == 0) ? getUnixTime() : event_time);
// Serialize and store the row data, for query-time retrieval.
std::string data;
auto status = serializeRowJSON(r, data);
if (!status.ok()) {
return status;
}
// Store the event data.
std::string event_key = "data." + dbNamespace() + "." + eid;
status = db->Put(kEvents, event_key, data);
// Record the event in the indexing bins, using the index time.
recordEvent(eid, event_time);
return status;
}
QueryData EventSubscriberPlugin::genTable(QueryContext& context) {
EventTime start = 0, stop = -1;
if (context.constraints["time"].getAll().size() > 0) {
// Use the 'time' constraint to optimize backing-store lookups.
for (const auto& constraint : context.constraints["time"].getAll()) {
EventTime expr = 0;
try {
expr = boost::lexical_cast<EventTime>(constraint.expr);
} catch (const boost::bad_lexical_cast& e) {
expr = 0;
}
if (constraint.op == EQUALS) {
stop = start = expr;
break;
} else if (constraint.op == GREATER_THAN) {
start = std::max(start, expr + 1);
} else if (constraint.op == GREATER_THAN_OR_EQUALS) {
start = std::max(start, expr);
} else if (constraint.op == LESS_THAN) {
stop = std::min(stop, expr - 1);
} else if (constraint.op == LESS_THAN_OR_EQUALS) {
stop = std::min(stop, expr);
}
}
} else if (kToolType == OSQUERY_TOOL_DAEMON && FLAGS_events_optimize) {
// If the daemon is querying a subscriber without a 'time' constraint and
// allows optimization, only emit events since the last query.
start = optimize_time_;
optimize_time_ = getUnixTime() - 1;
}
return get(start, stop);
}
Status INotifyEventPublisher::restartMonitoring() {
if (last_restart_ != 0 && getUnixTime() - last_restart_ < 10) {
return Status(1, "Overflow");
}
last_restart_ = getUnixTime();
VLOG(1) << "inotify was overflown, attempting to restart handle";
for (const auto& desc : descriptors_) {
removeMonitor(desc, true);
}
path_descriptors_.clear();
descriptor_paths_.clear();
configure();
return Status(0, "OK");
}
void Config::scheduledQueries(
std::function<void(const std::string& name, const ScheduledQuery& query)>
predicate) {
RecursiveLock lock(config_schedule_mutex_);
for (const PackRef& pack : *schedule_) {
for (const auto& it : pack->getSchedule()) {
std::string name = it.first;
// The query name may be synthetic.
if (pack->getName() != "main" && pack->getName() != "legacy_main") {
name = "pack" + FLAGS_pack_delimiter + pack->getName() +
FLAGS_pack_delimiter + it.first;
}
// They query may have failed and been added to the schedule's blacklist.
if (schedule_->blacklist_.count(name) > 0) {
auto blacklisted_query = schedule_->blacklist_.find(name);
if (getUnixTime() > blacklisted_query->second) {
// The blacklisted query passed the expiration time (remove).
schedule_->blacklist_.erase(blacklisted_query);
saveScheduleBlacklist(schedule_->blacklist_);
} else {
// The query is still blacklisted.
continue;
}
}
// Call the predicate.
predicate(name, it.second);
}
}
}
std::string BufferedLogForwarder::genIndex(bool results, size_t time) {
if (time == 0) {
time = getUnixTime();
}
return genIndexPrefix(results) + std::to_string(time) + "_" +
std::to_string(++log_index_);
}
Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
std::shared_ptr<DBHandle> db = nullptr;
try {
db = DBHandle::getInstance();
} catch (const std::runtime_error& e) {
return Status(1, e.what());
}
// Get and increment the EID for this module.
EventID eid = getEventID();
// Without encouraging a missing event time, do not support a 0-time.
r["time"] = std::to_string((event_time == 0) ? getUnixTime() : event_time);
// Serialize and store the row data, for query-time retrieval.
std::string data;
auto status = serializeRowJSON(r, data);
if (!status.ok()) {
return status;
}
// Use the last EventID and a checkpoint bucket size to periodically apply
// buffer eviction. Eviction occurs if the total count exceeds events_max.
if (last_eid_ % EVENTS_CHECKPOINT == 0) {
expireCheck();
}
// Store the event data.
std::string event_key = "data." + dbNamespace() + "." + eid;
status = db->Put(kEvents, event_key, data);
// Record the event in the indexing bins, using the index time.
recordEvent(eid, event_time);
return status;
}
Status ProcessEventSubscriber::Callback(const ECRef& ec, const SCRef& sc) {
// Check and set the valid state change.
// If this is an unacceptable change reset the state and clear row data.
if (ec->fields.count("success") && ec->fields.at("success") == "no") {
return Status(0, "OK");
}
if (!validAuditState(ec->type, state_).ok()) {
state_ = STATE_SYSCALL;
Row().swap(row_);
return Status(0, "OK");
}
// Fill in row fields based on the event state.
updateAuditRow(ec, row_);
// Only add the event if finished (aka a PATH event was emitted).
if (state_ == STATE_SYSCALL) {
// If the EXECVE state was not used, decode the cmdline value.
if (row_.at("cmdline_size").size() == 0) {
// This allows at most 1 decode call per potentially-encoded item.
row_["cmdline"] = decodeAuditValue(row_.at("cmdline"));
row_["cmdline_size"] = "1";
}
add(row_, getUnixTime());
Row().swap(row_);
}
return Status(0, "OK");
}
TEST_F(ConfigTests, test_schedule_blacklist) {
auto current_time = getUnixTime();
std::map<std::string, size_t> blacklist;
saveScheduleBlacklist(blacklist);
restoreScheduleBlacklist(blacklist);
EXPECT_EQ(blacklist.size(), 0U);
// Create some entries.
blacklist["test_1"] = current_time * 2;
blacklist["test_2"] = current_time * 3;
saveScheduleBlacklist(blacklist);
blacklist.clear();
restoreScheduleBlacklist(blacklist);
ASSERT_EQ(blacklist.count("test_1"), 1U);
ASSERT_EQ(blacklist.count("test_2"), 1U);
EXPECT_EQ(blacklist.at("test_1"), current_time * 2);
EXPECT_EQ(blacklist.at("test_2"), current_time * 3);
// Now save an expired query.
blacklist["test_1"] = 1;
saveScheduleBlacklist(blacklist);
blacklist.clear();
// When restoring, the values below the current time will not be included.
restoreScheduleBlacklist(blacklist);
EXPECT_EQ(blacklist.size(), 1U);
}
Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
// Get and increment the EID for this module.
EventID eid = getEventID();
// Without encouraging a missing event time, do not support a 0-time.
r["time"] = std::to_string((event_time == 0) ? getUnixTime() : event_time);
// Serialize and store the row data, for query-time retrieval.
std::string data;
auto status = serializeRowJSON(r, data);
if (!status.ok()) {
return status;
}
// Then remove the newline.
if (data.size() > 0 && data.back() == '\n') {
data.pop_back();
}
// Use the last EventID and a checkpoint bucket size to periodically apply
// buffer eviction. Eviction occurs if the total count exceeds events_max.
if (last_eid_ % EVENTS_CHECKPOINT == 0) {
expireCheck();
}
// Store the event data.
std::string event_key = "data." + dbNamespace() + "." + eid;
status = setDatabaseValue(kEvents, event_key, data);
// Record the event in the indexing bins, using the index time.
recordEvent(eid, event_time);
return status;
}
void EventPublisherPlugin::fire(const EventContextRef& ec, EventTime time) {
if (isEnding()) {
// Cannot emit/fire while ending
return;
}
EventContextID ec_id = 0;
{
WriteLock lock(ec_id_lock_);
ec_id = next_ec_id_++;
}
// Fill in EventContext ID and time if needed.
if (ec != nullptr) {
ec->id = ec_id;
if (ec->time == 0) {
if (time == 0) {
time = getUnixTime();
}
ec->time = time;
}
}
for (const auto& subscription : subscriptions_) {
auto es = EventFactory::getEventSubscriber(subscription->subscriber_name);
if (es != nullptr && es->state() == SUBSCRIBER_RUNNING) {
es->event_count_++;
fireCallback(subscription, ec);
}
}
}
bool Pack::checkDiscovery() {
stats_.total++;
auto current = getUnixTime();
if ((current - discovery_cache_.first) < FLAGS_pack_refresh_interval) {
stats_.hits++;
return discovery_cache_.second;
}
stats_.misses++;
discovery_cache_.first = current;
discovery_cache_.second = true;
for (const auto& q : discovery_queries_) {
auto sql = SQL(q);
if (!sql.ok()) {
LOG(WARNING) << "Discovery query failed (" << q
<< "): " << sql.getMessageString();
discovery_cache_.second = false;
break;
}
if (sql.rows().size() == 0) {
discovery_cache_.second = false;
break;
}
}
return discovery_cache_.second;
}
void Config::recordQueryStart(const std::string& name) {
// There should only ever be a single executing query in the schedule.
setDatabaseValue(kPersistentSettings, kExecutingQuery, name);
// Store the time this query name last executed for later results eviction.
// When configuration updates occur the previous schedule is searched for
// 'stale' query names, aka those that have week-old or longer last execute
// timestamps. Offending queries have their database results purged.
setDatabaseValue(
kPersistentSettings, "timestamp." + name, std::to_string(getUnixTime()));
}
bool WatcherRunner::createExtension(const std::string& extension) {
{
WatcherLocker locker;
if (Watcher::getState(extension).last_respawn_time >
getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) {
LOG(WARNING) << "Extension respawning too quickly: " << extension;
// Unlike a worker, if an extension respawns to quickly we give up.
return false;
}
}
// Check the path to the previously-discovered extension binary.
boost::system::error_code ec;
auto exec_path = fs::system_complete(fs::path(extension), ec);
if (!safePermissions(
exec_path.parent_path().string(), exec_path.string(), true)) {
// Extension binary has become unsafe.
LOG(WARNING) << RLOG(1382)
<< "Extension binary has unsafe permissions: " << extension;
return false;
}
auto ext_process =
PlatformProcess::launchExtension(exec_path.string(),
extension,
Flag::getValue("extensions_socket"),
Flag::getValue("extensions_timeout"),
Flag::getValue("extensions_interval"),
Flag::getValue("verbose"));
if (ext_process == nullptr) {
// Unrecoverable error, cannot create an extension process.
LOG(ERROR) << "Cannot create extension process: " << extension;
Initializer::shutdown(EXIT_FAILURE);
}
Watcher::setExtension(extension, ext_process);
Watcher::resetExtensionCounters(extension, getUnixTime());
VLOG(1) << "Created and monitoring extension child (" << ext_process->pid()
<< "): " << extension;
return true;
}
DWORD get_fattime(void)
{
// Returns current time packed into a DWORD variable
// return ((DWORD) (2015 - 1980) << 25) // Year 2013
// | ((DWORD) 8 << 21) // Month 7
// | ((DWORD) 2 << 16) // Mday 28
// | ((DWORD) 20 << 11) // Hour 0..24
// | ((DWORD) 30 << 5) // Min 0
// | ((DWORD) 0 >> 1); // Sec 0
return (DWORD)getUnixTime();
}
QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
QueryData results;
// Get the records for this time range.
auto indexes = getIndexes(start, stop);
auto records = getRecords(indexes);
std::string events_key = "data." + dbNamespace();
std::vector<std::string> mapped_records;
for (const auto& record : records) {
if (record.second >= start && (record.second <= stop || stop == 0)) {
mapped_records.push_back(events_key + "." + record.first);
}
}
if (FLAGS_events_optimize && !records.empty()) {
// If records were returned save the ordered-last as the optimization EID.
unsigned long int eidr = 0;
if (safeStrtoul(records.back().first, 10, eidr)) {
optimize_eid_ = static_cast<size_t>(eidr);
auto index_key = "optimize_id." + dbNamespace();
setDatabaseValue(kEvents, index_key, records.back().first);
}
}
// Select mapped_records using event_ids as keys.
std::string data_value;
for (const auto& record : mapped_records) {
Row r;
auto status = getDatabaseValue(kEvents, record, data_value);
if (data_value.length() == 0) {
// There is no record here, interesting error case.
continue;
}
status = deserializeRowJSON(data_value, r);
data_value.clear();
if (status.ok()) {
results.push_back(std::move(r));
}
}
if (getEventsExpiry() > 0) {
// Set the expire time to NOW - "configured lifetime".
// Index retrieval will apply the constraints checking and auto-expire.
expire_time_ = getUnixTime() - getEventsExpiry();
}
return results;
}
TEST_F(LoggerTests, test_logger_log_status) {
std::string warning = "Logger test is generating a warning status (2)";
auto now = getUnixTime();
// This will be printed to stdout.
LOG(WARNING) << warning;
// The second warning status will be sent to the logger plugin.
EXPECT_EQ(1U, LoggerTests::statuses_logged);
EXPECT_EQ(O_WARNING, LoggerTests::last_status.severity);
EXPECT_GT(LoggerTests::last_status.line, 0U);
EXPECT_EQ(warning, LoggerTests::last_status.message);
EXPECT_GE(now, LoggerTests::last_status.time);
EXPECT_EQ(getHostIdentifier(), LoggerTests::last_status.identifier);
}
TEST_F(EventsDatabaseTests, test_gentable) {
auto sub = std::make_shared<DBFakeEventSubscriber>();
auto status = sub->testAdd(1);
status = sub->testAdd(2);
status = sub->testAdd(11);
status = sub->testAdd(61);
status = sub->testAdd((1 * 3600) + 1);
status = sub->testAdd((2 * 3600) + 1);
ASSERT_EQ(0U, sub->optimize_time_);
ASSERT_EQ(0U, sub->expire_time_);
ASSERT_EQ(0U, sub->min_expiration_);
auto t = getUnixTime();
sub->testAdd(t - 1);
sub->testAdd(t);
sub->testAdd(t + 1);
// Test the expire workflow by creating a short expiration time.
sub->setEventsExpiry(10);
std::vector<std::string> keys;
scanDatabaseKeys("events", keys);
// 9 data records, 1 eid counter, 3 indexes, 15 index records.
// Depending on the moment, an additional 3 indexes may be introduced.
EXPECT_LE(16U, keys.size());
// Perform a "select" equivalent.
auto results = genRows(sub.get());
// Expect all non-expired results: 11, +
EXPECT_EQ(9U, results.size());
// The expiration time is now - events_expiry +/ 60.
EXPECT_LT(t - (sub->getEventsExpiry() * 2), sub->expire_time_ + 60);
EXPECT_GT(t, sub->expire_time_);
// The optimize time will not be changed.
ASSERT_EQ(0U, sub->optimize_time_);
results = genRows(sub.get());
EXPECT_EQ(3U, results.size());
results = genRows(sub.get());
EXPECT_EQ(3U, results.size());
keys.clear();
scanDatabaseKeys("events", keys);
EXPECT_LE(6U, keys.size());
}
void Config::recordQueryPerformance(const std::string& name,
size_t delay,
size_t size,
const Row& r0,
const Row& r1) {
RecursiveLock lock(config_performance_mutex_);
if (performance_.count(name) == 0) {
performance_[name] = QueryPerformance();
}
// Grab access to the non-const schedule item.
auto& query = performance_.at(name);
BIGINT_LITERAL diff = 0;
if (!r1.at("user_time").empty() && !r0.at("user_time").empty()) {
diff = AS_LITERAL(BIGINT_LITERAL, r1.at("user_time")) -
AS_LITERAL(BIGINT_LITERAL, r0.at("user_time"));
if (diff > 0) {
query.user_time += diff;
}
}
if (!r1.at("system_time").empty() && !r0.at("system_time").empty()) {
diff = AS_LITERAL(BIGINT_LITERAL, r1.at("system_time")) -
AS_LITERAL(BIGINT_LITERAL, r0.at("system_time"));
if (diff > 0) {
query.system_time += diff;
}
}
if (!r1.at("resident_size").empty() && !r0.at("resident_size").empty()) {
diff = AS_LITERAL(BIGINT_LITERAL, r1.at("resident_size")) -
AS_LITERAL(BIGINT_LITERAL, r0.at("resident_size"));
if (diff > 0) {
// Memory is stored as an average of RSS changes between query executions.
query.average_memory = (query.average_memory * query.executions) + diff;
query.average_memory = (query.average_memory / (query.executions + 1));
}
}
query.wall_time += delay;
query.output_size += size;
query.executions += 1;
query.last_executed = getUnixTime();
// Clear the executing query (remove the dirty bit).
setDatabaseValue(kPersistentSettings, kExecutingQuery, "");
}