Esempio n. 1
0
Status INotifyEventPublisher::restartMonitoring() {
  if (last_restart_ != 0 && getUnixTime() - last_restart_ < 10) {
    return Status(1, "Overflow");
  }

  last_restart_ = getUnixTime();
  VLOG(1) << "inotify was overflown, attempting to restart handle";

  // Create a copy of the descriptors, then remove each.
  auto descriptors = descriptors_;
  for (const auto& desc : descriptors) {
    removeMonitor(desc, true);
  }

  {
    // Then remove all path/descriptor mappings.
    WriteLock lock(mutex_);
    path_descriptors_.clear();
    descriptor_paths_.clear();
  }

  // Reconfigure ourself, the subscribers will not reconfigure.
  configure();
  return Status(0, "OK");
}
Esempio n. 2
0
TEST_F(PlistBenchmark, bench_parse_plist_content) {
  // using LOG(ERROR) as a quick hack so that gtest displays the log line even
  // when the test passes
  LOG(ERROR) << "Starting: " << getAsciiTime();
  LOG(ERROR) << "Performing " << FLAGS_iterations << " iterations";
  int time = getUnixTime();
  for (int i = 0; i < FLAGS_iterations; ++i) {
    std::string content = getPlistContent();
    pt::ptree tree;
    auto s = parsePlistContent(content, tree);
    EXPECT_TRUE(s.ok());
    EXPECT_EQ(s.toString(), "OK");
    EXPECT_EQ(tree.get<bool>("Disabled"), true);
    EXPECT_THROW(tree.get<bool>("foobar"), pt::ptree_bad_path);
    EXPECT_EQ(tree.get<std::string>("Label"), "com.apple.FileSyncAgent.sshd");
    std::vector<std::string> program_arguments = {
        "/System/Library/CoreServices/FileSyncAgent.app/Contents/Resources/"
        "FileSyncAgent_sshd-keygen-wrapper",
        "-i",
        "-f",
        "/System/Library/CoreServices/FileSyncAgent.app/Contents/Resources/"
        "FileSyncAgent_sshd_config",
    };
    pt::ptree program_arguments_tree = tree.get_child("ProgramArguments");
    std::vector<std::string> program_arguments_parsed;
    for (const auto& argument : program_arguments_tree) {
      program_arguments_parsed.push_back(argument.second.get<std::string>(""));
    }
    EXPECT_EQ(program_arguments_parsed, program_arguments);
  }
  LOG(ERROR) << "Ending:   " << getAsciiTime();
  LOG(ERROR) << "Benchmark executed in " << (getUnixTime() - time)
             << " seconds";
}
Esempio n. 3
0
void WatcherRunner::createWorker() {
  {
    WatcherLocker locker;
    if (Watcher::getState(Watcher::getWorker()).last_respawn_time >
        getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) {
      LOG(WARNING) << "osqueryd worker respawning too quickly: "
                   << Watcher::workerRestartCount() << " times";
      Watcher::workerRestarted();
      // The configured automatic delay.
      size_t delay = getWorkerLimit(RESPAWN_DELAY) * 1000;
      // Exponential back off for quickly-respawning clients.
      delay += pow(2, Watcher::workerRestartCount()) * 1000;
      pauseMilli(delay);
    }
  }

  // Get the path of the current process.
  auto qd =
      SQL::selectAllFrom("processes",
                         "pid",
                         EQUALS,
                         INTEGER(PlatformProcess::getCurrentProcess()->pid()));
  if (qd.size() != 1 || qd[0].count("path") == 0 || qd[0]["path"].size() == 0) {
    LOG(ERROR) << "osquery watcher cannot determine process path for worker";
    Initializer::requestShutdown(EXIT_FAILURE);
    return;
  }

  // Set an environment signaling to potential plugin-dependent workers to wait
  // for extensions to broadcast.
  if (Watcher::hasManagedExtensions()) {
    setEnvVar("OSQUERY_EXTENSIONS", "true");
  }

  // Get the complete path of the osquery process binary.
  boost::system::error_code ec;
  auto exec_path = fs::system_complete(fs::path(qd[0]["path"]), ec);
  if (!safePermissions(
          exec_path.parent_path().string(), exec_path.string(), true)) {
    // osqueryd binary has become unsafe.
    LOG(ERROR) << RLOG(1382)
               << "osqueryd has unsafe permissions: " << exec_path.string();
    Initializer::requestShutdown(EXIT_FAILURE);
    return;
  }

  auto worker = PlatformProcess::launchWorker(exec_path.string(), argc_, argv_);
  if (worker == nullptr) {
    // Unrecoverable error, cannot create a worker process.
    LOG(ERROR) << "osqueryd could not create a worker process";
    Initializer::shutdown(EXIT_FAILURE);
    return;
  }

  Watcher::setWorker(worker);
  Watcher::resetWorkerCounters(getUnixTime());
  VLOG(1) << "osqueryd watcher (" << PlatformProcess::getCurrentProcess()->pid()
          << ") executing worker (" << worker->pid() << ")";
}
Esempio n. 4
0
void FileLib::getFileStatus(
		const WIN32_FIND_DATAW &data, FileStatus *status) {
	status->accessTime_ = getUnixTime(data.ftLastAccessTime);
	status->attributes_ = data.dwFileAttributes;
	status->creationTime_ = getUnixTime(data.ftCreationTime);
	status->hardLinkCount_ = 1;
	status->modificationTime_ = getUnixTime(data.ftLastWriteTime);
	ULARGE_INTEGER size;
	size.HighPart = data.nFileSizeHigh;
	size.LowPart = data.nFileSizeLow;
	status->size_ = size.QuadPart;
}
Esempio n. 5
0
void Config::purge() {
  // The first use of purge is removing expired query results.
  std::vector<std::string> saved_queries;
  scanDatabaseKeys(kQueries, saved_queries);

  const auto& schedule = this->schedule_;
  auto queryExists = [&schedule](const std::string& query_name) {
    for (const auto& pack : schedule->packs_) {
      const auto& pack_queries = pack->getSchedule();
      if (pack_queries.count(query_name)) {
        return true;
      }
    }
    return false;
  };

  RecursiveLock lock(config_schedule_mutex_);
  // Iterate over each result set in the database.
  for (const auto& saved_query : saved_queries) {
    if (queryExists(saved_query)) {
      continue;
    }

    std::string content;
    getDatabaseValue(kPersistentSettings, "timestamp." + saved_query, content);
    if (content.empty()) {
      // No timestamp is set for this query, perhaps this is the first time
      // query results expiration is applied.
      setDatabaseValue(kPersistentSettings,
                       "timestamp." + saved_query,
                       std::to_string(getUnixTime()));
      continue;
    }

    // Parse the timestamp and compare.
    size_t last_executed = 0;
    try {
      last_executed = boost::lexical_cast<size_t>(content);
    } catch (const boost::bad_lexical_cast& /* e */) {
      // Erase the timestamp as is it potentially corrupt.
      deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query);
      continue;
    }

    if (last_executed < getUnixTime() - 592200) {
      // Query has not run in the last week, expire results and interval.
      deleteDatabaseValue(kQueries, saved_query);
      deleteDatabaseValue(kPersistentSettings, "interval." + saved_query);
      deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query);
      VLOG(1) << "Expiring results for scheduled query: " << saved_query;
    }
  }
}
Esempio n. 6
0
void FileLib::getFileStatus(
		const BY_HANDLE_FILE_INFORMATION &info, FileStatus *status) {
	status->accessTime_ = getUnixTime(info.ftLastAccessTime);
	status->attributes_ = info.dwFileAttributes;
	status->creationTime_ = getUnixTime(info.ftCreationTime);
	status->hardLinkCount_ = info.nNumberOfLinks;
	status->modificationTime_ = getUnixTime(info.ftLastWriteTime);
	ULARGE_INTEGER size;
	size.HighPart = info.nFileSizeHigh;
	size.LowPart = info.nFileSizeLow;
	status->size_ = size.QuadPart;
}
TEST_F(EventsDatabaseTests, test_gentable) {
  auto sub = std::make_shared<DBFakeEventSubscriber>();
  // Lie about the tool type to enable optimizations.
  auto default_type = kToolType;
  kToolType = OSQUERY_TOOL_DAEMON;
  ASSERT_EQ(sub->optimize_time_, 0U);
  ASSERT_EQ(sub->expire_time_, 0U);

  sub->testAdd(getUnixTime() - 1);
  sub->testAdd(getUnixTime());
  sub->testAdd(getUnixTime() + 1);

  // Test the expire workflow by creating a short expiration time.
  FLAGS_events_expiry = 10;

  std::vector<std::string> keys;
  scanDatabaseKeys("events", keys);
  EXPECT_GT(keys.size(), 10U);

  // Perform a "select" equivalent.
  QueryContext context;
  auto results = sub->genTable(context);

  // Expect all non-expired results: 11, +
  EXPECT_EQ(results.size(), 9U);
  // The expiration time is now - events_expiry.
  EXPECT_GT(sub->expire_time_, getUnixTime() - (FLAGS_events_expiry * 2));
  EXPECT_LT(sub->expire_time_, getUnixTime());
  // The optimize time will be changed too.
  ASSERT_GT(sub->optimize_time_, 0U);
  // Restore the tool type.
  kToolType = default_type;

  results = sub->genTable(context);
  EXPECT_EQ(results.size(), 3U);

  results = sub->genTable(context);
  EXPECT_EQ(results.size(), 3U);

  // The optimize time should have been written to the database.
  // It should be the same as the current (relative) optimize time.
  std::string content;
  getDatabaseValue("events", "optimize.DBFakePublisher.DBFakeSubscriber",
                   content);
  EXPECT_EQ(std::to_string(sub->optimize_time_), content);

  keys.clear();
  scanDatabaseKeys("events", keys);
  EXPECT_LT(keys.size(), 30U);
}
Esempio n. 8
0
Status INotifyEventPublisher::restartMonitoring(){
  if (last_restart_ != 0 && getUnixTime() - last_restart_ < 10) {
    return Status(1, "Overflow");
  }
  last_restart_ = getUnixTime();
  VLOG(1) << "Got an overflow, trying to restart...";
  for(const auto& desc : descriptors_){
    removeMonitor(desc, 1);
  }
  path_descriptors_.clear();
  descriptor_paths_.clear();
  configure();
  return Status(0, "OK");
}
Esempio n. 9
0
void FileLib::getFileStatus(const struct stat &stBuf, FileStatus *status) {
	status->accessTime_ = getUnixTime(stBuf.st_atime);
	status->blockCount_ = stBuf.st_blocks;
	status->blockSize_ = stBuf.st_blksize;
	status->changeTime_ = getUnixTime(stBuf.st_ctime);
	status->device_ = stBuf.st_dev;
	status->gid_ = stBuf.st_gid;
	status->hardLinkCount_ = stBuf.st_nlink;
	status->iNode_ = stBuf.st_ino;
	status->mode_ = stBuf.st_mode;
	status->modificationTime_ = getUnixTime(stBuf.st_mtime);
	status->rDevice_ = stBuf.st_rdev;
	status->size_ = stBuf.st_size;
	status->uid_ = stBuf.st_uid;
}
Esempio n. 10
0
QueryData EventSubscriberPlugin::genTable(QueryContext& context) {
  // Stop is an unsigned (-1), our end of time equivalent.
  EventTime start = 0, stop = -1;
  if (context.constraints["time"].getAll().size() > 0) {
    // Use the 'time' constraint to optimize backing-store lookups.
    for (const auto& constraint : context.constraints["time"].getAll()) {
      EventTime expr = timeFromRecord(constraint.expr);
      if (constraint.op == EQUALS) {
        stop = start = expr;
        break;
      } else if (constraint.op == GREATER_THAN) {
        start = std::max(start, expr + 1);
      } else if (constraint.op == GREATER_THAN_OR_EQUALS) {
        start = std::max(start, expr);
      } else if (constraint.op == LESS_THAN) {
        stop = std::min(stop, expr - 1);
      } else if (constraint.op == LESS_THAN_OR_EQUALS) {
        stop = std::min(stop, expr);
      }
    }
  } else if (kToolType == OSQUERY_TOOL_DAEMON && FLAGS_events_optimize) {
    // If the daemon is querying a subscriber without a 'time' constraint and
    // allows optimization, only emit events since the last query.
    start = optimize_time_;
    optimize_time_ = getUnixTime() - 1;

    // Store the optimize time such that it can be restored if the daemon is
    // restarted.
    auto index_key = "optimize." + dbNamespace();
    setDatabaseValue(kEvents, index_key, std::to_string(optimize_time_));
  }

  return get(start, stop);
}
Esempio n. 11
0
void EventPublisherPlugin::fire(const EventContextRef& ec, EventTime time) {
  EventContextID ec_id;

  if (isEnding()) {
    // Cannot emit/fire while ending
    return;
  }

  {
    boost::lock_guard<boost::mutex> lock(ec_id_lock_);
    ec_id = next_ec_id_++;
  }

  // Fill in EventContext ID and time if needed.
  if (ec != nullptr) {
    ec->id = ec_id;
    if (ec->time == 0) {
      if (time == 0) {
        time = getUnixTime();
      }
      // Todo: add a check to assure normalized (seconds) time.
      ec->time = time;
    }
  }

  for (const auto& subscription : subscriptions_) {
    auto es = EventFactory::getEventSubscriber(subscription->subscriber_name);
    if (es->state() == SUBSCRIBER_RUNNING) {
      fireCallback(subscription, ec);
    }
  }
}
Esempio n. 12
0
Status carvePaths(const std::set<std::string>& paths) {
  auto guid = generateNewUUID();

  pt::ptree tree;
  tree.put("carve_guid", guid);
  tree.put("time", getUnixTime());
  tree.put("status", "STARTING");
  tree.put("sha256", "");
  tree.put("size", -1);

  if (paths.size() > 1) {
    tree.put("path", boost::algorithm::join(paths, ","));
  } else {
    tree.put("path", *(paths.begin()));
  }

  std::ostringstream os;
  pt::write_json(os, tree, false);
  auto s = setDatabaseValue(kCarveDbDomain, kCarverDBPrefix + guid, os.str());
  if (!s.ok()) {
    return s;
  } else {
    auto requestId = Distributed::getCurrentRequestId();
    Dispatcher::addService(std::make_shared<Carver>(paths, guid, requestId));
  }
  return s;
}
Esempio n. 13
0
Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
  std::shared_ptr<DBHandle> db = nullptr;
  try {
    db = DBHandle::getInstance();
  } catch (const std::runtime_error& e) {
    return Status(1, e.what());
  }

  // Get and increment the EID for this module.
  EventID eid = getEventID();
  // Without encouraging a missing event time, do not support a 0-time.
  r["time"] = std::to_string((event_time == 0) ? getUnixTime() : event_time);

  // Serialize and store the row data, for query-time retrieval.
  std::string data;
  auto status = serializeRowJSON(r, data);
  if (!status.ok()) {
    return status;
  }

  // Store the event data.
  std::string event_key = "data." + dbNamespace() + "." + eid;
  status = db->Put(kEvents, event_key, data);
  // Record the event in the indexing bins, using the index time.
  recordEvent(eid, event_time);
  return status;
}
Esempio n. 14
0
QueryData EventSubscriberPlugin::genTable(QueryContext& context) {
  EventTime start = 0, stop = -1;
  if (context.constraints["time"].getAll().size() > 0) {
    // Use the 'time' constraint to optimize backing-store lookups.
    for (const auto& constraint : context.constraints["time"].getAll()) {
      EventTime expr = 0;
      try {
        expr = boost::lexical_cast<EventTime>(constraint.expr);
      } catch (const boost::bad_lexical_cast& e) {
        expr = 0;
      }
      if (constraint.op == EQUALS) {
        stop = start = expr;
        break;
      } else if (constraint.op == GREATER_THAN) {
        start = std::max(start, expr + 1);
      } else if (constraint.op == GREATER_THAN_OR_EQUALS) {
        start = std::max(start, expr);
      } else if (constraint.op == LESS_THAN) {
        stop = std::min(stop, expr - 1);
      } else if (constraint.op == LESS_THAN_OR_EQUALS) {
        stop = std::min(stop, expr);
      }
    }
  } else if (kToolType == OSQUERY_TOOL_DAEMON && FLAGS_events_optimize) {
    // If the daemon is querying a subscriber without a 'time' constraint and
    // allows optimization, only emit events since the last query.
    start = optimize_time_;
    optimize_time_ = getUnixTime() - 1;
  }

  return get(start, stop);
}
Esempio n. 15
0
Status INotifyEventPublisher::restartMonitoring() {
    if (last_restart_ != 0 && getUnixTime() - last_restart_ < 10) {
        return Status(1, "Overflow");
    }

    last_restart_ = getUnixTime();
    VLOG(1) << "inotify was overflown, attempting to restart handle";
    for (const auto& desc : descriptors_) {
        removeMonitor(desc, true);
    }

    path_descriptors_.clear();
    descriptor_paths_.clear();
    configure();
    return Status(0, "OK");
}
Esempio n. 16
0
void Config::scheduledQueries(
    std::function<void(const std::string& name, const ScheduledQuery& query)>
        predicate) {
  RecursiveLock lock(config_schedule_mutex_);
  for (const PackRef& pack : *schedule_) {
    for (const auto& it : pack->getSchedule()) {
      std::string name = it.first;
      // The query name may be synthetic.
      if (pack->getName() != "main" && pack->getName() != "legacy_main") {
        name = "pack" + FLAGS_pack_delimiter + pack->getName() +
               FLAGS_pack_delimiter + it.first;
      }
      // They query may have failed and been added to the schedule's blacklist.
      if (schedule_->blacklist_.count(name) > 0) {
        auto blacklisted_query = schedule_->blacklist_.find(name);
        if (getUnixTime() > blacklisted_query->second) {
          // The blacklisted query passed the expiration time (remove).
          schedule_->blacklist_.erase(blacklisted_query);
          saveScheduleBlacklist(schedule_->blacklist_);
        } else {
          // The query is still blacklisted.
          continue;
        }
      }
      // Call the predicate.
      predicate(name, it.second);
    }
  }
}
Esempio n. 17
0
std::string BufferedLogForwarder::genIndex(bool results, size_t time) {
  if (time == 0) {
    time = getUnixTime();
  }
  return genIndexPrefix(results) + std::to_string(time) + "_" +
         std::to_string(++log_index_);
}
Esempio n. 18
0
Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
  std::shared_ptr<DBHandle> db = nullptr;
  try {
    db = DBHandle::getInstance();
  } catch (const std::runtime_error& e) {
    return Status(1, e.what());
  }

  // Get and increment the EID for this module.
  EventID eid = getEventID();
  // Without encouraging a missing event time, do not support a 0-time.
  r["time"] = std::to_string((event_time == 0) ? getUnixTime() : event_time);

  // Serialize and store the row data, for query-time retrieval.
  std::string data;
  auto status = serializeRowJSON(r, data);
  if (!status.ok()) {
    return status;
  }

  // Use the last EventID and a checkpoint bucket size to periodically apply
  // buffer eviction. Eviction occurs if the total count exceeds events_max.
  if (last_eid_ % EVENTS_CHECKPOINT == 0) {
    expireCheck();
  }

  // Store the event data.
  std::string event_key = "data." + dbNamespace() + "." + eid;
  status = db->Put(kEvents, event_key, data);
  // Record the event in the indexing bins, using the index time.
  recordEvent(eid, event_time);
  return status;
}
Esempio n. 19
0
Status ProcessEventSubscriber::Callback(const ECRef& ec, const SCRef& sc) {
  // Check and set the valid state change.
  // If this is an unacceptable change reset the state and clear row data.
  if (ec->fields.count("success") && ec->fields.at("success") == "no") {
    return Status(0, "OK");
  }

  if (!validAuditState(ec->type, state_).ok()) {
    state_ = STATE_SYSCALL;
    Row().swap(row_);
    return Status(0, "OK");
  }

  // Fill in row fields based on the event state.
  updateAuditRow(ec, row_);

  // Only add the event if finished (aka a PATH event was emitted).
  if (state_ == STATE_SYSCALL) {
    // If the EXECVE state was not used, decode the cmdline value.
    if (row_.at("cmdline_size").size() == 0) {
      // This allows at most 1 decode call per potentially-encoded item.
      row_["cmdline"] = decodeAuditValue(row_.at("cmdline"));
      row_["cmdline_size"] = "1";
    }

    add(row_, getUnixTime());
    Row().swap(row_);
  }

  return Status(0, "OK");
}
Esempio n. 20
0
TEST_F(ConfigTests, test_schedule_blacklist) {
  auto current_time = getUnixTime();
  std::map<std::string, size_t> blacklist;
  saveScheduleBlacklist(blacklist);
  restoreScheduleBlacklist(blacklist);
  EXPECT_EQ(blacklist.size(), 0U);

  // Create some entries.
  blacklist["test_1"] = current_time * 2;
  blacklist["test_2"] = current_time * 3;
  saveScheduleBlacklist(blacklist);
  blacklist.clear();
  restoreScheduleBlacklist(blacklist);
  ASSERT_EQ(blacklist.count("test_1"), 1U);
  ASSERT_EQ(blacklist.count("test_2"), 1U);
  EXPECT_EQ(blacklist.at("test_1"), current_time * 2);
  EXPECT_EQ(blacklist.at("test_2"), current_time * 3);

  // Now save an expired query.
  blacklist["test_1"] = 1;
  saveScheduleBlacklist(blacklist);
  blacklist.clear();

  // When restoring, the values below the current time will not be included.
  restoreScheduleBlacklist(blacklist);
  EXPECT_EQ(blacklist.size(), 1U);
}
Esempio n. 21
0
Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
  // Get and increment the EID for this module.
  EventID eid = getEventID();
  // Without encouraging a missing event time, do not support a 0-time.
  r["time"] = std::to_string((event_time == 0) ? getUnixTime() : event_time);
  // Serialize and store the row data, for query-time retrieval.
  std::string data;
  auto status = serializeRowJSON(r, data);
  if (!status.ok()) {
    return status;
  }
  // Then remove the newline.
  if (data.size() > 0 && data.back() == '\n') {
    data.pop_back();
  }

  // Use the last EventID and a checkpoint bucket size to periodically apply
  // buffer eviction. Eviction occurs if the total count exceeds events_max.
  if (last_eid_ % EVENTS_CHECKPOINT == 0) {
    expireCheck();
  }

  // Store the event data.
  std::string event_key = "data." + dbNamespace() + "." + eid;
  status = setDatabaseValue(kEvents, event_key, data);
  // Record the event in the indexing bins, using the index time.
  recordEvent(eid, event_time);
  return status;
}
Esempio n. 22
0
void EventPublisherPlugin::fire(const EventContextRef& ec, EventTime time) {
  if (isEnding()) {
    // Cannot emit/fire while ending
    return;
  }

  EventContextID ec_id = 0;
  {
    WriteLock lock(ec_id_lock_);
    ec_id = next_ec_id_++;
  }

  // Fill in EventContext ID and time if needed.
  if (ec != nullptr) {
    ec->id = ec_id;
    if (ec->time == 0) {
      if (time == 0) {
        time = getUnixTime();
      }
      ec->time = time;
    }
  }

  for (const auto& subscription : subscriptions_) {
    auto es = EventFactory::getEventSubscriber(subscription->subscriber_name);
    if (es != nullptr && es->state() == SUBSCRIBER_RUNNING) {
      es->event_count_++;
      fireCallback(subscription, ec);
    }
  }
}
Esempio n. 23
0
bool Pack::checkDiscovery() {
  stats_.total++;
  auto current = getUnixTime();
  if ((current - discovery_cache_.first) < FLAGS_pack_refresh_interval) {
    stats_.hits++;
    return discovery_cache_.second;
  }

  stats_.misses++;
  discovery_cache_.first = current;
  discovery_cache_.second = true;
  for (const auto& q : discovery_queries_) {
    auto sql = SQL(q);
    if (!sql.ok()) {
      LOG(WARNING) << "Discovery query failed (" << q
                   << "): " << sql.getMessageString();
      discovery_cache_.second = false;
      break;
    }
    if (sql.rows().size() == 0) {
      discovery_cache_.second = false;
      break;
    }
  }
  return discovery_cache_.second;
}
Esempio n. 24
0
void Config::recordQueryStart(const std::string& name) {
  // There should only ever be a single executing query in the schedule.
  setDatabaseValue(kPersistentSettings, kExecutingQuery, name);
  // Store the time this query name last executed for later results eviction.
  // When configuration updates occur the previous schedule is searched for
  // 'stale' query names, aka those that have week-old or longer last execute
  // timestamps. Offending queries have their database results purged.
  setDatabaseValue(
      kPersistentSettings, "timestamp." + name, std::to_string(getUnixTime()));
}
Esempio n. 25
0
bool WatcherRunner::createExtension(const std::string& extension) {
  {
    WatcherLocker locker;
    if (Watcher::getState(extension).last_respawn_time >
        getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) {
      LOG(WARNING) << "Extension respawning too quickly: " << extension;
      // Unlike a worker, if an extension respawns to quickly we give up.
      return false;
    }
  }

  // Check the path to the previously-discovered extension binary.
  boost::system::error_code ec;
  auto exec_path = fs::system_complete(fs::path(extension), ec);
  if (!safePermissions(
          exec_path.parent_path().string(), exec_path.string(), true)) {
    // Extension binary has become unsafe.
    LOG(WARNING) << RLOG(1382)
                 << "Extension binary has unsafe permissions: " << extension;
    return false;
  }

  auto ext_process =
      PlatformProcess::launchExtension(exec_path.string(),
                                       extension,
                                       Flag::getValue("extensions_socket"),
                                       Flag::getValue("extensions_timeout"),
                                       Flag::getValue("extensions_interval"),
                                       Flag::getValue("verbose"));
  if (ext_process == nullptr) {
    // Unrecoverable error, cannot create an extension process.
    LOG(ERROR) << "Cannot create extension process: " << extension;
    Initializer::shutdown(EXIT_FAILURE);
  }

  Watcher::setExtension(extension, ext_process);
  Watcher::resetExtensionCounters(extension, getUnixTime());
  VLOG(1) << "Created and monitoring extension child (" << ext_process->pid()
          << "): " << extension;

  return true;
}
Esempio n. 26
0
DWORD get_fattime(void)
{
	// Returns current time packed into a DWORD variable
//	return ((DWORD) (2015 - 1980) << 25)	// Year 2013
//	| ((DWORD) 8 << 21)				// Month 7
//			| ((DWORD) 2 << 16)				// Mday 28
//			| ((DWORD) 20 << 11)				// Hour 0..24
//			| ((DWORD) 30 << 5)				// Min 0
//			| ((DWORD) 0 >> 1);				// Sec 0
	return (DWORD)getUnixTime();
}
Esempio n. 27
0
QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
  QueryData results;

  // Get the records for this time range.
  auto indexes = getIndexes(start, stop);
  auto records = getRecords(indexes);

  std::string events_key = "data." + dbNamespace();
  std::vector<std::string> mapped_records;
  for (const auto& record : records) {
    if (record.second >= start && (record.second <= stop || stop == 0)) {
      mapped_records.push_back(events_key + "." + record.first);
    }
  }

  if (FLAGS_events_optimize && !records.empty()) {
    // If records were returned save the ordered-last as the optimization EID.
    unsigned long int eidr = 0;
    if (safeStrtoul(records.back().first, 10, eidr)) {
      optimize_eid_ = static_cast<size_t>(eidr);
      auto index_key = "optimize_id." + dbNamespace();
      setDatabaseValue(kEvents, index_key, records.back().first);
    }
  }

  // Select mapped_records using event_ids as keys.
  std::string data_value;
  for (const auto& record : mapped_records) {
    Row r;
    auto status = getDatabaseValue(kEvents, record, data_value);
    if (data_value.length() == 0) {
      // There is no record here, interesting error case.
      continue;
    }
    status = deserializeRowJSON(data_value, r);
    data_value.clear();
    if (status.ok()) {
      results.push_back(std::move(r));
    }
  }

  if (getEventsExpiry() > 0) {
    // Set the expire time to NOW - "configured lifetime".
    // Index retrieval will apply the constraints checking and auto-expire.
    expire_time_ = getUnixTime() - getEventsExpiry();
  }

  return results;
}
Esempio n. 28
0
TEST_F(LoggerTests, test_logger_log_status) {
  std::string warning = "Logger test is generating a warning status (2)";
  auto now = getUnixTime();
  // This will be printed to stdout.
  LOG(WARNING) << warning;

  // The second warning status will be sent to the logger plugin.
  EXPECT_EQ(1U, LoggerTests::statuses_logged);

  EXPECT_EQ(O_WARNING, LoggerTests::last_status.severity);
  EXPECT_GT(LoggerTests::last_status.line, 0U);
  EXPECT_EQ(warning, LoggerTests::last_status.message);
  EXPECT_GE(now, LoggerTests::last_status.time);
  EXPECT_EQ(getHostIdentifier(), LoggerTests::last_status.identifier);
}
TEST_F(EventsDatabaseTests, test_gentable) {
  auto sub = std::make_shared<DBFakeEventSubscriber>();
  auto status = sub->testAdd(1);
  status = sub->testAdd(2);
  status = sub->testAdd(11);
  status = sub->testAdd(61);
  status = sub->testAdd((1 * 3600) + 1);
  status = sub->testAdd((2 * 3600) + 1);

  ASSERT_EQ(0U, sub->optimize_time_);
  ASSERT_EQ(0U, sub->expire_time_);
  ASSERT_EQ(0U, sub->min_expiration_);

  auto t = getUnixTime();
  sub->testAdd(t - 1);
  sub->testAdd(t);
  sub->testAdd(t + 1);

  // Test the expire workflow by creating a short expiration time.
  sub->setEventsExpiry(10);

  std::vector<std::string> keys;
  scanDatabaseKeys("events", keys);
  // 9 data records, 1 eid counter, 3 indexes, 15 index records.
  // Depending on the moment, an additional 3 indexes may be introduced.
  EXPECT_LE(16U, keys.size());

  // Perform a "select" equivalent.
  auto results = genRows(sub.get());

  // Expect all non-expired results: 11, +
  EXPECT_EQ(9U, results.size());
  // The expiration time is now - events_expiry +/ 60.
  EXPECT_LT(t - (sub->getEventsExpiry() * 2), sub->expire_time_ + 60);
  EXPECT_GT(t, sub->expire_time_);
  // The optimize time will not be changed.
  ASSERT_EQ(0U, sub->optimize_time_);

  results = genRows(sub.get());
  EXPECT_EQ(3U, results.size());

  results = genRows(sub.get());
  EXPECT_EQ(3U, results.size());

  keys.clear();
  scanDatabaseKeys("events", keys);
  EXPECT_LE(6U, keys.size());
}
Esempio n. 30
0
void Config::recordQueryPerformance(const std::string& name,
                                    size_t delay,
                                    size_t size,
                                    const Row& r0,
                                    const Row& r1) {
  RecursiveLock lock(config_performance_mutex_);
  if (performance_.count(name) == 0) {
    performance_[name] = QueryPerformance();
  }

  // Grab access to the non-const schedule item.
  auto& query = performance_.at(name);
  BIGINT_LITERAL diff = 0;
  if (!r1.at("user_time").empty() && !r0.at("user_time").empty()) {
    diff = AS_LITERAL(BIGINT_LITERAL, r1.at("user_time")) -
           AS_LITERAL(BIGINT_LITERAL, r0.at("user_time"));
    if (diff > 0) {
      query.user_time += diff;
    }
  }

  if (!r1.at("system_time").empty() && !r0.at("system_time").empty()) {
    diff = AS_LITERAL(BIGINT_LITERAL, r1.at("system_time")) -
           AS_LITERAL(BIGINT_LITERAL, r0.at("system_time"));
    if (diff > 0) {
      query.system_time += diff;
    }
  }

  if (!r1.at("resident_size").empty() && !r0.at("resident_size").empty()) {
    diff = AS_LITERAL(BIGINT_LITERAL, r1.at("resident_size")) -
           AS_LITERAL(BIGINT_LITERAL, r0.at("resident_size"));
    if (diff > 0) {
      // Memory is stored as an average of RSS changes between query executions.
      query.average_memory = (query.average_memory * query.executions) + diff;
      query.average_memory = (query.average_memory / (query.executions + 1));
    }
  }

  query.wall_time += delay;
  query.output_size += size;
  query.executions += 1;
  query.last_executed = getUnixTime();

  // Clear the executing query (remove the dirty bit).
  setDatabaseValue(kPersistentSettings, kExecutingQuery, "");
}