void EventSubscriberPlugin::expireRecords(const std::string& list_type,
const std::string& index,
bool all) {
auto db = DBHandle::getInstance();
auto record_key = "records." + dbNamespace();
auto data_key = "data." + dbNamespace();
// If the expirations is not removing all records, rewrite the persisting.
std::vector<std::string> persisting_records;
// Request all records within this list-size + bin offset.
auto expired_records = getRecords({list_type + "." + index});
for (const auto& record : expired_records) {
if (all) {
db->Delete(kEvents, data_key + "." + record.first);
} else if (record.second > expire_time_) {
persisting_records.push_back(record.first + ":" +
std::to_string(record.second));
}
}
// Either drop or overwrite the record list.
if (all) {
db->Delete(kEvents, record_key + "." + list_type + "." + index);
} else {
auto new_records = boost::algorithm::join(persisting_records, ",");
db->Put(kEvents, record_key + "." + list_type + "." + index, new_records);
}
}
void EventSubscriberPlugin::expireCheck(bool cleanup) {
auto data_key = "data." + dbNamespace();
auto eid_key = "eid." + dbNamespace();
// Min key will be the last surviving key.
size_t min_key = 0;
{
auto limit = getEventsMax();
std::vector<std::string> keys;
scanDatabaseKeys(kEvents, keys, data_key);
if (keys.size() <= limit) {
return;
}
// There is an overflow of events buffered for this subscriber.
LOG(WARNING) << "Expiring events for subscriber: " << getName()
<< " (limit " << limit << ")";
VLOG(1) << "Subscriber events " << getName() << " exceeded limit " << limit
<< " by: " << keys.size() - limit;
// Inspect the N-FLAGS_events_max -th event's value and expire before the
// time within the content.
std::string last_key;
getDatabaseValue(kEvents, eid_key, last_key);
// The EID is the next-index.
// EID - events_max is the most last-recent event to keep.
min_key = boost::lexical_cast<size_t>(last_key) - getEventsMax();
if (cleanup) {
// Scan each of the keys in keys, if their ID portion is < min_key.
// Nix them, this requires lots of conversions, use with care.
for (const auto& key : keys) {
if (std::stoul(key.substr(key.rfind('.') + 1)) < min_key) {
deleteDatabaseValue(kEvents, key);
}
}
}
}
// Convert the key index into a time using the content.
// The last-recent event is fetched and the corresponding time is used as
// the expiration time for the subscriber.
std::string content;
getDatabaseValue(kEvents, data_key + "." + std::to_string(min_key), content);
// Decode the value into a row structure to extract the time.
Row r;
if (!deserializeRowJSON(content, r) || r.count("time") == 0) {
return;
}
// The last time will become the implicit expiration time.
size_t last_time = boost::lexical_cast<size_t>(r.at("time"));
if (last_time > 0) {
expire_time_ = last_time;
}
// Finally, attempt an index query to trigger expirations.
// In this case the result set is not used.
getIndexes(expire_time_, 0);
}
Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime time) {
Status status;
auto db = DBHandle::getInstance();
std::string time_value = boost::lexical_cast<std::string>(time);
// The record is identified by the event type then module name.
std::string index_key = "indexes." + dbNamespace();
std::string record_key = "records." + dbNamespace();
// The list key includes the list type (bin size) and the list ID (bin).
std::string list_key;
std::string list_id;
for (auto time_list : kEventTimeLists) {
// The list_id is the MOST-Specific key ID, the bin for this list.
// If the event time was 13 and the time_list is 5 seconds, lid = 2.
list_id = boost::lexical_cast<std::string>(time / time_list);
// The list name identifies the 'type' of list.
list_key = boost::lexical_cast<std::string>(time_list);
// list_key = list_key + "." + list_id;
{
boost::lock_guard<boost::mutex> lock(event_record_lock_);
// Append the record (eid, unix_time) to the list bin.
std::string record_value;
status = db->Get(
kEvents, record_key + "." + list_key + "." + list_id, record_value);
if (record_value.length() == 0) {
// This is a new list_id for list_key, append the ID to the indirect
// lookup for this list_key.
std::string index_value;
status = db->Get(kEvents, index_key + "." + list_key, index_value);
if (index_value.length() == 0) {
// A new index.
index_value = list_id;
} else {
index_value += "," + list_id;
}
status = db->Put(kEvents, index_key + "." + list_key, index_value);
record_value = eid + ":" + time_value;
} else {
// Tokenize a record using ',' and the EID/time using ':'.
record_value += "," + eid + ":" + time_value;
}
status = db->Put(
kEvents, record_key + "." + list_key + "." + list_id, record_value);
if (!status.ok()) {
LOG(ERROR) << "Could not put Event Record key: " << record_key << "."
<< list_key << "." << list_id;
}
}
}
return Status(0, "OK");
}
QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
QueryData results;
// Get the records for this time range.
auto indexes = getIndexes(start, stop);
auto records = getRecords(indexes);
std::string events_key = "data." + dbNamespace();
std::vector<std::string> mapped_records;
for (const auto& record : records) {
if (record.second >= start && (record.second <= stop || stop == 0)) {
mapped_records.push_back(events_key + "." + record.first);
}
}
if (FLAGS_events_optimize && !records.empty()) {
// If records were returned save the ordered-last as the optimization EID.
unsigned long int eidr = 0;
if (safeStrtoul(records.back().first, 10, eidr)) {
optimize_eid_ = static_cast<size_t>(eidr);
auto index_key = "optimize_id." + dbNamespace();
setDatabaseValue(kEvents, index_key, records.back().first);
}
}
// Select mapped_records using event_ids as keys.
std::string data_value;
for (const auto& record : mapped_records) {
Row r;
auto status = getDatabaseValue(kEvents, record, data_value);
if (data_value.length() == 0) {
// There is no record here, interesting error case.
continue;
}
status = deserializeRowJSON(data_value, r);
data_value.clear();
if (status.ok()) {
results.push_back(std::move(r));
}
}
if (getEventsExpiry() > 0) {
// Set the expire time to NOW - "configured lifetime".
// Index retrieval will apply the constraints checking and auto-expire.
expire_time_ = getUnixTime() - getEventsExpiry();
}
return results;
}
TEST_F(EventsDatabaseTests, test_expire_check) {
auto sub = std::make_shared<DBFakeEventSubscriber>();
// Set the max number of buffered events to something reasonably small.
FLAGS_events_max = 10;
auto t = 10000;
// We are still at the mercy of the opaque EVENTS_CHECKPOINT define.
for (size_t x = 0; x < 3; x++) {
size_t num_events = 256 * x;
for (size_t i = 0; i < num_events; i++) {
sub->testAdd(t++);
}
// Since events tests are dependent, expect 257 + 3 events.
QueryContext context;
auto results = sub->genTable(context);
if (x == 0) {
// The first iteration is dependent on previous test state.
continue;
}
// The number of events should remain constant.
// In practice there may be an event still in the write queue.
EXPECT_LT(results.size(), 60U);
}
// Try again, this time with a scan
for (size_t k = 0; k < 3; k++) {
for (size_t x = 0; x < 3; x++) {
size_t num_events = 256 * x;
for (size_t i = 0; i < num_events; i++) {
sub->testAdd(t++);
}
// Records hold the event_id + time indexes.
// Data hosts the event_id + JSON content.
auto record_key = "records." + sub->dbNamespace();
auto data_key = "data." + sub->dbNamespace();
std::vector<std::string> records, datas;
scanDatabaseKeys(kEvents, records, record_key);
scanDatabaseKeys(kEvents, datas, data_key);
EXPECT_LT(records.size(), 20U);
EXPECT_LT(datas.size(), 60U);
}
}
}
std::vector<EventRecord> EventSubscriberPlugin::getRecords(
const std::set<std::string>& indexes) {
auto record_key = "records." + dbNamespace();
std::vector<EventRecord> records;
for (const auto& index : indexes) {
std::vector<std::string> bin_records;
{
std::string record_value;
getDatabaseValue(kEvents, record_key + "." + index, record_value);
if (record_value.empty()) {
// There are actually no events in this bin, interesting error case.
continue;
}
// Each list is tokenized into a record=event_id:time.
boost::split(bin_records, record_value, boost::is_any_of(",:"));
}
auto bin_it = bin_records.begin();
// Iterate over every 2 items: EID:TIME.
for (; bin_it != bin_records.end(); bin_it++) {
const auto& eid = *bin_it;
EventTime time = timeFromRecord(*(++bin_it));
records.push_back(std::make_pair(eid, time));
}
}
return records;
}
Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
std::shared_ptr<DBHandle> db = nullptr;
try {
db = DBHandle::getInstance();
} catch (const std::runtime_error& e) {
return Status(1, e.what());
}
// Get and increment the EID for this module.
EventID eid = getEventID();
// Without encouraging a missing event time, do not support a 0-time.
r["time"] = std::to_string((event_time == 0) ? getUnixTime() : event_time);
// Serialize and store the row data, for query-time retrieval.
std::string data;
auto status = serializeRowJSON(r, data);
if (!status.ok()) {
return status;
}
// Store the event data.
std::string event_key = "data." + dbNamespace() + "." + eid;
status = db->Put(kEvents, event_key, data);
// Record the event in the indexing bins, using the index time.
recordEvent(eid, event_time);
return status;
}
QueryData EventSubscriberPlugin::genTable(QueryContext& context) {
// Stop is an unsigned (-1), our end of time equivalent.
EventTime start = 0, stop = -1;
if (context.constraints["time"].getAll().size() > 0) {
// Use the 'time' constraint to optimize backing-store lookups.
for (const auto& constraint : context.constraints["time"].getAll()) {
EventTime expr = timeFromRecord(constraint.expr);
if (constraint.op == EQUALS) {
stop = start = expr;
break;
} else if (constraint.op == GREATER_THAN) {
start = std::max(start, expr + 1);
} else if (constraint.op == GREATER_THAN_OR_EQUALS) {
start = std::max(start, expr);
} else if (constraint.op == LESS_THAN) {
stop = std::min(stop, expr - 1);
} else if (constraint.op == LESS_THAN_OR_EQUALS) {
stop = std::min(stop, expr);
}
}
} else if (kToolType == OSQUERY_TOOL_DAEMON && FLAGS_events_optimize) {
// If the daemon is querying a subscriber without a 'time' constraint and
// allows optimization, only emit events since the last query.
start = optimize_time_;
optimize_time_ = getUnixTime() - 1;
// Store the optimize time such that it can be restored if the daemon is
// restarted.
auto index_key = "optimize." + dbNamespace();
setDatabaseValue(kEvents, index_key, std::to_string(optimize_time_));
}
return get(start, stop);
}
EventID EventSubscriberPlugin::getEventID() {
Status status;
auto db = DBHandle::getInstance();
// First get an event ID from the meta key.
std::string eid_key = "eid." + dbNamespace();
std::string last_eid_value;
std::string eid_value;
{
boost::lock_guard<boost::mutex> lock(event_id_lock_);
status = db->Get(kEvents, eid_key, last_eid_value);
if (!status.ok()) {
last_eid_value = "0";
}
size_t eid = boost::lexical_cast<size_t>(last_eid_value) + 1;
eid_value = boost::lexical_cast<std::string>(eid);
status = db->Put(kEvents, eid_key, eid_value);
}
if (!status.ok()) {
return "0";
}
return eid_value;
}
std::vector<EventRecord> EventSubscriberPlugin::getRecords(
const std::set<std::string>& indexes) {
auto db = DBHandle::getInstance();
auto record_key = "records." + dbNamespace();
std::vector<EventRecord> records;
for (const auto& index : indexes) {
std::string record_value;
if (!db->Get(kEvents, record_key + "." + index, record_value).ok()) {
return records;
}
if (record_value.length() == 0) {
// There are actually no events in this bin, interesting error case.
continue;
}
// Each list is tokenized into a record=event_id:time.
std::vector<std::string> bin_records;
boost::split(bin_records, record_value, boost::is_any_of(",:"));
auto bin_it = bin_records.begin();
for (; bin_it != bin_records.end(); bin_it++) {
std::string eid = *bin_it;
EventTime time = boost::lexical_cast<EventTime>(*(++bin_it));
records.push_back(std::make_pair(eid, time));
}
}
return records;
}
EventID EventSubscriberPlugin::getEventID() {
Status status;
// First get an event ID from the meta key.
std::string eid_key = "eid." + dbNamespace();
std::string last_eid_value;
std::string eid_value;
{
WriteLock lock(event_id_lock_);
status = getDatabaseValue(kEvents, eid_key, last_eid_value);
if (!status.ok() || last_eid_value.empty()) {
last_eid_value = "0";
}
last_eid_ = boost::lexical_cast<size_t>(last_eid_value) + 1;
eid_value = boost::lexical_cast<std::string>(last_eid_);
status = setDatabaseValue(kEvents, eid_key, eid_value);
}
if (!status.ok()) {
return "0";
}
return eid_value;
}
Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
// Get and increment the EID for this module.
EventID eid = getEventID();
// Without encouraging a missing event time, do not support a 0-time.
r["time"] = std::to_string((event_time == 0) ? getUnixTime() : event_time);
// Serialize and store the row data, for query-time retrieval.
std::string data;
auto status = serializeRowJSON(r, data);
if (!status.ok()) {
return status;
}
// Then remove the newline.
if (data.size() > 0 && data.back() == '\n') {
data.pop_back();
}
// Use the last EventID and a checkpoint bucket size to periodically apply
// buffer eviction. Eviction occurs if the total count exceeds events_max.
if (last_eid_ % EVENTS_CHECKPOINT == 0) {
expireCheck();
}
// Store the event data.
std::string event_key = "data." + dbNamespace() + "." + eid;
status = setDatabaseValue(kEvents, event_key, data);
// Record the event in the indexing bins, using the index time.
recordEvent(eid, event_time);
return status;
}
Status EventSubscriberPlugin::add(const Row& r, EventTime time) {
Status status;
std::shared_ptr<DBHandle> db;
try {
db = DBHandle::getInstance();
} catch (const std::runtime_error& e) {
return Status(1, e.what());
}
// Get and increment the EID for this module.
EventID eid = getEventID();
std::string event_key = "data." + dbNamespace() + "." + eid;
std::string data;
status = serializeRowJSON(r, data);
if (!status.ok()) {
return status;
}
// Store the event data.
status = db->Put(kEvents, event_key, data);
// Record the event in the indexing bins.
recordEvent(eid, time);
return status;
}
Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
std::shared_ptr<DBHandle> db = nullptr;
try {
db = DBHandle::getInstance();
} catch (const std::runtime_error& e) {
return Status(1, e.what());
}
// Get and increment the EID for this module.
EventID eid = getEventID();
// Without encouraging a missing event time, do not support a 0-time.
r["time"] = std::to_string((event_time == 0) ? getUnixTime() : event_time);
// Serialize and store the row data, for query-time retrieval.
std::string data;
auto status = serializeRowJSON(r, data);
if (!status.ok()) {
return status;
}
// Use the last EventID and a checkpoint bucket size to periodically apply
// buffer eviction. Eviction occurs if the total count exceeds events_max.
if (last_eid_ % EVENTS_CHECKPOINT == 0) {
expireCheck();
}
// Store the event data.
std::string event_key = "data." + dbNamespace() + "." + eid;
status = db->Put(kEvents, event_key, data);
// Record the event in the indexing bins, using the index time.
recordEvent(eid, event_time);
return status;
}
Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime et) {
std::string time_value = boost::lexical_cast<std::string>(et);
// The record is identified by the event type then module name.
std::string index_key = "indexes." + dbNamespace();
std::string record_key = "records." + dbNamespace();
// The list key includes the list type (bin size) and the list ID (bin).
std::string list_id;
// The list_id is the MOST-Specific key ID, the bin for this list.
// If the event time was 13 and the time_list is 5 seconds, lid = 2.
list_id = boost::lexical_cast<std::string>(et / 60);
WriteLock lock(event_record_lock_);
// Append the record (eid, unix_time) to the list bin.
std::string record_value;
getDatabaseValue(kEvents, record_key + ".60." + list_id, record_value);
if (record_value.length() == 0) {
// This is a new list_id for list_key, append the ID to the indirect
// lookup for this list_key.
std::string index_value;
getDatabaseValue(kEvents, index_key + ".60", index_value);
if (index_value.length() == 0) {
// A new index.
index_value = list_id;
} else {
index_value += "," + list_id;
}
setDatabaseValue(kEvents, index_key + ".60", index_value);
record_value = eid + ":" + time_value;
} else {
// Tokenize a record using ',' and the EID/time using ':'.
record_value += "," + eid + ":" + time_value;
}
auto status =
setDatabaseValue(kEvents, record_key + ".60." + list_id, record_value);
if (!status.ok()) {
LOG(ERROR) << "Could not put Event Record key: " << record_key;
}
return Status(0, "OK");
}
void EventSubscriberPlugin::expireCheck() {
auto db = DBHandle::getInstance();
auto data_key = "data." + dbNamespace();
auto eid_key = "eid." + dbNamespace();
std::vector<std::string> keys;
db->ScanPrefix(kEvents, keys, data_key);
if (keys.size() <= FLAGS_events_max) {
return;
}
// There is an overflow of events buffered for this subscriber.
LOG(WARNING) << "Expiring events for subscriber: " << getName() << " limit ("
<< FLAGS_events_max << ") exceeded: " << keys.size();
// Inspect the N-FLAGS_events_max -th event's value and expire before the
// time within the content.
std::string last_key;
db->Get(kEvents, eid_key, last_key);
// The EID is the next-index.
size_t max_key = boost::lexical_cast<size_t>(last_key) - FLAGS_events_max - 1;
// Convert the key index into a time using the content.
std::string content;
db->Get(kEvents, data_key + "." + std::to_string(max_key), content);
// Decode the value into a row structure to extract the time.
Row r;
if (!deserializeRowJSON(content, r) || r.count("time") == 0) {
return;
}
// The last time will become the implicit expiration time.
size_t last_time = boost::lexical_cast<size_t>(r.at("time"));
if (last_time > 0) {
expire_time_ = last_time;
}
// Finally, attempt an index query to trigger expirations.
// In this case the result set is not used.
getIndexes(expire_time_ - 1, -1);
}
Status EventSubscriberPlugin::expireIndexes(
const std::string& list_type,
const std::vector<std::string>& indexes,
const std::vector<std::string>& expirations) {
auto db = DBHandle::getInstance();
auto index_key = "indexes." + dbNamespace();
auto record_key = "records." + dbNamespace();
auto data_key = "data." + dbNamespace();
// Get the records list for the soon-to-be expired records.
std::vector<std::string> record_indexes;
for (const auto& bin : expirations) {
record_indexes.push_back(list_type + "." + bin);
}
auto expired_records = getRecords(record_indexes);
// Remove the records using the list of expired indexes.
std::vector<std::string> persisting_indexes = indexes;
for (const auto& bin : expirations) {
db->Delete(kEvents, record_key + "." + list_type + "." + bin);
persisting_indexes.erase(
std::remove(persisting_indexes.begin(), persisting_indexes.end(), bin),
persisting_indexes.end());
}
// Update the list of indexes with the non-expired indexes.
auto new_indexes = boost::algorithm::join(persisting_indexes, ",");
db->Put(kEvents, index_key + "." + list_type, new_indexes);
// Delete record events.
for (const auto& record : expired_records) {
db->Delete(kEvents, data_key + "." + record.first);
}
return Status(0, "OK");
}
std::vector<std::string> EventSubscriberPlugin::getIndexes(EventTime start,
EventTime stop) {
auto index_key = "indexes." + dbNamespace();
std::vector<std::string> indexes;
EventTime l_start = (start > 0) ? start / 60 : 0;
EventTime r_stop = (stop > 0) ? stop / 60 + 1 : 0;
std::string content;
getDatabaseValue(kEvents, index_key + ".60", content);
if (content.empty()) {
return indexes;
}
std::vector<std::string> bins, expirations;
boost::split(bins, content, boost::is_any_of(","));
for (const auto& bin : bins) {
auto step = timeFromRecord(bin);
auto step_start = step * 60;
auto step_stop = (step + 1) * 60;
if (step_stop < expire_time_) {
expirations.push_back(bin);
} else if (step_start < expire_time_) {
expireRecords("60", bin, false);
}
if (step >= l_start && (r_stop == 0 || step < r_stop)) {
indexes.push_back("60." + bin);
}
}
// Rewrite the index lists and delete each expired item.
if (!expirations.empty()) {
expireIndexes("60", bins, expirations);
}
// Return indexes in binning order.
std::sort(indexes.begin(),
indexes.end(),
[](const std::string& left, const std::string& right) {
auto n1 = timeFromRecord(left.substr(left.find(".") + 1));
auto n2 = timeFromRecord(right.substr(right.find(".") + 1));
return n1 < n2;
});
// Update the new time that events expire to now - expiry.
return indexes;
}
QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
QueryData results;
Status status;
std::shared_ptr<DBHandle> db;
try {
db = DBHandle::getInstance();
} catch (const std::runtime_error& e) {
LOG(ERROR) << "Cannot retrieve subscriber results database is locked";
return results;
}
// Get the records for this time range.
auto indexes = getIndexes(start, stop);
auto records = getRecords(indexes);
std::vector<EventRecord> mapped_records;
for (const auto& record : records) {
if (record.second >= start && (record.second <= stop || stop == 0)) {
mapped_records.push_back(record);
}
}
std::string events_key = "data." + dbNamespace();
if (FLAGS_events_expiry > 0) {
// Set the expire time to NOW - "configured lifetime".
// Index retrieval will apply the constraints checking and auto-expire.
expire_time_ = getUnixTime() - FLAGS_events_expiry;
}
// Select mapped_records using event_ids as keys.
std::string data_value;
for (const auto& record : mapped_records) {
Row r;
status = db->Get(kEvents, events_key + "." + record.first, data_value);
if (data_value.length() == 0) {
// THere is no record here, interesting error case.
continue;
}
status = deserializeRowJSON(data_value, r);
if (status.ok()) {
results.push_back(r);
}
}
return results;
}
void EventSubscriberPlugin::expireIndexes(
const std::string& list_type,
const std::vector<std::string>& indexes,
const std::vector<std::string>& expirations) {
auto index_key = "indexes." + dbNamespace();
// Construct a mutable list of persisting indexes to rewrite as records.
std::vector<std::string> persisting_indexes = indexes;
// Remove the records using the list of expired indexes.
for (const auto& bin : expirations) {
expireRecords(list_type, bin, true);
persisting_indexes.erase(
std::remove(persisting_indexes.begin(), persisting_indexes.end(), bin),
persisting_indexes.end());
}
// Update the list of indexes with the non-expired indexes.
auto new_indexes = boost::algorithm::join(persisting_indexes, ",");
setDatabaseValue(kEvents, index_key + "." + list_type, new_indexes);
}
QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
QueryData results;
// Get the records for this time range.
auto db = DBHandle::getInstance();
auto indexes = getIndexes(start, stop);
auto records = getRecords(indexes);
std::string events_key = "data." + dbNamespace();
std::vector<std::string> mapped_records;
for (const auto& record : records) {
if (record.second >= start && (record.second <= stop || stop == 0)) {
mapped_records.push_back(events_key + "." + record.first);
}
}
// Select mapped_records using event_ids as keys.
std::string data_value;
for (const auto& record : mapped_records) {
Row r;
auto status = db->Get(kEvents, record, data_value);
if (data_value.length() == 0) {
// There is no record here, interesting error case.
continue;
}
status = deserializeRowJSON(data_value, r);
data_value.clear();
if (status.ok()) {
results.push_back(std::move(r));
}
}
if (FLAGS_events_expiry > 0) {
// Set the expire time to NOW - "configured lifetime".
// Index retrieval will apply the constraints checking and auto-expire.
expire_time_ = getUnixTime() - FLAGS_events_expiry;
}
return results;
}
std::set<std::string> EventSubscriberPlugin::getIndexes(EventTime start,
EventTime stop,
int list_key) {
auto db = DBHandle::getInstance();
auto index_key = "indexes." + dbNamespace();
std::set<std::string> indexes;
// Keep track of the tail/head of account time while bin searching.
EventTime start_max = stop, stop_min = stop, local_start, local_stop;
auto types = kEventTimeLists.size();
// List types are sized bins of time containing records for this namespace.
for (size_t i = 0; i < types; ++i) {
auto size = kEventTimeLists[i];
if (list_key > 0 && i != list_key) {
// A specific list_type was requested, only return bins of this key.
continue;
}
std::string time_list;
auto list_type = boost::lexical_cast<std::string>(size);
auto status = db->Get(kEvents, index_key + "." + list_type, time_list);
if (time_list.length() == 0) {
// No events in this binning size.
return indexes;
}
if (list_key == 0 && i == (types - 1) && types > 1) {
// Relax the requested start/stop bounds.
if (start != start_max) {
start = (start / size) * size;
start_max = ((start / size) + 1) * size;
if (start_max < stop) {
start_max = start + kEventTimeLists[types - 2];
}
}
if (stop != stop_min) {
stop = ((stop / size) + 1) * size;
stop_min = (stop / size) * size;
if (stop_min > start) {
stop_min = stop_min - kEventTimeLists[types - 1];
}
}
} else if (list_key > 0 || types == 1) {
// Relax the requested bounds to fit the requested/only index.
start = (start / size) * size;
start_max = ((start_max / size) + 1) * size;
}
// (1) The first iteration will have 1 range (start to start_max=stop).
// (2) Intermediate iterations will have 2 (start-start_max, stop-stop_min).
// For each iteration the range collapses based on the coverage using
// the first bin's start time and the last bin's stop time.
// (3) The last iteration's range includes relaxed bounds outside the
// requested start to stop range.
std::vector<std::string> all_bins, bins, expirations;
boost::split(all_bins, time_list, boost::is_any_of(","));
for (const auto& bin : all_bins) {
// Bins are identified by the binning size step.
auto step = boost::lexical_cast<EventTime>(bin);
// Check if size * step -> size * (step + 1) is within a range.
int bin_start = size * step, bin_stop = size * (step + 1);
if (expire_events_ && expire_time_ > 0) {
if (bin_stop <= expire_time_) {
expirations.push_back(bin);
} else if (bin_start < expire_time_) {
expireRecords(list_type, bin, false);
}
}
if (bin_start >= start && bin_stop <= start_max) {
bins.push_back(bin);
} else if ((bin_start >= stop_min && bin_stop <= stop) || stop == 0) {
bins.push_back(bin);
}
}
// Rewrite the index lists and delete each expired item.
if (expirations.size() > 0) {
expireIndexes(list_type, all_bins, expirations);
}
if (bins.size() != 0) {
// If more precision was achieved though this list's binning.
local_start = boost::lexical_cast<EventTime>(bins.front()) * size;
start_max = (local_start < start_max) ? local_start : start_max;
local_stop = (boost::lexical_cast<EventTime>(bins.back()) + 1) * size;
stop_min = (local_stop < stop_min) ? local_stop : stop_min;
}
for (const auto& bin : bins) {
indexes.insert(list_type + "." + bin);
}
if (start == start_max && stop == stop_min) {
break;
}
}
// Update the new time that events expire to now - expiry.
return indexes;
}
