TEST_F(EventsDatabaseTests, test_record_expiration) {
auto sub = std::make_shared<FakeEventSubscriber>();
// No expiration
auto indexes = sub->getIndexes(0, 60);
auto records = sub->getRecords(indexes);
EXPECT_EQ(records.size(), 3); // 1, 2, 11
sub->expire_events_ = true;
sub->expire_time_ = 10;
indexes = sub->getIndexes(0, 60);
records = sub->getRecords(indexes);
EXPECT_EQ(records.size(), 1); // 11
}
void EventSubscriberPlugin::expireRecords(const std::string& list_type,
const std::string& index,
bool all) {
auto db = DBHandle::getInstance();
auto record_key = "records." + dbNamespace();
auto data_key = "data." + dbNamespace();
// If the expirations is not removing all records, rewrite the persisting.
std::vector<std::string> persisting_records;
// Request all records within this list-size + bin offset.
auto expired_records = getRecords({list_type + "." + index});
for (const auto& record : expired_records) {
if (all) {
db->Delete(kEvents, data_key + "." + record.first);
} else if (record.second > expire_time_) {
persisting_records.push_back(record.first + ":" +
std::to_string(record.second));
}
}
// Either drop or overwrite the record list.
if (all) {
db->Delete(kEvents, record_key + "." + list_type + "." + index);
} else {
auto new_records = boost::algorithm::join(persisting_records, ",");
db->Put(kEvents, record_key + "." + list_type + "." + index, new_records);
}
}
Post::Post(QWidget *parent):
QWidget(parent)
{
setFixedSize(parent->minimumWidth(),25);
setObjectName("Post");
manager=new QNetworkAccessManager(this);
Config::setManager(manager);
auto layout=new QHBoxLayout(this);
layout->setMargin(0);
layout->setSpacing(0);
commentM=new QComboBox(this);
commentM->addItems({tr("Top"),tr("Slide"),tr("Bottom")});
commentM->setCurrentIndex(1);
commentM->setFixedWidth(commentM->sizeHint().width());
layout->addWidget(commentM);
commentC=new QPushButton(this);
commentC->setFixedWidth(25);
setColor(Qt::white);
connect(commentC,&QPushButton::clicked,[this](){
QColor color=QColorDialog::getColor(getColor(),lApp->mainWidget());
if(color.isValid()){
setColor(color);
}
});
layout->addWidget(commentC);
commentL=new QLineEdit(this);
commentL->setFocus();
layout->addWidget(commentL);
commentS=new QComboBox(this);
layout->addWidget(commentS);
commentB=new QPushButton(tr("Post"),this);
commentB->setDefault(true);
commentB->setFixedWidth(55);
commentB->setToolTip(tr("DA☆ZE!"));
layout->addWidget(commentB);
commentA=new QAction(this);
commentA->setShortcut(QKeySequence("Ctrl+Enter"));
connect(commentB,&QPushButton::clicked,commentA,&QAction::trigger);
connect(commentL,&QLineEdit::returnPressed,commentA,&QAction::trigger);
connect(commentA,&QAction::triggered,[this](){
if(!commentL->text().isEmpty()){
postComment();
commentL->clear();
hide();
}
});
connect(Danmaku::instance(),&Danmaku::modelReset,[this](){
commentS->clear();
int w=0;
for(const Record *r:getRecords()){
commentS->addItem(r->string,(quintptr)r);
w=qMax(w,commentS->fontMetrics().width(r->string));
}
commentS->setVisible(commentS->count()>=2);
commentS->setFixedWidth(w+30);
});
hide();
}
TEST_F(EventsDatabaseTests, test_record_range) {
auto sub = std::make_shared<DBFakeEventSubscriber>();
auto status = sub->testAdd(1);
status = sub->testAdd(2);
status = sub->testAdd(11);
status = sub->testAdd(61);
status = sub->testAdd((1 * 3600) + 1);
status = sub->testAdd((2 * 3600) + 1);
// Search within a specific record range.
auto indexes = sub->getIndexes(0, 10);
EXPECT_EQ(1U, indexes.size());
auto records = sub->getRecords(indexes);
// This will return 3 results, ::get filters records by an absolute range.
EXPECT_EQ(3U, records.size()); // 1, 2, 11
// Search within a large bound.
indexes = sub->getIndexes(3, 3601);
// This will include the 0-10 bucket meaning 1, 2 will show up.
records = sub->getRecords(indexes);
EXPECT_EQ(5U, records.size()); // 1, 2, 11, 61, 3601
// Get all of the records.
indexes = sub->getIndexes(0, 3 * 3600);
records = sub->getRecords(indexes);
EXPECT_EQ(6U, records.size()); // 1, 2, 11, 61, 3601, 7201
// stop = 0 is an alias for everything.
indexes = sub->getIndexes(0, 0);
records = sub->getRecords(indexes);
EXPECT_EQ(6U, records.size());
for (size_t j = 0; j < 30; j++) {
// 110 is 10 below an index (60.2).
sub->testAdd(110 + static_cast<int>(j));
}
indexes = sub->getIndexes(110, 0);
auto output = boost::algorithm::join(indexes, ", ");
EXPECT_EQ("60.1, 60.2, 60.60, 60.120", output);
records = sub->getRecords(indexes);
EXPECT_EQ(33U, records.size()); // (61) 110 - 139 + 3601, 7201
}
QList<QSqlRecord> GenericDAO::getRecords(const QString& from, const QString& where, const QVariantList& bindValues, const QString& sort) const
{
QString sql = QString("SELECT * FROM %1").arg(from.isEmpty() ? m_table : from);
if (!where.isEmpty())
sql += " WHERE " + where;
if (!sort.isEmpty())
sql += " ORDER BY " + sort;
return getRecords(sql, bindValues);
}
int CSMWorld::InfoCollection::getInfoIndex (const std::string& id, const std::string& topic) const
{
std::string fullId = Misc::StringUtils::lowerCase (topic) + "#" + id;
std::pair<RecordConstIterator, RecordConstIterator> range = getTopicRange (topic);
for (; range.first!=range.second; ++range.first)
if (Misc::StringUtils::ciEqual(range.first->get().mId, fullId))
return std::distance (getRecords().begin(), range.first);
return -1;
}
TEST_F(EventsDatabaseTests, test_record_expiration) {
auto sub = std::make_shared<DBFakeEventSubscriber>();
// No expiration
auto indexes = sub->getIndexes(0, 5000);
auto records = sub->getRecords(indexes);
EXPECT_EQ(records.size(), 5U); // 1, 2, 11, 61, 3601
sub->expire_events_ = true;
sub->expire_time_ = 10;
indexes = sub->getIndexes(0, 5000);
records = sub->getRecords(indexes);
EXPECT_EQ(records.size(), 3U); // 11, 61, 3601
indexes = sub->getIndexes(0, 5000, 0);
records = sub->getRecords(indexes);
EXPECT_EQ(records.size(), 3U); // 11, 61, 3601
indexes = sub->getIndexes(0, 5000, 1);
records = sub->getRecords(indexes);
EXPECT_EQ(records.size(), 3U); // 11, 61, 3601
indexes = sub->getIndexes(0, 5000, 2);
records = sub->getRecords(indexes);
EXPECT_EQ(records.size(), 3U); // 11, 61, 3601
// Check that get/deletes did not act on cache.
// This implies that RocksDB is flushing the requested delete records.
sub->expire_time_ = 0;
indexes = sub->getIndexes(0, 5000);
records = sub->getRecords(indexes);
EXPECT_EQ(records.size(), 3U); // 11, 61, 3601
}
int CSMWorld::InfoCollection::getAppendIndex (const std::string& id, UniversalId::Type type) const
{
std::string::size_type separator = id.find_last_of ('#');
if (separator==std::string::npos)
throw std::runtime_error ("invalid info ID: " + id);
std::pair<RecordConstIterator, RecordConstIterator> range = getTopicRange (id.substr (0, separator));
if (range.first==range.second)
return Collection<Info, IdAccessor<Info> >::getAppendIndex (id, type);
return std::distance (getRecords().begin(), range.second);
}
void CSMWorld::InfoCollection::load (const Info& record, bool base)
{
int index = searchId (record.mId);
if (index==-1)
{
// new record
Record<Info> record2;
record2.mState = base ? RecordBase::State_BaseOnly : RecordBase::State_ModifiedOnly;
(base ? record2.mBase : record2.mModified) = record;
int index = -1;
std::string topic = Misc::StringUtils::lowerCase (record2.get().mTopicId);
if (!record2.get().mPrev.empty())
{
index = getInfoIndex (record2.get().mPrev, topic);
if (index!=-1)
++index;
}
if (index==-1 && !record2.get().mNext.empty())
{
index = getInfoIndex (record2.get().mNext, topic);
}
if (index==-1)
{
Range range = getTopicRange (topic);
index = std::distance (getRecords().begin(), range.second);
}
insertRecord (record2, index);
}
else
{
// old record
Record<Info> record2 = getRecord (index);
if (base)
record2.mBase = record;
else
record2.setModified (record);
setRecord (index, record2);
}
}
QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
QueryData results;
// Get the records for this time range.
auto indexes = getIndexes(start, stop);
auto records = getRecords(indexes);
std::string events_key = "data." + dbNamespace();
std::vector<std::string> mapped_records;
for (const auto& record : records) {
if (record.second >= start && (record.second <= stop || stop == 0)) {
mapped_records.push_back(events_key + "." + record.first);
}
}
if (FLAGS_events_optimize && !records.empty()) {
// If records were returned save the ordered-last as the optimization EID.
unsigned long int eidr = 0;
if (safeStrtoul(records.back().first, 10, eidr)) {
optimize_eid_ = static_cast<size_t>(eidr);
auto index_key = "optimize_id." + dbNamespace();
setDatabaseValue(kEvents, index_key, records.back().first);
}
}
// Select mapped_records using event_ids as keys.
std::string data_value;
for (const auto& record : mapped_records) {
Row r;
auto status = getDatabaseValue(kEvents, record, data_value);
if (data_value.length() == 0) {
// There is no record here, interesting error case.
continue;
}
status = deserializeRowJSON(data_value, r);
data_value.clear();
if (status.ok()) {
results.push_back(std::move(r));
}
}
if (getEventsExpiry() > 0) {
// Set the expire time to NOW - "configured lifetime".
// Index retrieval will apply the constraints checking and auto-expire.
expire_time_ = getUnixTime() - getEventsExpiry();
}
return results;
}
TEST_F(EventsDatabaseTests, test_record_range) {
auto sub = std::make_shared<DBFakeEventSubscriber>();
// Search within a specific record range.
auto indexes = sub->getIndexes(0, 10);
auto records = sub->getRecords(indexes);
EXPECT_EQ(records.size(), 2U); // 1, 2
// Search within a large bound.
indexes = sub->getIndexes(3, 3601);
// This will include the 0-10 bucket meaning 1, 2 will show up.
records = sub->getRecords(indexes);
EXPECT_EQ(records.size(), 5U); // 1, 2, 11, 61, 3601
// Get all of the records.
indexes = sub->getIndexes(0, 3 * 3600);
records = sub->getRecords(indexes);
EXPECT_EQ(records.size(), 8U); // 1, 2, 11, 61, 3601, 7201, 7211, 7261
// stop = 0 is an alias for everything.
indexes = sub->getIndexes(0, 0);
records = sub->getRecords(indexes);
EXPECT_EQ(records.size(), 8U);
}
QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
QueryData results;
Status status;
std::shared_ptr<DBHandle> db;
try {
db = DBHandle::getInstance();
} catch (const std::runtime_error& e) {
LOG(ERROR) << "Cannot retrieve subscriber results database is locked";
return results;
}
// Get the records for this time range.
auto indexes = getIndexes(start, stop);
auto records = getRecords(indexes);
std::vector<EventRecord> mapped_records;
for (const auto& record : records) {
if (record.second >= start && (record.second <= stop || stop == 0)) {
mapped_records.push_back(record);
}
}
std::string events_key = "data." + dbNamespace();
if (FLAGS_events_expiry > 0) {
// Set the expire time to NOW - "configured lifetime".
// Index retrieval will apply the constraints checking and auto-expire.
expire_time_ = getUnixTime() - FLAGS_events_expiry;
}
// Select mapped_records using event_ids as keys.
std::string data_value;
for (const auto& record : mapped_records) {
Row r;
status = db->Get(kEvents, events_key + "." + record.first, data_value);
if (data_value.length() == 0) {
// THere is no record here, interesting error case.
continue;
}
status = deserializeRowJSON(data_value, r);
if (status.ok()) {
results.push_back(r);
}
}
return results;
}
QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
QueryData results;
// Get the records for this time range.
auto db = DBHandle::getInstance();
auto indexes = getIndexes(start, stop);
auto records = getRecords(indexes);
std::string events_key = "data." + dbNamespace();
std::vector<std::string> mapped_records;
for (const auto& record : records) {
if (record.second >= start && (record.second <= stop || stop == 0)) {
mapped_records.push_back(events_key + "." + record.first);
}
}
// Select mapped_records using event_ids as keys.
std::string data_value;
for (const auto& record : mapped_records) {
Row r;
auto status = db->Get(kEvents, record, data_value);
if (data_value.length() == 0) {
// There is no record here, interesting error case.
continue;
}
status = deserializeRowJSON(data_value, r);
data_value.clear();
if (status.ok()) {
results.push_back(std::move(r));
}
}
if (FLAGS_events_expiry > 0) {
// Set the expire time to NOW - "configured lifetime".
// Index retrieval will apply the constraints checking and auto-expire.
expire_time_ = getUnixTime() - FLAGS_events_expiry;
}
return results;
}
Status EventSubscriberPlugin::expireIndexes(
const std::string& list_type,
const std::vector<std::string>& indexes,
const std::vector<std::string>& expirations) {
auto db = DBHandle::getInstance();
auto index_key = "indexes." + dbNamespace();
auto record_key = "records." + dbNamespace();
auto data_key = "data." + dbNamespace();
// Get the records list for the soon-to-be expired records.
std::vector<std::string> record_indexes;
for (const auto& bin : expirations) {
record_indexes.push_back(list_type + "." + bin);
}
auto expired_records = getRecords(record_indexes);
// Remove the records using the list of expired indexes.
std::vector<std::string> persisting_indexes = indexes;
for (const auto& bin : expirations) {
db->Delete(kEvents, record_key + "." + list_type + "." + bin);
persisting_indexes.erase(
std::remove(persisting_indexes.begin(), persisting_indexes.end(), bin),
persisting_indexes.end());
}
// Update the list of indexes with the non-expired indexes.
auto new_indexes = boost::algorithm::join(persisting_indexes, ",");
db->Put(kEvents, index_key + "." + list_type, new_indexes);
// Delete record events.
for (const auto& record : expired_records) {
db->Delete(kEvents, data_key + "." + record.first);
}
return Status(0, "OK");
}
CSMWorld::InfoCollection::Range CSMWorld::InfoCollection::getTopicRange (const std::string& topic)
const
{
std::string topic2 = Misc::StringUtils::lowerCase (topic);
std::map<std::string, int>::const_iterator iter = getIdMap().lower_bound (topic2);
// Skip invalid records: The beginning of a topic string could be identical to another topic
// string.
for (; iter!=getIdMap().end(); ++iter)
{
std::string testTopicId =
Misc::StringUtils::lowerCase (getRecord (iter->second).get().mTopicId);
if (testTopicId==topic2)
break;
std::size_t size = topic2.size();
if (testTopicId.size()<size || testTopicId.substr (0, size)!=topic2)
return Range (getRecords().end(), getRecords().end());
}
if (iter==getIdMap().end())
return Range (getRecords().end(), getRecords().end());
RecordConstIterator begin = getRecords().begin()+iter->second;
// Find end
RecordConstIterator end = begin;
for (; end!=getRecords().end(); ++end)
if (!Misc::StringUtils::ciEqual(end->get().mTopicId, topic2))
break;
return Range (begin, end);
}
void PeventTest::setDBPointer(database *db)
{
// createBasic(db);
types = getRecords(db);
}
INT_PTR CALLBACK stats_dlgproc(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam) {
HWND dlg;
LPNMHDR lpn = (LPNMHDR) lParam;
switch (uMsg) {
case WM_INITDIALOG:
dlg = CreateDialog(plugin.hDllInstance,
MAKEINTRESOURCE(IDD_PROCESSING),
hwndDlg, process_dlgproc);
memset(tmp, 0, sizeof(tmp));
GetPrivateProfileString("ml_stats", "query", "type = \"0\"",
tmp, sizeof(tmp), iniFile);
SetDlgItemText(hwndDlg, IDC_QUERY, tmp);
m_pRecords = getRecords(tmp);
TCITEM tie;
m_hwndDlg = hwndDlg;
m_hwndTab = GetDlgItem(hwndDlg, IDC_STATS_TABS);
tie.mask = TCIF_TEXT;
for (int i = 0; i < sizeof(tabs) / sizeof(pref_tab); i++) {
tie.pszText = tabs[i].title;
TabCtrl_InsertItem(m_hwndTab, i, &tie);
}
DestroyWindow(dlg);
TabCtrl_SetCurSel(m_hwndTab, 0);
OnSelChanged(hwndDlg);
break;
case WM_NOTIFY:
if(lpn) if(lpn->code==TCN_SELCHANGE) OnSelChanged(hwndDlg);
break;
case WM_COMMAND:
switch (LOWORD(wParam)) {
case IDC_ABOUT:
LoadString(plugin.hDllInstance, IDS_ABOUTBOX, tmp, sizeof(tmp));
MessageBox(hwndDlg, tmp, "Media Library Stastics",
MB_OK | MB_ICONINFORMATION);
break;
case IDC_FILTER:
dlg = CreateDialog(plugin.hDllInstance,
MAKEINTRESOURCE(IDD_PROCESSING),
hwndDlg, process_dlgproc);
memset(tmp, 0, sizeof(tmp));
GetDlgItemText(hwndDlg, IDC_QUERY, tmp, sizeof(tmp));
m_pRecords = getRecords(tmp);
DestroyWindow(dlg);
OnSelChanged(hwndDlg);
break;
case IDOK:
memset(tmp, 0, sizeof(tmp));
GetDlgItemText(hwndDlg, IDC_QUERY, tmp, sizeof(tmp));
WritePrivateProfileString("ml_stats", "query", tmp, iniFile);
case IDCANCEL:
EndDialog(hwndDlg, 0);
break;
}
break;
}
return 0;
}
bool Post::isValid()
{
return !getRecords().isEmpty();
}
