void Sequencer::filter(PSortedRun run, std::shared_ptr<QP::IQueryProcessor> q, std::vector<PSortedRun>* results) const {
if (run->empty()) {
return;
}
PSortedRun result(new SortedRun);
auto lkey = TimeSeriesValue(q->lowerbound(), 0u, 0);
auto rkey = TimeSeriesValue(q->upperbound(), ~0u, 0);
auto begin = std::lower_bound(run->begin(), run->end(), lkey);
auto end = std::upper_bound(run->begin(), run->end(), rkey);
std::copy(begin, end, std::back_inserter(*result));
results->push_back(move(result));
}
void Sequencer::filter(PSortedRun run, SearchQuery const& q, std::vector<PSortedRun>* results) const {
if (run->empty()) {
return;
}
SearchPredicate search_pred(q);
PSortedRun result(new SortedRun);
auto lkey = TimeSeriesValue(q.lowerbound, 0u, 0u, 0u);
auto rkey = TimeSeriesValue(q.upperbound, ~0u, 0u, 0u);
auto begin = std::lower_bound(run->begin(), run->end(), lkey);
auto end = std::upper_bound(run->begin(), run->end(), rkey);
copy_if(begin, end, std::back_inserter(*result), search_pred);
results->push_back(move(result));
}
Sequencer::Sequencer(const aku_FineTuneParams &config)
: window_size_(config.window_size)
, top_timestamp_()
, checkpoint_(0u)
, sequence_number_ {0}
, run_locks_(RUN_LOCK_FLAGS_SIZE)
, c_threshold_(config.compression_threshold)
{
key_.reset(new SortedRun());
key_->push_back(TimeSeriesValue());
}
Sequencer::Sequencer(PageHeader const* page, aku_Config config)
: window_size_(config.window_size)
, page_(page)
, top_timestamp_()
, checkpoint_(0u)
, sequence_number_ {0}
, run_locks_(RUN_LOCK_FLAGS_SIZE)
, space_estimate_(0u)
, c_threshold_(config.compression_threshold)
{
key_.reset(new SortedRun());
key_->push_back(TimeSeriesValue());
}
// move sorted runs to ready_ collection
int Sequencer::make_checkpoint_(aku_Timestamp new_checkpoint) {
int flag = sequence_number_.fetch_add(1) + 1;
if (flag % 2 != 0) {
auto old_top = get_timestamp_(checkpoint_);
checkpoint_ = new_checkpoint;
vector<PSortedRun> new_runs;
for (auto& sorted_run: runs_) {
auto it = lower_bound(sorted_run->begin(), sorted_run->end(), TimeSeriesValue(old_top, AKU_LIMITS_MAX_ID, 0u, 0u));
// Check that compression threshold is reached
if (it == sorted_run->begin()) {
// all timestamps are newer than old_top, do nothing
new_runs.push_back(move(sorted_run));
continue;
} else if (it == sorted_run->end()) {
// all timestamps are older than old_top, move them
ready_.push_back(move(sorted_run));
} else {
// it is in between of the sorted run - split
PSortedRun run(new SortedRun());
copy(sorted_run->begin(), it, back_inserter(*run)); // copy old
ready_.push_back(move(run));
run.reset(new SortedRun());
copy(it, sorted_run->end(), back_inserter(*run)); // copy new
new_runs.push_back(move(run));
}
}
Lock guard(runs_resize_lock_);
space_estimate_ = 0u;
for (auto& sorted_run: new_runs) {
space_estimate_ += sorted_run->size() * SPACE_PER_ELEMENT;
}
swap(runs_, new_runs);
size_t ready_size = 0u;
for (auto& sorted_run: ready_) {
ready_size += sorted_run->size();
}
if (ready_size < c_threshold_) {
// If ready doesn't contains enough data compression wouldn't be efficient,
// we need to wait for more data to come
flag = sequence_number_.fetch_add(1) + 1;
// We should make sorted runs in ready_ array searchable again
for (auto& sorted_run: ready_) {
runs_.push_back(sorted_run);
}
ready_.clear();
}
}
return flag;
}
Sequencer::Sequencer(PageHeader const* page, const aku_FineTuneParams &config)
: window_size_(config.window_size)
, page_(page)
, top_timestamp_()
, checkpoint_(0u)
, sequence_number_ {0}
, run_locks_(RUN_LOCK_FLAGS_SIZE)
, c_threshold_(config.compression_threshold)
{
key_.reset(new SortedRun());
key_->push_back(TimeSeriesValue());
if (page) {
auto cnt = page->get_entries_count();
if (cnt != 0) {
auto ts = page->read_timestamp_at(cnt - 1);
checkpoint_ = get_checkpoint_(ts);
top_timestamp_ = get_timestamp_(ts);
}
}
}
aku_Status Sequencer::merge_and_compress(PageHeader* target, bool enforce_write) {
bool owns_lock = sequence_number_.load() % 2; // progress_flag_ must be odd to start
if (!owns_lock) {
return AKU_EBUSY;
}
if (ready_.size() == 0) {
return AKU_ENO_DATA;
}
aku_Status status = AKU_SUCCESS;
while(!ready_.empty()) {
UncompressedChunk chunk_header;
chunk_header.paramids.reserve(c_threshold_);
chunk_header.timestamps.reserve(c_threshold_);
chunk_header.values.reserve(c_threshold_);
int threshold = (int)c_threshold_;
auto push_to_header = [&](TimeSeriesValue const& val) {
if (threshold-->0) {
val.add_to_header(&chunk_header);
return true;
}
return false;
};
kway_merge<TimeOrderMergePredicate, AKU_CURSOR_DIR_FORWARD>(ready_, push_to_header);
if (enforce_write || chunk_header.paramids.size() >= c_threshold_) {
UncompressedChunk reindexed_header;
if (!CompressionUtil::convert_from_time_order(chunk_header, &reindexed_header)) {
AKU_PANIC("Invalid chunk");
}
status = target->complete_chunk(reindexed_header);
} else {
// Wait for more data
status = AKU_ENO_DATA;
}
if (status != AKU_SUCCESS) {
PSortedRun run(new SortedRun());
for (int i = 0; i < (int)chunk_header.paramids.size(); i++) {
run->push_back(TimeSeriesValue(chunk_header.timestamps.at(i),
chunk_header.paramids.at(i),
chunk_header.values.at(i)));
}
ready_.push_back(std::move(run));
if (status == AKU_ENO_DATA) {
status = AKU_SUCCESS;
}
break;
}
}
if(!ready_.empty()) {
Lock guard(runs_resize_lock_);
for(auto sorted_run: ready_) {
runs_.push_back(sorted_run);
}
ready_.clear();
}
sequence_number_.fetch_add(1); // progress_flag_ is even again
return status;
}
