/*!
* Update data-object table for certain secondary index.
*
* @index_data is what client provided
* @data is what was downloaded from the storage
*/
data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, dnet_indexes_request *request,
const data_pointer &index_data, const data_pointer &data, update_index_action action)
{
dnet_indexes indexes;
if (!data.empty())
indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table");
// Construct index entry
index_entry request_index;
memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id));
request_index.data = index_data;
auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(),
request_index, dnet_raw_id_less_than<skip_data>());
if (it != indexes.indexes.end() && it->index == request_index.index) {
// It's already there
if (action == insert_data) {
if (it->data == request_index.data) {
// All's ok, keep it untouched
return data;
} else {
// Data is not correct, replace it by new one
it->data = request_index.data;
}
} else {
// Anyway, destroy it
indexes.indexes.erase(it);
}
} else {
// Index is not created yet
if (action == insert_data) {
// Just insert it
indexes.indexes.insert(it, 1, request_index);
} else {
// All's ok, keep it untouched
return data;
}
}
indexes.shard_id = request->shard_id;
indexes.shard_count = request->shard_count;
msgpack::sbuffer buffer;
msgpack::pack(&buffer, indexes);
data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size());
new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC));
new_buffer.write(buffer.data(), buffer.size());
return std::move(new_buffer);
}
int process_find_indexes(dnet_net_state *state, dnet_cmd *cmd, dnet_indexes_request *request)
{
local_session sess(state->n);
const bool intersection = request->flags & DNET_INDEXES_FLAGS_INTERSECT;
const bool unite = request->flags & DNET_INDEXES_FLAGS_UNITE;
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_FIND: indexes count: %u, flags: %llu\n",
(unsigned) request->entries_count, (unsigned long long) request->flags);
if (intersection && unite) {
return -ENOTSUP;
}
std::vector<find_indexes_result_entry> result;
std::map<dnet_raw_id, size_t, dnet_raw_id_less_than<> > result_map;
dnet_indexes tmp;
int err = -1;
dnet_id id = cmd->id;
size_t data_offset = 0;
char *data_start = reinterpret_cast<char *>(request->entries);
for (uint64_t i = 0; i < request->entries_count; ++i) {
dnet_indexes_request_entry &request_entry = *reinterpret_cast<dnet_indexes_request_entry *>(data_start + data_offset);
data_offset += sizeof(dnet_indexes_request_entry) + request_entry.size;
memcpy(id.id, request_entry.id.id, sizeof(id.id));
int ret = 0;
data_pointer data = sess.read(id, &ret);
if (ret) {
dnet_log(state->n, DNET_LOG_DEBUG, "%s: INDEXES_FIND, err: %d\n",
dnet_dump_id(&id), ret);
}
if (ret && unite) {
if (err != -1)
err = ret;
continue;
} else if (ret && intersection) {
return ret;
}
err = 0;
tmp.indexes.clear();
indexes_unpack(state->n, &id, data, &tmp, "process_find_indexes");
if (unite) {
for (size_t j = 0; j < tmp.indexes.size(); ++j) {
const index_entry &entry = tmp.indexes[j];
auto it = result_map.find(entry.index);
if (it == result_map.end()) {
it = result_map.insert(std::make_pair(entry.index, result.size())).first;
result.resize(result.size() + 1);
result.back().id = entry.index;
}
result[it->second].indexes.emplace_back(request_entry.id, entry.data);
}
} else if (intersection && i == 0) {
result.resize(tmp.indexes.size());
for (size_t j = 0; j < tmp.indexes.size(); ++j) {
find_indexes_result_entry &entry = result[j];
entry.id = tmp.indexes[j].index;
entry.indexes.emplace_back(
request_entry.id,
tmp.indexes[j].data);
}
} else if (intersection) {
// Remove all objects from result, which are not presented for this index
auto it = std::set_intersection(result.begin(), result.end(),
tmp.indexes.begin(), tmp.indexes.end(),
result.begin(),
dnet_raw_id_less_than<skip_data>());
result.resize(it - result.begin());
// Remove all objects from this index, which are not presented in result
std::set_intersection(tmp.indexes.begin(), tmp.indexes.end(),
result.begin(), result.end(),
tmp.indexes.begin(),
dnet_raw_id_less_than<skip_data>());
// As lists contain othe same objects - it's possible to add index data by one cycle
auto jt = tmp.indexes.begin();
for (auto kt = result.begin(); kt != result.end(); ++kt, ++jt) {
kt->indexes.emplace_back(request_entry.id, jt->data);
}
}
}
// if (err != 0)
// return err;
dnet_log(state->n, DNET_LOG_DEBUG, "%s: INDEXES_FIND: result of find: %zu objects\n",
dnet_dump_id(&id), result.size());
msgpack::sbuffer buffer;
msgpack::pack(&buffer, result);
cmd->flags &= ~DNET_FLAGS_NEED_ACK;
dnet_send_reply(state, cmd, buffer.data(), buffer.size(), 0);
return err;
}
/*!
* Update data-object table for certain secondary index.
*
* @index_data is what client provided
* @data is what was downloaded from the storage
*/
data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, const dnet_indexes_request *request,
const data_pointer &index_data, const data_pointer &data, uint32_t action,
std::vector<dnet_indexes_reply_entry> * &removed, uint32_t limit)
{
elliptics_timer timer;
dnet_indexes indexes;
if (!data.empty())
indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table");
const int64_t timer_unpack = timer.restart();
// Construct index entry
dnet_index_entry request_index;
memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id));
request_index.data = index_data;
dnet_current_time(&request_index.time);
auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(), request_index, dnet_raw_id_less_than<skip_data>());
const int64_t timer_lower_bound = timer.restart();
if (it != indexes.indexes.end() && it->index == request_index.index) {
// It's already there
if (action == DNET_INDEXES_FLAGS_INTERNAL_INSERT) {
// Item exists, update it's data and time if it's capped collection
if (!removed && it->data == request_index.data) {
const int64_t timer_compare = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_compare));
// All's ok, keep it untouched
return data;
}
it->data = request_index.data;
it->time = request_index.time;
} else {
// Anyway, destroy it
indexes.indexes.erase(it);
}
} else {
// Index is not created yet
if (action == DNET_INDEXES_FLAGS_INTERNAL_INSERT) {
// Remove extra elements from capped collection
if (removed && limit != 0 && indexes.indexes.size() + 1 > limit) {
dnet_indexes_reply_entry entry;
memset(&entry, 0, sizeof(entry));
auto position = it - indexes.indexes.begin();
while (indexes.indexes.size() + 1 > limit && !indexes.indexes.empty()) {
auto jt = std::min_element(indexes.indexes.begin(), indexes.indexes.end(), entry_time_less_than);
// jt will be removed, so it should be modified to be still valid
if (position > jt - indexes.indexes.begin())
--position;
memcpy(entry.id.id, jt->index.id, DNET_ID_SIZE);
entry.status = DNET_INDEXES_CAPPED_REMOVED;
indexes.indexes.erase(jt);
removed->push_back(entry);
}
it = indexes.indexes.begin() + position;
}
// And just insert new index
indexes.indexes.insert(it, 1, request_index);
} else {
const int64_t timer_compare = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_compare));
// All's ok, keep it untouched
return data;
}
}
const int64_t timer_update = timer.restart();
indexes.shard_id = request->shard_id;
indexes.shard_count = request->shard_count;
msgpack::sbuffer buffer;
msgpack::pack(&buffer, indexes);
const int64_t timer_pack = timer.restart();
data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size());
new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC));
new_buffer.write(buffer.data(), buffer.size());
const int64_t timer_write = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, update: %lld ms, pack: %lld ms, write: %lld ms\n",
id_str, data.size(), new_buffer.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_update), lld(timer_pack), lld(timer_write));
return std::move(new_buffer);
}
int process_find_indexes(dnet_net_state *state, dnet_cmd *cmd, const dnet_id &request_id, dnet_indexes_request *request, bool more)
{
local_session sess(state->n);
const bool intersection = request->flags & DNET_INDEXES_FLAGS_INTERSECT;
const bool unite = request->flags & DNET_INDEXES_FLAGS_UNITE;
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_FIND: indexes count: %u, flags: %llu, more: %d\n",
(unsigned) request->entries_count, (unsigned long long) request->flags, int(more));
if ((intersection && unite) || !(intersection || unite)) {
return -EINVAL;
}
std::vector<find_indexes_result_entry> result;
std::vector<data_pointer> data_cache;
std::map<dnet_raw_id, size_t, dnet_raw_id_less_than<> > result_map;
dnet_indexes tmp;
int err = -1;
dnet_id id = request_id;
size_t data_offset = 0;
char *data_start = reinterpret_cast<char *>(request->entries);
for (uint64_t i = 0; i < request->entries_count; ++i) {
dnet_indexes_request_entry &request_entry = *reinterpret_cast<dnet_indexes_request_entry *>(data_start + data_offset);
data_offset += sizeof(dnet_indexes_request_entry) + request_entry.size;
memcpy(id.id, request_entry.id.id, sizeof(id.id));
int ret = 0;
data_pointer data = sess.read(id, &ret);
data_cache.push_back(data);
if (ret) {
dnet_log(state->n, DNET_LOG_DEBUG, "%s: INDEXES_FIND, err: %d\n",
dnet_dump_id(&id), ret);
}
if (ret && unite) {
if (err == -1)
err = ret;
continue;
} else if (ret && intersection) {
return ret;
}
err = 0;
tmp.indexes.clear();
indexes_unpack(state->n, &id, data, &tmp, "process_find_indexes");
if (unite) {
for (size_t j = 0; j < tmp.indexes.size(); ++j) {
const auto &entry = tmp.indexes[j];
auto it = result_map.find(entry.index);
if (it == result_map.end()) {
it = result_map.insert(std::make_pair(entry.index, result.size())).first;
result.resize(result.size() + 1);
result.back().id = entry.index;
}
index_entry result_entry = { request_entry.id, entry.data };
result[it->second].indexes.push_back(result_entry);
}
} else if (intersection && i == 0) {
result.resize(tmp.indexes.size());
for (size_t j = 0; j < tmp.indexes.size(); ++j) {
auto &entry = result[j];
entry.id = tmp.indexes[j].index;
index_entry result_entry = { request_entry.id, tmp.indexes[j].data };
entry.indexes.push_back(result_entry);
}
} else if (intersection) {
// Remove all objects from result, which are not presented for this index
auto it = std::set_intersection(result.begin(), result.end(),
tmp.indexes.begin(), tmp.indexes.end(),
result.begin(),
dnet_raw_id_less_than<skip_data>());
result.resize(it - result.begin());
// Remove all objects from this index, which are not presented in result
std::set_intersection(tmp.indexes.begin(), tmp.indexes.end(),
result.begin(), result.end(),
tmp.indexes.begin(),
dnet_raw_id_less_than<skip_data>());
// As lists contain othe same objects - it's possible to add index data by one cycle
auto jt = tmp.indexes.begin();
for (auto kt = result.begin(); kt != result.end(); ++kt, ++jt) {
index_entry result_entry = { request_entry.id, jt->data };
kt->indexes.push_back(result_entry);
}
}
}
if (err != 0)
return err;
dnet_log(state->n, DNET_LOG_DEBUG, "%s: INDEXES_FIND: result of find: %zu objects\n",
dnet_dump_id(&id), result.size());
msgpack::sbuffer buffer;
msgpack::pack(&buffer, result);
if (!more) {
/*
* Unset NEED_ACK flag if and only if it is the last reply.
* We have to send positive reply in such case, also we don't want to send
* useless acknowledge packet.
*/
cmd->flags &= ~DNET_FLAGS_NEED_ACK;
}
dnet_cmd cmd_copy = *cmd;
dnet_setup_id(&cmd_copy.id, cmd->id.group_id, request_id.id);
dnet_send_reply(state, &cmd_copy, buffer.data(), buffer.size(), more);
return err;
}
/*!
* Update data-object table for certain secondary index.
*
* @index_data is what client provided
* @data is what was downloaded from the storage
*/
data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, dnet_indexes_request *request,
const data_pointer &index_data, const data_pointer &data, update_index_action action)
{
elliptics_timer timer;
raw_dnet_indexes indexes;
if (!data.empty())
indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table");
const int64_t timer_unpack = timer.restart();
// Construct index entry
raw_index_entry request_index;
memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id));
request_index.data.data = index_data.data();
request_index.data.size = index_data.size();
auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(), request_index);
const int64_t timer_lower_bound = timer.restart();
if (it != indexes.indexes.end() && it->index == request_index.index) {
// It's already there
if (action == insert_data) {
if (it->data == request_index.data) {
const int64_t timer_compare = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_compare));
// All's ok, keep it untouched
return data;
} else {
// Data is not correct, replace it by new one
it->data = request_index.data;
}
} else {
// Anyway, destroy it
indexes.indexes.erase(it);
}
} else {
// Index is not created yet
if (action == insert_data) {
// Just insert it
indexes.indexes.insert(it, 1, request_index);
} else {
const int64_t timer_compare = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_compare));
// All's ok, keep it untouched
return data;
}
}
const int64_t timer_update = timer.restart();
indexes.shard_id = request->shard_id;
indexes.shard_count = request->shard_count;
msgpack::sbuffer buffer;
msgpack::pack(&buffer, indexes);
const int64_t timer_pack = timer.restart();
data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size());
new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC));
new_buffer.write(buffer.data(), buffer.size());
const int64_t timer_write = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, update: %lld ms, pack: %lld ms, write: %lld ms\n",
id_str, data.size(), new_buffer.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_update), lld(timer_pack), lld(timer_write));
return std::move(new_buffer);
}