int process_internal_indexes_entry(dnet_node *node, const dnet_indexes_request &request,
dnet_indexes_request_entry &entry, std::vector<dnet_indexes_reply_entry> * &removed)
{
elliptics_timer timer;
local_session sess(node);
dnet_id id;
memset(&id, 0, sizeof(id));
memcpy(id.id, entry.id.id, DNET_ID_SIZE);
const data_pointer entry_data = data_pointer::from_raw(entry.data, entry.size);
if (node->log->log_level >= DNET_LOG_DEBUG) {
char index_buffer[DNET_ID_SIZE * 2 + 1];
char object_buffer[DNET_DUMP_NUM * 2 + 1];
dnet_log(node, DNET_LOG_DEBUG, "INDEXES_INTERNAL: index: %s, object: %s, flags: 0x%llx\n",
dnet_dump_id_len_raw(entry.id.id, DNET_ID_SIZE, index_buffer),
dnet_dump_id_len_raw(request.id.id, DNET_DUMP_NUM, object_buffer),
static_cast<unsigned long long>(entry.flags));
}
uint32_t action = entry.flags & (DNET_INDEXES_FLAGS_INTERNAL_INSERT
| DNET_INDEXES_FLAGS_INTERNAL_REMOVE | DNET_INDEXES_FLAGS_INTERNAL_REMOVE_ALL);
const bool capped = entry.flags & DNET_INDEXES_FLAGS_INTERNAL_CAPPED_COLLECTION;
if (!capped)
removed = NULL;
switch (action) {
case DNET_INDEXES_FLAGS_INTERNAL_INSERT:
case DNET_INDEXES_FLAGS_INTERNAL_REMOVE:
break;
case DNET_INDEXES_FLAGS_INTERNAL_REMOVE_ALL: {
const int64_t timer_checks = timer.restart();
int err = sess.remove(id);
const int64_t timer_remove = timer.restart();
DNET_DUMP_ID_LEN(id_str, &id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: id: %s, checks: %lld ms, remove: %lld ms\n",
id_str, lld(timer_checks), lld(timer_remove));
removed = NULL;
return err;
}
default: {
dnet_log(node, DNET_LOG_ERROR, "INDEXES_INTERNAL: invalid flags: 0x%llx\n",
static_cast<unsigned long long>(entry.flags));
removed = NULL;
return -EINVAL;
}
}
const int64_t timer_checks = timer.restart();
int err = 0;
data_pointer data = sess.read(id, &err);
const int64_t timer_read = timer.restart();
data_pointer new_data = convert_index_table(node, &id, &request, entry_data, data, action, removed, entry.limit);
const int64_t timer_convert = timer.restart();
const bool data_equal = data == new_data;
const int64_t timer_compare = timer.restart();
int64_t timer_write = 0;
if (data_equal) {
dnet_log(node, DNET_LOG_DEBUG, "INDEXES_INTERNAL: data is the same\n");
err = 0;
} else {
dnet_log(node, DNET_LOG_DEBUG, "INDEXES_INTERNAL: data is different\n");
err = sess.write(id, new_data);
timer_write = timer.restart();
}
DNET_DUMP_ID_LEN(id_str, &id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: id: %s, data size: %zu, new data size: %zu, checks: %lld ms, "
"read: %lld ms, convert: %lld ms, compare: %lld ms, write: %lld ms\n",
id_str, data.size(), new_data.size(), lld(timer_checks), lld(timer_read),
lld(timer_convert), lld(timer_compare), lld(timer_write));
return err;
}
int process_internal_indexes(dnet_net_state *state, dnet_cmd *cmd, dnet_indexes_request *request)
{
elliptics_timer timer;
local_session sess(state->n);
if (request->entries_count != 1) {
return -EINVAL;
}
dnet_indexes_request_entry &entry = request->entries[0];
const data_pointer entry_data = data_pointer::from_raw(entry.data, entry.size);
if (state->n->log->log_level >= DNET_LOG_DEBUG) {
char index_buffer[DNET_DUMP_NUM * 2 + 1];
char object_buffer[DNET_DUMP_NUM * 2 + 1];
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_INTERNAL: index: %s, object: %s\n",
dnet_dump_id_len_raw(entry.id.id, DNET_DUMP_NUM, index_buffer),
dnet_dump_id_len_raw(request->id.id, DNET_DUMP_NUM, object_buffer));
}
update_index_action action;
if (entry.flags & insert_data) {
action = insert_data;
} else if (entry.flags & remove_data) {
action = remove_data;
} else {
dnet_log(state->n, DNET_LOG_ERROR, "INDEXES_INTERNAL: invalid flags: 0x%llx\n",
static_cast<unsigned long long>(entry.flags));
return -EINVAL;
}
const int64_t timer_checks = timer.restart();
int err = 0;
data_pointer data = sess.read(cmd->id, &err);
const int64_t timer_read = timer.restart();
data_pointer new_data = convert_index_table(state->n, &cmd->id, request, entry_data, data, action);
const int64_t timer_convert = timer.restart();
const bool data_equal = data == new_data;
const int64_t timer_compare = timer.restart();
int64_t timer_write = timer_compare;
if (data_equal) {
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_INTERNAL: data is the same\n");
err = 0;
} else {
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_INTERNAL: data is different\n");
err = sess.write(cmd->id, new_data);
timer_write = timer.restart();
}
data_buffer buffer(sizeof(dnet_indexes_reply) + sizeof(dnet_indexes_reply_entry));
dnet_indexes_reply reply;
dnet_indexes_reply_entry reply_entry;
memset(&reply, 0, sizeof(reply));
memset(&reply_entry, 0, sizeof(reply_entry));
reply.entries_count = 1;
reply_entry.id = entry.id;
reply_entry.status = err;
buffer.write(reply);
buffer.write(reply_entry);
data_pointer reply_data = std::move(buffer);
if (!err) {
cmd->flags &= (DNET_FLAGS_NEED_ACK | DNET_FLAGS_MORE);
}
dnet_send_reply(state, cmd, reply_data.data(), reply_data.size(), err ? 1 : 0);
const int64_t timer_send = timer.restart();
DNET_DUMP_ID_LEN(id_str, &cmd->id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(state->n, DNET_LOG_INFO, "INDEXES_INTERNAL: id: %s, data size: %zu, new data size: %zu, checks: %lld ms,"
"read: %lld ms, convert: %lld ms, write: %lld ms, send: %lld md\n",
id_str, data.size(), new_data.size(), lld(timer_checks), lld(timer_read),
lld(timer_convert), lld(timer_write), lld(timer_send));
return err;
}
/*!
* Update data-object table for certain secondary index.
*
* @index_data is what client provided
* @data is what was downloaded from the storage
*/
data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, const dnet_indexes_request *request,
const data_pointer &index_data, const data_pointer &data, uint32_t action,
std::vector<dnet_indexes_reply_entry> * &removed, uint32_t limit)
{
elliptics_timer timer;
dnet_indexes indexes;
if (!data.empty())
indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table");
const int64_t timer_unpack = timer.restart();
// Construct index entry
dnet_index_entry request_index;
memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id));
request_index.data = index_data;
dnet_current_time(&request_index.time);
auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(), request_index, dnet_raw_id_less_than<skip_data>());
const int64_t timer_lower_bound = timer.restart();
if (it != indexes.indexes.end() && it->index == request_index.index) {
// It's already there
if (action == DNET_INDEXES_FLAGS_INTERNAL_INSERT) {
// Item exists, update it's data and time if it's capped collection
if (!removed && it->data == request_index.data) {
const int64_t timer_compare = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_compare));
// All's ok, keep it untouched
return data;
}
it->data = request_index.data;
it->time = request_index.time;
} else {
// Anyway, destroy it
indexes.indexes.erase(it);
}
} else {
// Index is not created yet
if (action == DNET_INDEXES_FLAGS_INTERNAL_INSERT) {
// Remove extra elements from capped collection
if (removed && limit != 0 && indexes.indexes.size() + 1 > limit) {
dnet_indexes_reply_entry entry;
memset(&entry, 0, sizeof(entry));
auto position = it - indexes.indexes.begin();
while (indexes.indexes.size() + 1 > limit && !indexes.indexes.empty()) {
auto jt = std::min_element(indexes.indexes.begin(), indexes.indexes.end(), entry_time_less_than);
// jt will be removed, so it should be modified to be still valid
if (position > jt - indexes.indexes.begin())
--position;
memcpy(entry.id.id, jt->index.id, DNET_ID_SIZE);
entry.status = DNET_INDEXES_CAPPED_REMOVED;
indexes.indexes.erase(jt);
removed->push_back(entry);
}
it = indexes.indexes.begin() + position;
}
// And just insert new index
indexes.indexes.insert(it, 1, request_index);
} else {
const int64_t timer_compare = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_compare));
// All's ok, keep it untouched
return data;
}
}
const int64_t timer_update = timer.restart();
indexes.shard_id = request->shard_id;
indexes.shard_count = request->shard_count;
msgpack::sbuffer buffer;
msgpack::pack(&buffer, indexes);
const int64_t timer_pack = timer.restart();
data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size());
new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC));
new_buffer.write(buffer.data(), buffer.size());
const int64_t timer_write = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, update: %lld ms, pack: %lld ms, write: %lld ms\n",
id_str, data.size(), new_buffer.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_update), lld(timer_pack), lld(timer_write));
return std::move(new_buffer);
}
/*!
* Update data-object table for certain secondary index.
*
* @index_data is what client provided
* @data is what was downloaded from the storage
*/
data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, dnet_indexes_request *request,
const data_pointer &index_data, const data_pointer &data, update_index_action action)
{
elliptics_timer timer;
raw_dnet_indexes indexes;
if (!data.empty())
indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table");
const int64_t timer_unpack = timer.restart();
// Construct index entry
raw_index_entry request_index;
memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id));
request_index.data.data = index_data.data();
request_index.data.size = index_data.size();
auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(), request_index);
const int64_t timer_lower_bound = timer.restart();
if (it != indexes.indexes.end() && it->index == request_index.index) {
// It's already there
if (action == insert_data) {
if (it->data == request_index.data) {
const int64_t timer_compare = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_compare));
// All's ok, keep it untouched
return data;
} else {
// Data is not correct, replace it by new one
it->data = request_index.data;
}
} else {
// Anyway, destroy it
indexes.indexes.erase(it);
}
} else {
// Index is not created yet
if (action == insert_data) {
// Just insert it
indexes.indexes.insert(it, 1, request_index);
} else {
const int64_t timer_compare = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_compare));
// All's ok, keep it untouched
return data;
}
}
const int64_t timer_update = timer.restart();
indexes.shard_id = request->shard_id;
indexes.shard_count = request->shard_count;
msgpack::sbuffer buffer;
msgpack::pack(&buffer, indexes);
const int64_t timer_pack = timer.restart();
data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size());
new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC));
new_buffer.write(buffer.data(), buffer.size());
const int64_t timer_write = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, update: %lld ms, pack: %lld ms, write: %lld ms\n",
id_str, data.size(), new_buffer.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_update), lld(timer_pack), lld(timer_write));
return std::move(new_buffer);
}
