/*!
* Update data-object table for certain secondary index.
*
* @index_data is what client provided
* @data is what was downloaded from the storage
*/
data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, dnet_indexes_request *request,
const data_pointer &index_data, const data_pointer &data, update_index_action action)
{
dnet_indexes indexes;
if (!data.empty())
indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table");
// Construct index entry
index_entry request_index;
memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id));
request_index.data = index_data;
auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(),
request_index, dnet_raw_id_less_than<skip_data>());
if (it != indexes.indexes.end() && it->index == request_index.index) {
// It's already there
if (action == insert_data) {
if (it->data == request_index.data) {
// All's ok, keep it untouched
return data;
} else {
// Data is not correct, replace it by new one
it->data = request_index.data;
}
} else {
// Anyway, destroy it
indexes.indexes.erase(it);
}
} else {
// Index is not created yet
if (action == insert_data) {
// Just insert it
indexes.indexes.insert(it, 1, request_index);
} else {
// All's ok, keep it untouched
return data;
}
}
indexes.shard_id = request->shard_id;
indexes.shard_count = request->shard_count;
msgpack::sbuffer buffer;
msgpack::pack(&buffer, indexes);
data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size());
new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC));
new_buffer.write(buffer.data(), buffer.size());
return std::move(new_buffer);
}
/*!
* Update data-object table for certain secondary index.
*
* @index_data is what client provided
* @data is what was downloaded from the storage
*/
data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, const dnet_indexes_request *request,
const data_pointer &index_data, const data_pointer &data, uint32_t action,
std::vector<dnet_indexes_reply_entry> * &removed, uint32_t limit)
{
elliptics_timer timer;
dnet_indexes indexes;
if (!data.empty())
indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table");
const int64_t timer_unpack = timer.restart();
// Construct index entry
dnet_index_entry request_index;
memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id));
request_index.data = index_data;
dnet_current_time(&request_index.time);
auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(), request_index, dnet_raw_id_less_than<skip_data>());
const int64_t timer_lower_bound = timer.restart();
if (it != indexes.indexes.end() && it->index == request_index.index) {
// It's already there
if (action == DNET_INDEXES_FLAGS_INTERNAL_INSERT) {
// Item exists, update it's data and time if it's capped collection
if (!removed && it->data == request_index.data) {
const int64_t timer_compare = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_compare));
// All's ok, keep it untouched
return data;
}
it->data = request_index.data;
it->time = request_index.time;
} else {
// Anyway, destroy it
indexes.indexes.erase(it);
}
} else {
// Index is not created yet
if (action == DNET_INDEXES_FLAGS_INTERNAL_INSERT) {
// Remove extra elements from capped collection
if (removed && limit != 0 && indexes.indexes.size() + 1 > limit) {
dnet_indexes_reply_entry entry;
memset(&entry, 0, sizeof(entry));
auto position = it - indexes.indexes.begin();
while (indexes.indexes.size() + 1 > limit && !indexes.indexes.empty()) {
auto jt = std::min_element(indexes.indexes.begin(), indexes.indexes.end(), entry_time_less_than);
// jt will be removed, so it should be modified to be still valid
if (position > jt - indexes.indexes.begin())
--position;
memcpy(entry.id.id, jt->index.id, DNET_ID_SIZE);
entry.status = DNET_INDEXES_CAPPED_REMOVED;
indexes.indexes.erase(jt);
removed->push_back(entry);
}
it = indexes.indexes.begin() + position;
}
// And just insert new index
indexes.indexes.insert(it, 1, request_index);
} else {
const int64_t timer_compare = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_compare));
// All's ok, keep it untouched
return data;
}
}
const int64_t timer_update = timer.restart();
indexes.shard_id = request->shard_id;
indexes.shard_count = request->shard_count;
msgpack::sbuffer buffer;
msgpack::pack(&buffer, indexes);
const int64_t timer_pack = timer.restart();
data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size());
new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC));
new_buffer.write(buffer.data(), buffer.size());
const int64_t timer_write = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, update: %lld ms, pack: %lld ms, write: %lld ms\n",
id_str, data.size(), new_buffer.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_update), lld(timer_pack), lld(timer_write));
return std::move(new_buffer);
}
static data_pointer copy(const data_pointer &other)
{
return copy(other.data(), other.size());
}
MZC_INLINE MStringW::MStringW(data_pointer pData) :
m_pchData(pData->data())
{
}
MZC_INLINE void MStringW::SetData(data_pointer pData)
{
m_pchData = pData->data();
}
/*!
* Update data-object table for certain secondary index.
*
* @index_data is what client provided
* @data is what was downloaded from the storage
*/
data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, dnet_indexes_request *request,
const data_pointer &index_data, const data_pointer &data, update_index_action action)
{
elliptics_timer timer;
raw_dnet_indexes indexes;
if (!data.empty())
indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table");
const int64_t timer_unpack = timer.restart();
// Construct index entry
raw_index_entry request_index;
memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id));
request_index.data.data = index_data.data();
request_index.data.size = index_data.size();
auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(), request_index);
const int64_t timer_lower_bound = timer.restart();
if (it != indexes.indexes.end() && it->index == request_index.index) {
// It's already there
if (action == insert_data) {
if (it->data == request_index.data) {
const int64_t timer_compare = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_compare));
// All's ok, keep it untouched
return data;
} else {
// Data is not correct, replace it by new one
it->data = request_index.data;
}
} else {
// Anyway, destroy it
indexes.indexes.erase(it);
}
} else {
// Index is not created yet
if (action == insert_data) {
// Just insert it
indexes.indexes.insert(it, 1, request_index);
} else {
const int64_t timer_compare = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_compare));
// All's ok, keep it untouched
return data;
}
}
const int64_t timer_update = timer.restart();
indexes.shard_id = request->shard_id;
indexes.shard_count = request->shard_count;
msgpack::sbuffer buffer;
msgpack::pack(&buffer, indexes);
const int64_t timer_pack = timer.restart();
data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size());
new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC));
new_buffer.write(buffer.data(), buffer.size());
const int64_t timer_write = timer.restart();
DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
"unpack: %lld ms, lower_bound: %lld ms, update: %lld ms, pack: %lld ms, write: %lld ms\n",
id_str, data.size(), new_buffer.size(), lld(timer_unpack), lld(timer_lower_bound),
lld(timer_update), lld(timer_pack), lld(timer_write));
return std::move(new_buffer);
}