int process_internal_indexes_entry(dnet_node *node, const dnet_indexes_request &request, dnet_indexes_request_entry &entry, std::vector<dnet_indexes_reply_entry> * &removed) { elliptics_timer timer; local_session sess(node); dnet_id id; memset(&id, 0, sizeof(id)); memcpy(id.id, entry.id.id, DNET_ID_SIZE); const data_pointer entry_data = data_pointer::from_raw(entry.data, entry.size); if (node->log->log_level >= DNET_LOG_DEBUG) { char index_buffer[DNET_ID_SIZE * 2 + 1]; char object_buffer[DNET_DUMP_NUM * 2 + 1]; dnet_log(node, DNET_LOG_DEBUG, "INDEXES_INTERNAL: index: %s, object: %s, flags: 0x%llx\n", dnet_dump_id_len_raw(entry.id.id, DNET_ID_SIZE, index_buffer), dnet_dump_id_len_raw(request.id.id, DNET_DUMP_NUM, object_buffer), static_cast<unsigned long long>(entry.flags)); } uint32_t action = entry.flags & (DNET_INDEXES_FLAGS_INTERNAL_INSERT | DNET_INDEXES_FLAGS_INTERNAL_REMOVE | DNET_INDEXES_FLAGS_INTERNAL_REMOVE_ALL); const bool capped = entry.flags & DNET_INDEXES_FLAGS_INTERNAL_CAPPED_COLLECTION; if (!capped) removed = NULL; switch (action) { case DNET_INDEXES_FLAGS_INTERNAL_INSERT: case DNET_INDEXES_FLAGS_INTERNAL_REMOVE: break; case DNET_INDEXES_FLAGS_INTERNAL_REMOVE_ALL: { const int64_t timer_checks = timer.restart(); int err = sess.remove(id); const int64_t timer_remove = timer.restart(); DNET_DUMP_ID_LEN(id_str, &id, DNET_DUMP_NUM); typedef long long int lld; dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: id: %s, checks: %lld ms, remove: %lld ms\n", id_str, lld(timer_checks), lld(timer_remove)); removed = NULL; return err; } default: { dnet_log(node, DNET_LOG_ERROR, "INDEXES_INTERNAL: invalid flags: 0x%llx\n", static_cast<unsigned long long>(entry.flags)); removed = NULL; return -EINVAL; } } const int64_t timer_checks = timer.restart(); int err = 0; data_pointer data = sess.read(id, &err); const int64_t timer_read = timer.restart(); data_pointer new_data = convert_index_table(node, &id, &request, entry_data, data, action, removed, entry.limit); const int64_t timer_convert = timer.restart(); const bool data_equal = data == new_data; const int64_t timer_compare = timer.restart(); int64_t timer_write = 0; if (data_equal) { dnet_log(node, DNET_LOG_DEBUG, "INDEXES_INTERNAL: data is the same\n"); err = 0; } else { dnet_log(node, DNET_LOG_DEBUG, "INDEXES_INTERNAL: data is different\n"); err = sess.write(id, new_data); timer_write = timer.restart(); } DNET_DUMP_ID_LEN(id_str, &id, DNET_DUMP_NUM); typedef long long int lld; dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: id: %s, data size: %zu, new data size: %zu, checks: %lld ms, " "read: %lld ms, convert: %lld ms, compare: %lld ms, write: %lld ms\n", id_str, data.size(), new_data.size(), lld(timer_checks), lld(timer_read), lld(timer_convert), lld(timer_compare), lld(timer_write)); return err; }
int process_internal_indexes(dnet_net_state *state, dnet_cmd *cmd, dnet_indexes_request *request) { elliptics_timer timer; local_session sess(state->n); if (request->entries_count != 1) { return -EINVAL; } dnet_indexes_request_entry &entry = request->entries[0]; const data_pointer entry_data = data_pointer::from_raw(entry.data, entry.size); if (state->n->log->log_level >= DNET_LOG_DEBUG) { char index_buffer[DNET_DUMP_NUM * 2 + 1]; char object_buffer[DNET_DUMP_NUM * 2 + 1]; dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_INTERNAL: index: %s, object: %s\n", dnet_dump_id_len_raw(entry.id.id, DNET_DUMP_NUM, index_buffer), dnet_dump_id_len_raw(request->id.id, DNET_DUMP_NUM, object_buffer)); } update_index_action action; if (entry.flags & insert_data) { action = insert_data; } else if (entry.flags & remove_data) { action = remove_data; } else { dnet_log(state->n, DNET_LOG_ERROR, "INDEXES_INTERNAL: invalid flags: 0x%llx\n", static_cast<unsigned long long>(entry.flags)); return -EINVAL; } const int64_t timer_checks = timer.restart(); int err = 0; data_pointer data = sess.read(cmd->id, &err); const int64_t timer_read = timer.restart(); data_pointer new_data = convert_index_table(state->n, &cmd->id, request, entry_data, data, action); const int64_t timer_convert = timer.restart(); const bool data_equal = data == new_data; const int64_t timer_compare = timer.restart(); int64_t timer_write = timer_compare; if (data_equal) { dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_INTERNAL: data is the same\n"); err = 0; } else { dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_INTERNAL: data is different\n"); err = sess.write(cmd->id, new_data); timer_write = timer.restart(); } data_buffer buffer(sizeof(dnet_indexes_reply) + sizeof(dnet_indexes_reply_entry)); dnet_indexes_reply reply; dnet_indexes_reply_entry reply_entry; memset(&reply, 0, sizeof(reply)); memset(&reply_entry, 0, sizeof(reply_entry)); reply.entries_count = 1; reply_entry.id = entry.id; reply_entry.status = err; buffer.write(reply); buffer.write(reply_entry); data_pointer reply_data = std::move(buffer); if (!err) { cmd->flags &= (DNET_FLAGS_NEED_ACK | DNET_FLAGS_MORE); } dnet_send_reply(state, cmd, reply_data.data(), reply_data.size(), err ? 1 : 0); const int64_t timer_send = timer.restart(); DNET_DUMP_ID_LEN(id_str, &cmd->id, DNET_DUMP_NUM); typedef long long int lld; dnet_log(state->n, DNET_LOG_INFO, "INDEXES_INTERNAL: id: %s, data size: %zu, new data size: %zu, checks: %lld ms," "read: %lld ms, convert: %lld ms, write: %lld ms, send: %lld md\n", id_str, data.size(), new_data.size(), lld(timer_checks), lld(timer_read), lld(timer_convert), lld(timer_write), lld(timer_send)); return err; }
/*! * Update data-object table for certain secondary index. * * @index_data is what client provided * @data is what was downloaded from the storage */ data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, const dnet_indexes_request *request, const data_pointer &index_data, const data_pointer &data, uint32_t action, std::vector<dnet_indexes_reply_entry> * &removed, uint32_t limit) { elliptics_timer timer; dnet_indexes indexes; if (!data.empty()) indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table"); const int64_t timer_unpack = timer.restart(); // Construct index entry dnet_index_entry request_index; memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id)); request_index.data = index_data; dnet_current_time(&request_index.time); auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(), request_index, dnet_raw_id_less_than<skip_data>()); const int64_t timer_lower_bound = timer.restart(); if (it != indexes.indexes.end() && it->index == request_index.index) { // It's already there if (action == DNET_INDEXES_FLAGS_INTERNAL_INSERT) { // Item exists, update it's data and time if it's capped collection if (!removed && it->data == request_index.data) { const int64_t timer_compare = timer.restart(); DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM); typedef long long int lld; dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu," "unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n", id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound), lld(timer_compare)); // All's ok, keep it untouched return data; } it->data = request_index.data; it->time = request_index.time; } else { // Anyway, destroy it indexes.indexes.erase(it); } } else { // Index is not created yet if (action == DNET_INDEXES_FLAGS_INTERNAL_INSERT) { // Remove extra elements from capped collection if (removed && limit != 0 && indexes.indexes.size() + 1 > limit) { dnet_indexes_reply_entry entry; memset(&entry, 0, sizeof(entry)); auto position = it - indexes.indexes.begin(); while (indexes.indexes.size() + 1 > limit && !indexes.indexes.empty()) { auto jt = std::min_element(indexes.indexes.begin(), indexes.indexes.end(), entry_time_less_than); // jt will be removed, so it should be modified to be still valid if (position > jt - indexes.indexes.begin()) --position; memcpy(entry.id.id, jt->index.id, DNET_ID_SIZE); entry.status = DNET_INDEXES_CAPPED_REMOVED; indexes.indexes.erase(jt); removed->push_back(entry); } it = indexes.indexes.begin() + position; } // And just insert new index indexes.indexes.insert(it, 1, request_index); } else { const int64_t timer_compare = timer.restart(); DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM); typedef long long int lld; dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu," "unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n", id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound), lld(timer_compare)); // All's ok, keep it untouched return data; } } const int64_t timer_update = timer.restart(); indexes.shard_id = request->shard_id; indexes.shard_count = request->shard_count; msgpack::sbuffer buffer; msgpack::pack(&buffer, indexes); const int64_t timer_pack = timer.restart(); data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size()); new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC)); new_buffer.write(buffer.data(), buffer.size()); const int64_t timer_write = timer.restart(); DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM); typedef long long int lld; dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu," "unpack: %lld ms, lower_bound: %lld ms, update: %lld ms, pack: %lld ms, write: %lld ms\n", id_str, data.size(), new_buffer.size(), lld(timer_unpack), lld(timer_lower_bound), lld(timer_update), lld(timer_pack), lld(timer_write)); return std::move(new_buffer); }
/*! * Update data-object table for certain secondary index. * * @index_data is what client provided * @data is what was downloaded from the storage */ data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, dnet_indexes_request *request, const data_pointer &index_data, const data_pointer &data, update_index_action action) { elliptics_timer timer; raw_dnet_indexes indexes; if (!data.empty()) indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table"); const int64_t timer_unpack = timer.restart(); // Construct index entry raw_index_entry request_index; memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id)); request_index.data.data = index_data.data(); request_index.data.size = index_data.size(); auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(), request_index); const int64_t timer_lower_bound = timer.restart(); if (it != indexes.indexes.end() && it->index == request_index.index) { // It's already there if (action == insert_data) { if (it->data == request_index.data) { const int64_t timer_compare = timer.restart(); DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM); typedef long long int lld; dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu," "unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n", id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound), lld(timer_compare)); // All's ok, keep it untouched return data; } else { // Data is not correct, replace it by new one it->data = request_index.data; } } else { // Anyway, destroy it indexes.indexes.erase(it); } } else { // Index is not created yet if (action == insert_data) { // Just insert it indexes.indexes.insert(it, 1, request_index); } else { const int64_t timer_compare = timer.restart(); DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM); typedef long long int lld; dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu," "unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n", id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound), lld(timer_compare)); // All's ok, keep it untouched return data; } } const int64_t timer_update = timer.restart(); indexes.shard_id = request->shard_id; indexes.shard_count = request->shard_count; msgpack::sbuffer buffer; msgpack::pack(&buffer, indexes); const int64_t timer_pack = timer.restart(); data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size()); new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC)); new_buffer.write(buffer.data(), buffer.size()); const int64_t timer_write = timer.restart(); DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM); typedef long long int lld; dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu," "unpack: %lld ms, lower_bound: %lld ms, update: %lld ms, pack: %lld ms, write: %lld ms\n", id_str, data.size(), new_buffer.size(), lld(timer_unpack), lld(timer_lower_bound), lld(timer_update), lld(timer_pack), lld(timer_write)); return std::move(new_buffer); }