int process_internal_indexes(dnet_net_state *state, dnet_cmd *cmd, dnet_indexes_request *request)
{
if (request->entries_count == 0) {
return -EINVAL;
}
data_buffer buffer(sizeof(dnet_indexes_reply) + request->entries_count * 2 * sizeof(dnet_indexes_reply_entry));
dnet_indexes_reply reply;
memset(&reply, 0, sizeof(reply));
buffer.write(reply);
size_t entries_count = request->entries_count;
dnet_indexes_reply_entry reply_entry;
memset(&reply_entry, 0, sizeof(reply_entry));
dnet_indexes_reply_entry reply_entry_removed;
memset(&reply_entry, 0, sizeof(reply_entry_removed));
std::vector<dnet_indexes_reply_entry> removed;
int err = -1;
for (uint64_t i = 0; i < request->entries_count; ++i) {
dnet_indexes_request_entry &entry = request->entries[i];
removed.clear();
auto *tmp = &removed;
int ret = process_internal_indexes_entry(state->n, *request, entry, tmp);
reply_entry.id = entry.id;
reply_entry.status = ret;
buffer.write(reply_entry);
if (tmp) {
entries_count += removed.size();
for (auto it = removed.begin(); it != removed.end(); ++it)
buffer.write(*it);
}
if (!ret) {
err = 0;
} else if (err == -1) {
err = ret;
}
}
if (!err) {
data_pointer reply_data = std::move(buffer);
reply_data.data<dnet_indexes_reply>()->entries_count = entries_count;
cmd->flags &= (DNET_FLAGS_NEED_ACK | DNET_FLAGS_MORE);
dnet_send_reply(state, cmd, reply_data.data(), reply_data.size(), 0);
return 0;
}
return err;
}
static int smack_backend_write(struct smack_backend *s, void *state, struct dnet_cmd *cmd, void *data)
{
struct dnet_node *n = dnet_get_node_from_state(state);
int err;
struct index idx;
struct dnet_io_attr *io = data;
struct dnet_file_info *info;
struct dnet_addr_attr *a;
dnet_convert_io_attr(io);
data += sizeof(struct dnet_io_attr);
smack_setup_idx(&idx, io->id);
idx.data_size = io->size;
err = smack_write(s->smack, &idx, data);
if (err < 0)
goto err_out_exit;
#if 0
err = smack_backend_lookup_raw(s, &idx, state, cmd);
if (err)
goto err_out_exit;
#else
a = malloc(sizeof(struct dnet_addr_attr) + sizeof(struct dnet_file_info));
if (!a) {
err = -ENOMEM;
goto err_out_exit;
}
info = (struct dnet_file_info *)(a + 1);
dnet_fill_addr_attr(n, a);
dnet_convert_addr_attr(a);
memset(info, 0, sizeof(struct dnet_file_info));
dnet_convert_file_info(info);
err = dnet_send_reply(state, cmd, a, sizeof(struct dnet_addr_attr) + sizeof(struct dnet_file_info), 0);
#endif
dnet_backend_log(DNET_LOG_INFO, "%s: SMACK: : WRITE: Ok: offset: %llu, size: %llu.\n",
dnet_dump_id(&cmd->id), (unsigned long long)io->offset, (unsigned long long)io->size);
err_out_exit:
return err;
}
int process_find_indexes(dnet_net_state *state, dnet_cmd *cmd, dnet_indexes_request *request)
{
local_session sess(state->n);
const bool intersection = request->flags & DNET_INDEXES_FLAGS_INTERSECT;
const bool unite = request->flags & DNET_INDEXES_FLAGS_UNITE;
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_FIND: indexes count: %u, flags: %llu\n",
(unsigned) request->entries_count, (unsigned long long) request->flags);
if (intersection && unite) {
return -ENOTSUP;
}
std::vector<find_indexes_result_entry> result;
std::map<dnet_raw_id, size_t, dnet_raw_id_less_than<> > result_map;
dnet_indexes tmp;
int err = -1;
dnet_id id = cmd->id;
size_t data_offset = 0;
char *data_start = reinterpret_cast<char *>(request->entries);
for (uint64_t i = 0; i < request->entries_count; ++i) {
dnet_indexes_request_entry &request_entry = *reinterpret_cast<dnet_indexes_request_entry *>(data_start + data_offset);
data_offset += sizeof(dnet_indexes_request_entry) + request_entry.size;
memcpy(id.id, request_entry.id.id, sizeof(id.id));
int ret = 0;
data_pointer data = sess.read(id, &ret);
if (ret) {
dnet_log(state->n, DNET_LOG_DEBUG, "%s: INDEXES_FIND, err: %d\n",
dnet_dump_id(&id), ret);
}
if (ret && unite) {
if (err != -1)
err = ret;
continue;
} else if (ret && intersection) {
return ret;
}
err = 0;
tmp.indexes.clear();
indexes_unpack(state->n, &id, data, &tmp, "process_find_indexes");
if (unite) {
for (size_t j = 0; j < tmp.indexes.size(); ++j) {
const index_entry &entry = tmp.indexes[j];
auto it = result_map.find(entry.index);
if (it == result_map.end()) {
it = result_map.insert(std::make_pair(entry.index, result.size())).first;
result.resize(result.size() + 1);
result.back().id = entry.index;
}
result[it->second].indexes.emplace_back(request_entry.id, entry.data);
}
} else if (intersection && i == 0) {
result.resize(tmp.indexes.size());
for (size_t j = 0; j < tmp.indexes.size(); ++j) {
find_indexes_result_entry &entry = result[j];
entry.id = tmp.indexes[j].index;
entry.indexes.emplace_back(
request_entry.id,
tmp.indexes[j].data);
}
} else if (intersection) {
// Remove all objects from result, which are not presented for this index
auto it = std::set_intersection(result.begin(), result.end(),
tmp.indexes.begin(), tmp.indexes.end(),
result.begin(),
dnet_raw_id_less_than<skip_data>());
result.resize(it - result.begin());
// Remove all objects from this index, which are not presented in result
std::set_intersection(tmp.indexes.begin(), tmp.indexes.end(),
result.begin(), result.end(),
tmp.indexes.begin(),
dnet_raw_id_less_than<skip_data>());
// As lists contain othe same objects - it's possible to add index data by one cycle
auto jt = tmp.indexes.begin();
for (auto kt = result.begin(); kt != result.end(); ++kt, ++jt) {
kt->indexes.emplace_back(request_entry.id, jt->data);
}
}
}
// if (err != 0)
// return err;
dnet_log(state->n, DNET_LOG_DEBUG, "%s: INDEXES_FIND: result of find: %zu objects\n",
dnet_dump_id(&id), result.size());
msgpack::sbuffer buffer;
msgpack::pack(&buffer, result);
cmd->flags &= ~DNET_FLAGS_NEED_ACK;
dnet_send_reply(state, cmd, buffer.data(), buffer.size(), 0);
return err;
}
int process_internal_indexes(dnet_net_state *state, dnet_cmd *cmd, dnet_indexes_request *request)
{
local_session sess(state->n);
if (request->entries_count != 1) {
return -EINVAL;
}
dnet_indexes_request_entry &entry = request->entries[0];
const data_pointer entry_data = data_pointer::from_raw(entry.data, entry.size);
if (state->n->log->log_level >= DNET_LOG_DEBUG) {
char index_buffer[DNET_DUMP_NUM * 2 + 1];
char object_buffer[DNET_DUMP_NUM * 2 + 1];
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_INTERNAL: index: %s, object: %s\n",
dnet_dump_id_len_raw(entry.id.id, DNET_DUMP_NUM, index_buffer),
dnet_dump_id_len_raw(request->id.id, DNET_DUMP_NUM, object_buffer));
}
update_index_action action;
if (entry.flags & insert_data) {
action = insert_data;
} else if (entry.flags & remove_data) {
action = remove_data;
} else {
dnet_log(state->n, DNET_LOG_ERROR, "INDEXES_INTERNAL: invalid flags: 0x%llx\n",
static_cast<unsigned long long>(entry.flags));
return -EINVAL;
}
int err = 0;
data_pointer data = sess.read(cmd->id, &err);
data_pointer new_data = convert_index_table(state->n, &cmd->id, request, entry_data, data, action);
if (data == new_data) {
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_INTERNAL: data is the same\n");
err = 0;
} else {
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_INTERNAL: data is different\n");
err = sess.write(cmd->id, new_data);
}
data_buffer buffer(sizeof(dnet_indexes_reply) + sizeof(dnet_indexes_reply_entry));
dnet_indexes_reply reply;
dnet_indexes_reply_entry reply_entry;
memset(&reply, 0, sizeof(reply));
memset(&reply_entry, 0, sizeof(reply_entry));
reply.entries_count = 1;
reply_entry.id = entry.id;
reply_entry.status = err;
buffer.write(reply);
buffer.write(reply_entry);
data_pointer reply_data = std::move(buffer);
if (!err) {
cmd->flags &= (DNET_FLAGS_NEED_ACK | DNET_FLAGS_MORE);
}
dnet_send_reply(state, cmd, reply_data.data(), reply_data.size(), err ? 1 : 0);
return err;
}
int process_find_indexes(dnet_net_state *state, dnet_cmd *cmd, const dnet_id &request_id, dnet_indexes_request *request, bool more)
{
local_session sess(state->n);
const bool intersection = request->flags & DNET_INDEXES_FLAGS_INTERSECT;
const bool unite = request->flags & DNET_INDEXES_FLAGS_UNITE;
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_FIND: indexes count: %u, flags: %llu, more: %d\n",
(unsigned) request->entries_count, (unsigned long long) request->flags, int(more));
if ((intersection && unite) || !(intersection || unite)) {
return -EINVAL;
}
std::vector<find_indexes_result_entry> result;
std::vector<data_pointer> data_cache;
std::map<dnet_raw_id, size_t, dnet_raw_id_less_than<> > result_map;
dnet_indexes tmp;
int err = -1;
dnet_id id = request_id;
size_t data_offset = 0;
char *data_start = reinterpret_cast<char *>(request->entries);
for (uint64_t i = 0; i < request->entries_count; ++i) {
dnet_indexes_request_entry &request_entry = *reinterpret_cast<dnet_indexes_request_entry *>(data_start + data_offset);
data_offset += sizeof(dnet_indexes_request_entry) + request_entry.size;
memcpy(id.id, request_entry.id.id, sizeof(id.id));
int ret = 0;
data_pointer data = sess.read(id, &ret);
data_cache.push_back(data);
if (ret) {
dnet_log(state->n, DNET_LOG_DEBUG, "%s: INDEXES_FIND, err: %d\n",
dnet_dump_id(&id), ret);
}
if (ret && unite) {
if (err == -1)
err = ret;
continue;
} else if (ret && intersection) {
return ret;
}
err = 0;
tmp.indexes.clear();
indexes_unpack(state->n, &id, data, &tmp, "process_find_indexes");
if (unite) {
for (size_t j = 0; j < tmp.indexes.size(); ++j) {
const auto &entry = tmp.indexes[j];
auto it = result_map.find(entry.index);
if (it == result_map.end()) {
it = result_map.insert(std::make_pair(entry.index, result.size())).first;
result.resize(result.size() + 1);
result.back().id = entry.index;
}
index_entry result_entry = { request_entry.id, entry.data };
result[it->second].indexes.push_back(result_entry);
}
} else if (intersection && i == 0) {
result.resize(tmp.indexes.size());
for (size_t j = 0; j < tmp.indexes.size(); ++j) {
auto &entry = result[j];
entry.id = tmp.indexes[j].index;
index_entry result_entry = { request_entry.id, tmp.indexes[j].data };
entry.indexes.push_back(result_entry);
}
} else if (intersection) {
// Remove all objects from result, which are not presented for this index
auto it = std::set_intersection(result.begin(), result.end(),
tmp.indexes.begin(), tmp.indexes.end(),
result.begin(),
dnet_raw_id_less_than<skip_data>());
result.resize(it - result.begin());
// Remove all objects from this index, which are not presented in result
std::set_intersection(tmp.indexes.begin(), tmp.indexes.end(),
result.begin(), result.end(),
tmp.indexes.begin(),
dnet_raw_id_less_than<skip_data>());
// As lists contain othe same objects - it's possible to add index data by one cycle
auto jt = tmp.indexes.begin();
for (auto kt = result.begin(); kt != result.end(); ++kt, ++jt) {
index_entry result_entry = { request_entry.id, jt->data };
kt->indexes.push_back(result_entry);
}
}
}
if (err != 0)
return err;
dnet_log(state->n, DNET_LOG_DEBUG, "%s: INDEXES_FIND: result of find: %zu objects\n",
dnet_dump_id(&id), result.size());
msgpack::sbuffer buffer;
msgpack::pack(&buffer, result);
if (!more) {
/*
* Unset NEED_ACK flag if and only if it is the last reply.
* We have to send positive reply in such case, also we don't want to send
* useless acknowledge packet.
*/
cmd->flags &= ~DNET_FLAGS_NEED_ACK;
}
dnet_cmd cmd_copy = *cmd;
dnet_setup_id(&cmd_copy.id, cmd->id.group_id, request_id.id);
dnet_send_reply(state, &cmd_copy, buffer.data(), buffer.size(), more);
return err;
}
int dnet_cmd_bulk_check(struct dnet_net_state *orig, struct dnet_cmd *cmd, void *data)
{
struct dnet_bulk_id *ids = (struct dnet_bulk_id *)data;
struct dnet_meta_container mc;
struct dnet_meta_update mu;
struct dnet_id raw;
int i;
int err = 0;
int num;
if (!(cmd->size % sizeof(struct dnet_bulk_id))) {
num = cmd->size / sizeof(struct dnet_bulk_id);
dnet_log(orig->n, DNET_LOG_DEBUG, "BULK: received %d entries\n", num);
for (i = 0; i < num; ++i) {
/* Send empty reply every DNET_BULK_CHECK_PING records to prevent timeout */
if (i % DNET_BULK_CHECK_PING == 0 && i > 0) {
dnet_send_reply(orig, cmd, NULL, 0, 1);
}
dnet_log(orig->n, DNET_LOG_DEBUG, "BULK: processing ID %s\n", dnet_dump_id_str(ids[i].id.id));
mc.data = NULL;
dnet_setup_id(&mc.id, 0, ids[i].id.id);
err = orig->n->cb->meta_read(orig->n->cb->command_private, &ids[i].id, &mc.data);
if (mc.data) {
mc.size = err;
dnet_log(orig->n, DNET_LOG_DEBUG, "BULK: %d bytes of metadata found, searching for META_UPDATE group_id=%d\n",
mc.size, orig->n->st->idc->group->group_id);
if (dnet_get_meta_update(orig->n, &mc, &mu))
{
dnet_convert_meta_update(&ids[i].last_update);
dnet_log(orig->n, DNET_LOG_DEBUG, "BULK: mu.tsec=%lu, mu.tnsec=%lu, mu.flags=%02lx\n",
(unsigned long)mu.tm.tsec, (unsigned long)mu.tm.tnsec, (unsigned long)mu.flags);
dnet_log(orig->n, DNET_LOG_DEBUG,
"BULK: last_update.tsec=%lu, last_update.tnsec=%lu, last_update.flags=%02lx\n",
(unsigned long)ids[i].last_update.tm.tsec, (unsigned long)ids[i].last_update.tm.tnsec,
(unsigned long)ids[i].last_update.flags);
if ((mu.flags & DNET_IO_FLAGS_REMOVED) || (mu.tm.tsec < ids[i].last_update.tm.tsec) ||
((mu.tm.tnsec < ids[i].last_update.tm.tnsec) && (mu.tm.tsec == ids[i].last_update.tm.tsec))) {
err = 0;
} else {
/* File is not needed to be updated */
dnet_setup_id(&raw, orig->n->id.group_id, ids[i].id.id);
err = dnet_stat_local(orig, &raw);
if (err) {
/* File was not found in the storage */
mu.tm.tsec = 1;
mu.flags = 0;
} else {
err = dnet_meta_update_check_status(orig->n, &mc);
if (err) {
dnet_log(orig->n, DNET_LOG_ERROR,
"BULK: %s: couldn't update meta CHECK_STATUS err: %d\n",
dnet_dump_id_str(ids[i].id.id), err);
}
}
}
memcpy(&ids[i].last_update, &mu, sizeof(struct dnet_meta_update));
dnet_convert_meta_update(&ids[i].last_update);
}
free(mc.data);
} else {
/* Meta is not present - set timestamp to very old one */
dnet_convert_meta_update(&ids[i].last_update);
ids[i].last_update.tm.tsec = 1;
ids[i].last_update.flags = 0;
dnet_convert_meta_update(&ids[i].last_update);
}
}
} else {
dnet_log(orig->n, DNET_LOG_ERROR, "BULK: received corrupted data, size = %llu, sizeof(dnet_bulk_id) = %zu\n",
(unsigned long long)cmd->size, sizeof(struct dnet_bulk_id));
err = -1;
goto err_out_exit;
}
return dnet_send_reply(orig, cmd, data, sizeof(struct dnet_bulk_id) * num, 0);
err_out_exit:
return err;
}
int process_internal_indexes(dnet_net_state *state, dnet_cmd *cmd, dnet_indexes_request *request)
{
elliptics_timer timer;
local_session sess(state->n);
if (request->entries_count != 1) {
return -EINVAL;
}
dnet_indexes_request_entry &entry = request->entries[0];
const data_pointer entry_data = data_pointer::from_raw(entry.data, entry.size);
if (state->n->log->log_level >= DNET_LOG_DEBUG) {
char index_buffer[DNET_DUMP_NUM * 2 + 1];
char object_buffer[DNET_DUMP_NUM * 2 + 1];
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_INTERNAL: index: %s, object: %s\n",
dnet_dump_id_len_raw(entry.id.id, DNET_DUMP_NUM, index_buffer),
dnet_dump_id_len_raw(request->id.id, DNET_DUMP_NUM, object_buffer));
}
update_index_action action;
if (entry.flags & insert_data) {
action = insert_data;
} else if (entry.flags & remove_data) {
action = remove_data;
} else {
dnet_log(state->n, DNET_LOG_ERROR, "INDEXES_INTERNAL: invalid flags: 0x%llx\n",
static_cast<unsigned long long>(entry.flags));
return -EINVAL;
}
const int64_t timer_checks = timer.restart();
int err = 0;
data_pointer data = sess.read(cmd->id, &err);
const int64_t timer_read = timer.restart();
data_pointer new_data = convert_index_table(state->n, &cmd->id, request, entry_data, data, action);
const int64_t timer_convert = timer.restart();
const bool data_equal = data == new_data;
const int64_t timer_compare = timer.restart();
int64_t timer_write = timer_compare;
if (data_equal) {
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_INTERNAL: data is the same\n");
err = 0;
} else {
dnet_log(state->n, DNET_LOG_DEBUG, "INDEXES_INTERNAL: data is different\n");
err = sess.write(cmd->id, new_data);
timer_write = timer.restart();
}
data_buffer buffer(sizeof(dnet_indexes_reply) + sizeof(dnet_indexes_reply_entry));
dnet_indexes_reply reply;
dnet_indexes_reply_entry reply_entry;
memset(&reply, 0, sizeof(reply));
memset(&reply_entry, 0, sizeof(reply_entry));
reply.entries_count = 1;
reply_entry.id = entry.id;
reply_entry.status = err;
buffer.write(reply);
buffer.write(reply_entry);
data_pointer reply_data = std::move(buffer);
if (!err) {
cmd->flags &= (DNET_FLAGS_NEED_ACK | DNET_FLAGS_MORE);
}
dnet_send_reply(state, cmd, reply_data.data(), reply_data.size(), err ? 1 : 0);
const int64_t timer_send = timer.restart();
DNET_DUMP_ID_LEN(id_str, &cmd->id, DNET_DUMP_NUM);
typedef long long int lld;
dnet_log(state->n, DNET_LOG_INFO, "INDEXES_INTERNAL: id: %s, data size: %zu, new data size: %zu, checks: %lld ms,"
"read: %lld ms, convert: %lld ms, write: %lld ms, send: %lld md\n",
id_str, data.size(), new_data.size(), lld(timer_checks), lld(timer_read),
lld(timer_convert), lld(timer_write), lld(timer_send));
return err;
}
