int dnet_trans_alloc_send(struct dnet_session *s, struct dnet_trans_control *ctl)
{
struct dnet_node *n = s->node;
struct dnet_net_state *st;
struct dnet_addr *addr = NULL;
int err;
if (dnet_session_get_cflags(s) & DNET_FLAGS_DIRECT) {
st = dnet_state_search_by_addr(n, &s->direct_addr);
addr = &s->direct_addr;
} else if(dnet_session_get_cflags(s) & DNET_FLAGS_FORWARD) {
st = dnet_state_search_by_addr(n, &s->forward_addr);
addr = &s->forward_addr;
}else {
st = dnet_state_get_first(n, &ctl->id);
}
if (!st) {
dnet_log(n, DNET_LOG_ERROR, "%s: direct: %d, direct-addr: %s, forward: %d: trans_send: could not find network state for address",
dnet_dump_id(&ctl->id),
!!(dnet_session_get_cflags(s) & DNET_FLAGS_DIRECT), dnet_addr_string(&s->direct_addr),
!!(dnet_session_get_cflags(s) & DNET_FLAGS_FORWARD));
err = dnet_trans_send_fail(s, addr, ctl, -ENXIO, 1);
} else {
err = dnet_trans_alloc_send_state(s, st, ctl);
dnet_state_put(st);
}
return err;
}
static int dnet_merge_direct(struct dnet_session *s, struct dnet_meta_container *mc)
{
struct dnet_node *n = s->node;
struct dnet_net_state *base;
int cflags = 0;
int err;
dnet_log(n, DNET_LOG_DEBUG, "in dnet_merge_direct mc->size = %u\n", mc->size);
base = dnet_node_state(n);
if (!base) {
err = -ENOENT;
goto err_out_exit;
}
err = n->cb->send(base, n->cb->command_private, &mc->id);
dnet_log(n, DNET_LOG_DEBUG, "in dnet_merge_direct after n->cb->send err = %d\n\n", err);
if (err < 0)
goto err_out_put;
dnet_log(n, DNET_LOG_DEBUG, "in dnet_merge_direct2 mc->size = %u\n", mc->size);
err = dnet_write_metadata(s, mc, 0, cflags);
if (err <= 0)
goto err_out_put;
err = 0;
err_out_put:
dnet_state_put(base);
err_out_exit:
return err;
}
static int dnet_merge_remove_local(struct dnet_node *n, struct dnet_id *id, int full_process)
{
char buf[sizeof(struct dnet_cmd)];
struct dnet_cmd *cmd;
struct dnet_net_state *base;
int err = -ENOENT;
memset(buf, 0, sizeof(buf));
cmd = (struct dnet_cmd *)buf;
memcpy(&cmd->id, id, sizeof(struct dnet_id));
cmd->size = 0;
cmd->cmd = DNET_CMD_DEL;
if (!full_process)
cmd->flags = DNET_ATTR_DELETE_HISTORY;
base = dnet_node_state(n);
if (base) {
err = dnet_process_meta(base, cmd, NULL);
dnet_state_put(base);
}
return err;
}
struct dnet_node *dnet_server_node_create(struct dnet_config *cfg)
{
struct dnet_node *n;
struct dnet_raw_id *ids = NULL;
int id_num = 0;
int err = -ENOMEM;
n = dnet_node_create(cfg);
if (!n) {
goto err_out_exit;
}
if (!n->notify_hash_size) {
n->notify_hash_size = DNET_DEFAULT_NOTIFY_HASH_SIZE;
err = dnet_notify_init(n);
if (err)
goto err_out_node_destroy;
dnet_log(n, DNET_LOG_NOTICE, "No notify hash size provided, using default %d.\n",
n->notify_hash_size);
}
err = dnet_cache_init(n);
if (err)
goto err_out_notify_exit;
if (cfg->flags & DNET_CFG_JOIN_NETWORK) {
int s;
err = dnet_locks_init(n, cfg->oplock_num);
if (err)
goto err_out_cache_cleanup;
ids = dnet_ids_init(n, cfg->history_env, &id_num, cfg->storage_free);
if (!ids)
goto err_out_locks_destroy;
n->addr.addr_len = sizeof(n->addr.addr);
err = dnet_socket_create(n, cfg, &n->addr, 1);
if (err < 0)
goto err_out_ids_cleanup;
s = err;
dnet_setup_id(&n->id, cfg->group_id, ids[0].id);
n->st = dnet_state_create(n, cfg->group_id, ids, id_num, &n->addr, s, &err, DNET_JOIN, dnet_state_accept_process);
if (!n->st) {
close(s);
goto err_out_state_destroy;
}
free(ids);
ids = NULL;
err = dnet_srw_init(n, cfg);
if (err) {
dnet_log(n, DNET_LOG_ERROR, "srw: initialization failure: %s %d\n", strerror(-err), err);
}
}
dnet_log(n, DNET_LOG_DSA, "New server node has been created at %s, ids: %d.\n",
dnet_dump_node(n), id_num);
return n;
err_out_state_destroy:
dnet_srw_cleanup(n);
dnet_state_put(n->st);
err_out_ids_cleanup:
free(ids);
err_out_locks_destroy:
dnet_locks_destroy(n);
err_out_cache_cleanup:
dnet_cache_cleanup(n);
err_out_notify_exit:
dnet_notify_exit(n);
err_out_node_destroy:
dnet_node_destroy(n);
err_out_exit:
return NULL;
}
struct dnet_node *dnet_server_node_create(struct dnet_config_data *cfg_data)
{
struct dnet_node *n;
struct dnet_config *cfg = &cfg_data->cfg_state;
struct dnet_addr *addrs = cfg_data->cfg_addrs;
int addr_num = cfg_data->cfg_addr_num;
int err = -ENOMEM;
sigset_t previous_sigset;
sigset_t sigset;
sigemptyset(&sigset);
sigaddset(&sigset, SIGINT);
sigaddset(&sigset, SIGTERM);
sigaddset(&sigset, SIGALRM);
sigaddset(&sigset, SIGQUIT);
pthread_sigmask(SIG_BLOCK, &sigset, &previous_sigset);
n = dnet_node_create(cfg);
if (!n)
goto err_out_exit;
n->config_data = cfg_data;
err = dnet_server_io_init(n);
if (err)
goto err_out_node_destroy;
err = dnet_monitor_init(n, cfg);
if (err)
goto err_out_node_destroy;
err = dnet_node_check_stack(n);
if (err)
goto err_out_monitor_destroy;
if (!n->notify_hash_size) {
n->notify_hash_size = DNET_DEFAULT_NOTIFY_HASH_SIZE;
err = dnet_notify_init(n);
if (err)
goto err_out_monitor_destroy;
dnet_log(n, DNET_LOG_NOTICE, "No notify hash size provided, using default %d.",
n->notify_hash_size);
}
err = dnet_local_addr_add(n, addrs, addr_num);
if (err)
goto err_out_notify_exit;
if (cfg->flags & DNET_CFG_JOIN_NETWORK) {
int s;
struct dnet_addr la;
err = dnet_locks_init(n, 1024);
if (err) {
dnet_log(n, DNET_LOG_ERROR, "failed to init locks: %s %d", strerror(-err), err);
goto err_out_addr_cleanup;
}
n->route = dnet_route_list_create(n);
if (!n->route) {
dnet_log(n, DNET_LOG_ERROR, "failed to create route list: %s %d", strerror(-err), err);
goto err_out_locks_destroy;
}
err = dnet_create_addr(&la, NULL, cfg->port, cfg->family);
if (err < 0) {
dnet_log(n, DNET_LOG_ERROR, "Failed to get address info for 0.0.0.0:%d, family: %d, err: %d: %s.",
cfg->port, cfg->family, err, strerror(-err));
goto err_out_route_list_destroy;
}
err = dnet_socket_create_listening(n, &la);
if (err < 0)
goto err_out_route_list_destroy;
s = err;
if (s < 0) {
err = s;
dnet_log(n, DNET_LOG_ERROR, "failed to create socket: %s %d", strerror(-err), err);
goto err_out_route_list_destroy;
}
n->st = dnet_state_create(n, NULL, 0, n->addrs, s, &err, DNET_JOIN, 1, 0, 1, n->addrs, n->addr_num);
if (!n->st) {
dnet_log(n, DNET_LOG_ERROR, "failed to create state: %s %d", strerror(-err), err);
goto err_out_state_destroy;
}
// @dnet_state_create() returns state pointer which holds 2 references - one for originally created state
// and another for caller in case he wants to do something with the state. The first reference is owned
// by network thread given state was attached to, and it can already release it.
dnet_state_put(n->st);
err = dnet_backend_init_all(n);
if (err) {
dnet_log(n, DNET_LOG_ERROR, "failed to init backends: %s %d", strerror(-err), err);
goto err_out_state_destroy;
}
if (!cfg->srw.config) {
dnet_log(n, DNET_LOG_INFO, "srw: no config");
n->srw = NULL;
} else {
err = dnet_srw_init(n, cfg);
if (err) {
dnet_log(n, DNET_LOG_ERROR, "srw: initialization failure: %s %d", strerror(-err), err);
goto err_out_backends_cleanup;
}
}
}
dnet_log(n, DNET_LOG_DEBUG, "New server node has been created at port %d.", cfg->port);
pthread_sigmask(SIG_SETMASK, &previous_sigset, NULL);
return n;
dnet_srw_cleanup(n);
err_out_backends_cleanup:
dnet_set_need_exit(n);
dnet_backend_cleanup_all(n);
err_out_state_destroy:
dnet_state_put(n->st);
err_out_route_list_destroy:
dnet_route_list_destroy(n->route);
err_out_locks_destroy:
dnet_locks_destroy(n);
err_out_addr_cleanup:
dnet_local_addr_cleanup(n);
err_out_notify_exit:
n->need_exit = err;
dnet_notify_exit(n);
err_out_monitor_destroy:
dnet_monitor_exit(n);
err_out_node_destroy:
dnet_node_destroy(n);
err_out_exit:
pthread_sigmask(SIG_SETMASK, &previous_sigset, NULL);
return NULL;
}
struct dnet_node *dnet_server_node_create(struct dnet_config_data *cfg_data, struct dnet_config *cfg, struct dnet_addr *addrs, int addr_num)
{
struct dnet_node *n;
struct dnet_raw_id *ids = NULL;
int id_num = 0;
int err = -ENOMEM;
sigset_t previous_sigset;
sigset_t sigset;
sigemptyset(&sigset);
sigaddset(&sigset, SIGINT);
sigaddset(&sigset, SIGTERM);
sigaddset(&sigset, SIGALRM);
sigaddset(&sigset, SIGQUIT);
pthread_sigmask(SIG_BLOCK, &sigset, &previous_sigset);
n = dnet_node_create(cfg);
if (!n)
goto err_out_exit;
n->config_data = cfg_data;
err = dnet_node_check_stack(n);
if (err)
goto err_out_node_destroy;
if (!n->notify_hash_size) {
n->notify_hash_size = DNET_DEFAULT_NOTIFY_HASH_SIZE;
err = dnet_notify_init(n);
if (err)
goto err_out_node_destroy;
dnet_log(n, DNET_LOG_NOTICE, "No notify hash size provided, using default %d.\n",
n->notify_hash_size);
}
err = dnet_cache_init(n);
if (err)
goto err_out_notify_exit;
err = dnet_local_addr_add(n, addrs, addr_num);
if (err)
goto err_out_cache_cleanup;
if (cfg->flags & DNET_CFG_JOIN_NETWORK) {
struct dnet_addr la;
int s;
err = dnet_locks_init(n, 1024);
if (err)
goto err_out_addr_cleanup;
ids = dnet_ids_init(n, cfg->history_env, &id_num, cfg->storage_free, cfg_data->cfg_addrs, cfg_data->cfg_remotes);
if (!ids)
goto err_out_locks_destroy;
memset(&la, 0, sizeof(struct dnet_addr));
la.addr_len = sizeof(la.addr);
la.family = cfg->family;
err = dnet_socket_create(n, NULL, cfg->port, &la, 1);
if (err < 0)
goto err_out_ids_cleanup;
s = err;
dnet_setup_id(&n->id, cfg->group_id, ids[0].id);
n->st = dnet_state_create(n, cfg->group_id, ids, id_num, &la, s, &err, DNET_JOIN, -1, dnet_state_accept_process);
if (!n->st) {
close(s);
goto err_out_state_destroy;
}
free(ids);
ids = NULL;
if (!cfg->srw.config) {
dnet_log(n, DNET_LOG_INFO, "srw: no config\n");
n->srw = NULL;
} else {
err = dnet_srw_init(n, cfg);
if (err) {
dnet_log(n, DNET_LOG_ERROR, "srw: initialization failure: %s %d\n", strerror(-err), err);
goto err_out_state_destroy;
}
}
}
dnet_log(n, DNET_LOG_DEBUG, "New server node has been created at port %d, ids: %d.\n", cfg->port, id_num);
pthread_sigmask(SIG_SETMASK, &previous_sigset, NULL);
return n;
dnet_srw_cleanup(n);
err_out_state_destroy:
dnet_state_put(n->st);
err_out_ids_cleanup:
free(ids);
err_out_locks_destroy:
dnet_locks_destroy(n);
err_out_addr_cleanup:
dnet_local_addr_cleanup(n);
err_out_cache_cleanup:
dnet_cache_cleanup(n);
err_out_notify_exit:
dnet_notify_exit(n);
err_out_node_destroy:
dnet_node_destroy(n);
err_out_exit:
pthread_sigmask(SIG_SETMASK, &previous_sigset, NULL);
return NULL;
}
void dnet_trans_destroy(struct dnet_trans *t)
{
struct dnet_net_state *st = NULL;
struct timeval tv;
long diff;
if (!t)
return;
dnet_node_set_trace_id(t->cmd.trace_id, t->cmd.flags & DNET_FLAGS_TRACE_BIT);
gettimeofday(&tv, NULL);
diff = 1000000 * (tv.tv_sec - t->start.tv_sec) + (tv.tv_usec - t->start.tv_usec);
if (t->st && t->st->n) {
st = t->st;
pthread_mutex_lock(&st->trans_lock);
list_del_init(&t->trans_list_entry);
if (t->trans_entry.rb_parent_color) {
dnet_trans_remove_nolock(st, t);
}
pthread_mutex_unlock(&st->trans_lock);
} else if (!list_empty(&t->trans_list_entry)) {
assert(0);
}
if (t->complete) {
t->cmd.flags |= DNET_FLAGS_DESTROY;
t->complete(t->st ? dnet_state_addr(t->st) : NULL, &t->cmd, t->priv);
}
if (st && st->n && t->command != 0) {
char str[64];
char io_buf[1024] = "";
struct tm tm;
if (t->cmd.status != -ETIMEDOUT) {
if (st->stall) {
dnet_log(st->n, DNET_LOG_INFO, "%s/%d: reseting state stall counter",
dnet_state_dump_addr(st), t->cmd.backend_id);
}
st->stall = 0;
}
localtime_r((time_t *)&t->start.tv_sec, &tm);
strftime(str, sizeof(str), "%F %R:%S", &tm);
if ((t->command == DNET_CMD_READ || t->command == DNET_CMD_WRITE) &&
(t->alloc_size >= sizeof(struct dnet_cmd) + sizeof(struct dnet_io_attr))) {
struct dnet_cmd *local_cmd = (struct dnet_cmd *)(t + 1);
struct dnet_io_attr *local_io = (struct dnet_io_attr *)(local_cmd + 1);
double backend_weight = 0.;
dnet_get_backend_weight(st, t->cmd.backend_id, local_io->flags, &backend_weight);
snprintf(io_buf, sizeof(io_buf), ", weight: %f, %s",
backend_weight, dnet_print_io(local_io));
}
dnet_log(st->n, DNET_LOG_INFO, "%s: %s: destruction %s, stall: %d, "
"time: %ld, started: %s.%06lu, cached status: %d%s",
dnet_dump_id(&t->cmd.id),
dnet_cmd_string(t->cmd.cmd),
dnet_print_trans(t),
t->st->stall,
diff,
str, t->start.tv_usec,
t->cmd.status, io_buf);
}
dnet_state_put(t->st);
dnet_state_put(t->orig);
dnet_node_unset_trace_id();
free(t);
}
static int dnet_bulk_add_id(struct dnet_node *n, struct dnet_bulk_array *bulk_array, struct dnet_id *id,
struct dnet_meta_container *mc, struct dnet_check_params *params)
{
int err = 0;
struct dnet_bulk_state tmp;
struct dnet_bulk_state *state = NULL;
struct dnet_net_state *st = dnet_state_get_first(n, id);
struct dnet_bulk_id *bulk_id;
struct dnet_meta_update mu;
dnet_log(n, DNET_LOG_DEBUG, "BULK: adding ID %s to array\n", dnet_dump_id(id));
if (!st)
return -1;
memcpy(&tmp.addr, &st->addr, sizeof(struct dnet_addr));
dnet_state_put(st);
dnet_log(n, DNET_LOG_DEBUG, "BULK: Searching state in states array\n");
state = bsearch(&tmp, bulk_array->states, bulk_array->num, sizeof(struct dnet_bulk_state), dnet_compare_bulk_state);
if (!state)
return -1;
if (!dnet_get_meta_update(n, mc, &mu))
return -ENOENT;
dnet_log(n, DNET_LOG_DEBUG, "BULK: addr = %s state->num = %d\n", dnet_server_convert_dnet_addr(&state->addr), state->num);
//pthread_mutex_lock(&state->state_lock);
if (state->num >= DNET_BULK_IDS_SIZE || state->num < 0)
goto err_out_unlock;
bulk_id = &state->ids[state->num];
memset(bulk_id, 0, sizeof(struct dnet_bulk_id));
memcpy(&bulk_id->id, &id->id, DNET_ID_SIZE);
dnet_log(n, DNET_LOG_DEBUG, "BULK: ID: %s, last_update->tsec=%llu, last_update->tnsec=%llu, flags=%02llx\n",
dnet_dump_id_str(bulk_id->id.id), (unsigned long long)mu.tm.tsec, (unsigned long long)mu.tm.tnsec,
(unsigned long long)mu.flags);
dnet_convert_meta_update(&mu);
memcpy(&bulk_id->last_update, &mu, sizeof(struct dnet_meta_update));
state->num++;
dnet_log(n, DNET_LOG_DEBUG, "BULK: addr = %s state->num = %d\n", dnet_server_convert_dnet_addr(&state->addr), state->num);
if (state->num == DNET_BULK_IDS_SIZE) {
err = dnet_request_bulk_check(n, state, params);
state->num = 0;
if (err)
goto err_out_unlock;
}
//pthread_mutex_unlock(&state->state_lock);
return 0;
err_out_unlock:
//pthread_mutex_unlock(&state->state_lock);
return -2;
}
int dnet_request_bulk_check(struct dnet_node *n, struct dnet_bulk_state *state, struct dnet_check_params *params)
{
struct dnet_trans_control ctl;
struct dnet_net_state *st;
struct dnet_bulk_check_priv *p;
struct timespec wait_ts;
int err;
p = (struct dnet_bulk_check_priv *)malloc(sizeof(struct dnet_bulk_check_priv));
if (!p) {
err = -ENOMEM;
goto err_out_exit;
}
atomic_init(&p->refcnt, 2);
p->db = params->db;
dnet_check_temp_db_get(p->db);
p->w = dnet_wait_alloc(0);
if (!p->w) {
err = -ENOMEM;
goto err_out_free;
}
p->group_num = params->group_num;
p->groups = (int *)malloc(sizeof(int) * params->group_num);
if (!p->groups) {
err = -ENOMEM;
goto err_out_put;
}
memcpy(p->groups, params->groups, sizeof(int) * params->group_num);
memset(&ctl, 0, sizeof(struct dnet_trans_control));
ctl.cmd = DNET_CMD_LIST;
ctl.complete = dnet_bulk_check_complete;
ctl.priv = p;
ctl.cflags = DNET_FLAGS_NEED_ACK | DNET_FLAGS_NOLOCK | DNET_ATTR_BULK_CHECK;
ctl.data = state->ids;
ctl.size = sizeof(struct dnet_bulk_id) * state->num;
st = dnet_state_search_by_addr(n, &state->addr);
if (!st) {
err = -ENOENT;
goto err_out_put;
}
dnet_setup_id(&ctl.id, st->idc->group->group_id, st->idc->ids[0].raw.id);
dnet_wait_get(p->w);
dnet_log(n, DNET_LOG_DEBUG, "BULK: sending %u bytes of data to %s (%s)\n",
ctl.size, dnet_dump_id(&ctl.id), dnet_server_convert_dnet_addr(&state->addr));
err = dnet_trans_alloc_send_state(st, &ctl);
dnet_state_put(st);
wait_ts = n->wait_ts;
wait_ts.tv_sec *= DNET_BULK_IDS_SIZE;
err = dnet_wait_event(p->w, p->w->cond != 0, &wait_ts);
if (err)
goto err_out_put;
if (p->w->status) {
err = p->w->status;
goto err_out_put;
}
dnet_wait_put(p->w);
if (atomic_dec_and_test(&p->refcnt)) {
free(p->groups);
free(p);
}
return 0;
err_out_put:
dnet_wait_put(p->w);
err_out_free:
if (atomic_dec_and_test(&p->refcnt)) {
free(p->groups);
free(p);
}
err_out_exit:
dnet_log(n, DNET_LOG_ERROR, "Bulk check exited with status %d\n", err);
return err;
}
