static as_status
as_admin_execute(aerospike* as, as_error* err, const as_policy_admin* policy, uint8_t* buffer, uint8_t* end)
{
uint32_t timeout_ms = (policy)? policy->timeout : as->config.policies.admin.timeout;
if (timeout_ms <= 0) {
timeout_ms = DEFAULT_TIMEOUT;
}
uint64_t deadline_ms = as_socket_deadline(timeout_ms);
as_node* node = as_node_get_random(as->cluster);
if (! node) {
return as_error_set_message(err, AEROSPIKE_ERR_CLIENT, "Failed to find server node.");
}
int fd;
as_status status = as_node_get_connection(err, node, &fd);
if (status) {
as_node_release(node);
return status;
}
status = as_admin_send(err, fd, buffer, end, deadline_ms);
if (status) {
as_close(fd);
as_node_release(node);
return status;
}
status = as_socket_read_deadline(err, fd, buffer, HEADER_SIZE, deadline_ms);
if (status) {
as_close(fd);
as_node_release(node);
return status;
}
as_node_put_connection(node, fd);
as_node_release(node);
status = buffer[RESULT_CODE];
if (status) {
return as_error_set_message(err, status, as_error_string(status));
}
return status;
}
static as_status
as_admin_read_list(aerospike* as, as_error* err, const as_policy_admin* policy, uint8_t* command, uint8_t* end, as_admin_parse_fn parse_fn, as_vector* list)
{
int timeout_ms = (policy)? policy->timeout : as->config.policies.admin.timeout;
if (timeout_ms <= 0) {
timeout_ms = DEFAULT_TIMEOUT;
}
uint64_t deadline_ms = cf_getms() + timeout_ms;
as_node* node = as_node_get_random(as->cluster);
if (! node) {
return as_error_set_message(err, AEROSPIKE_ERR_CLIENT, "Failed to find server node.");
}
int fd;
as_status status = as_node_get_connection(err, node, &fd);
if (status) {
as_node_release(node);
return status;
}
status = as_admin_send(err, fd, command, end, deadline_ms);
if (status) {
as_close(fd);
as_node_release(node);
return status;
}
status = as_admin_read_blocks(err, fd, deadline_ms, parse_fn, list);
if (status) {
as_close(fd);
as_node_release(node);
return status;
}
as_node_put_connection(node, fd);
as_node_release(node);
return status;
}
static void
as_cluster_find_nodes_to_add(as_cluster* cluster, as_vector* /* <as_host> */ friends, as_vector* /* <as_node*> */ nodes_to_add)
{
as_error err;
as_error_init(&err);
as_vector addresses;
as_vector_inita(&addresses, sizeof(struct sockaddr_in), 5);
as_node_info node_info;
for (uint32_t i = 0; i < friends->size; i++) {
as_host* friend = as_vector_get(friends, i);
as_vector_clear(&addresses);
as_status status = as_lookup(&err, friend->name, friend->port, &addresses);
if (status != AEROSPIKE_OK) {
as_log_warn("%s %s", as_error_string(status), err.message);
continue;
}
for (uint32_t i = 0; i < addresses.size; i++) {
struct sockaddr_in* addr = as_vector_get(&addresses, i);
status = as_lookup_node(cluster, &err, addr, &node_info);
if (status == AEROSPIKE_OK) {
as_node* node = as_cluster_find_node(cluster->nodes, nodes_to_add, node_info.name);
if (node) {
// Duplicate node name found. This usually occurs when the server
// services list contains both internal and external IP addresses
// for the same node. Add new host to list of alias filters
// and do not add new node.
as_close(node_info.fd);
as_address* a = as_node_get_address_full(node);
as_log_info("Node %s:%d already exists with nodeid %s and address %s:%d",
friend->name, friend->port, node->name, a->name,
(int)cf_swap_from_be16(a->addr.sin_port));
node->friends++;
as_node_add_address(node, friend, addr);
continue;
}
node = as_node_create(cluster, friend, addr, &node_info);
as_address* a = as_node_get_address_full(node);
as_log_info("Add node %s %s:%d", node_info.name, a->name, (int)cf_swap_from_be16(a->addr.sin_port));
as_vector_append(nodes_to_add, &node);
}
else {
as_log_warn("Failed to connect to friend %s:%d. %s %s", friend->name, friend->port, as_error_string(status), err.message);
}
}
static as_status
as_cluster_seed_nodes(as_cluster* cluster, as_error* err, bool enable_warnings)
{
// Add all nodes at once to avoid copying entire array multiple times.
as_vector nodes_to_add;
as_vector_inita(&nodes_to_add, sizeof(as_node*), 64);
as_vector addresses;
as_vector_inita(&addresses, sizeof(struct sockaddr_in), 5);
as_node_info node_info;
as_error error_local;
as_error_init(&error_local); // AEROSPIKE_ERR_TIMEOUT doesn't come with a message; make sure it's initialized.
as_status status = AEROSPIKE_OK;
as_seeds* seeds = as_seeds_reserve(cluster);
for (uint32_t i = 0; i < seeds->size; i++) {
as_seed* seed = &seeds->array[i];
as_vector_clear(&addresses);
status = as_lookup(&error_local, seed->name, seed->port, &addresses);
if (status != AEROSPIKE_OK) {
if (enable_warnings) {
as_log_warn("Failed to lookup %s:%d. %s %s", seed->name, seed->port, as_error_string(status), error_local.message);
}
continue;
}
for (uint32_t i = 0; i < addresses.size; i++) {
struct sockaddr_in* addr = as_vector_get(&addresses, i);
status = as_lookup_node(cluster, &error_local, addr, &node_info);
if (status == AEROSPIKE_OK) {
as_host host;
if (as_strncpy(host.name, seed->name, sizeof(host.name))) {
as_log_warn("Hostname has been truncated: %s", host.name);
}
host.port = seed->port;
as_node* node = as_cluster_find_node_in_vector(&nodes_to_add, node_info.name);
if (node) {
as_close(node_info.fd);
as_node_add_address(node, &host, addr);
}
else {
node = as_node_create(cluster, &host, addr, &node_info);
as_address* a = as_node_get_address_full(node);
as_log_info("Add node %s %s:%d", node->name, a->name, (int)cf_swap_from_be16(a->addr.sin_port));
as_vector_append(&nodes_to_add, &node);
}
}
else {
if (enable_warnings) {
as_log_warn("Failed to connect to seed %s:%d. %s %s", seed->name, seed->port, as_error_string(status), error_local.message);
}
}
}
}
as_seeds_release(seeds);
if (nodes_to_add.size > 0) {
as_cluster_add_nodes(cluster, &nodes_to_add);
status = AEROSPIKE_OK;
}
else {
status = as_error_set_message(err, AEROSPIKE_ERR_CLIENT, "Failed to seed cluster");
}
as_vector_destroy(&nodes_to_add);
as_vector_destroy(&addresses);
return status;
}
static as_status
as_lookup_node(as_cluster* cluster, as_error* err, struct sockaddr_in* addr, as_node_info* node_info)
{
uint64_t deadline = as_socket_deadline(cluster->conn_timeout_ms);
int fd;
as_status status = as_info_create_socket(cluster, err, addr, deadline, &fd);
if (status) {
return status;
}
char* response = 0;
status = as_info_command(err, fd, "node\nfeatures\n", true, deadline, 0, &response);
if (status) {
as_close(fd);
return status;
}
as_vector values;
as_vector_inita(&values, sizeof(as_name_value), 2);
as_info_parse_multi_response(response, &values);
if (values.size != 2) {
goto Error;
}
as_name_value* nv = as_vector_get(&values, 0);
char* node_name = nv->value;
if (node_name == 0 || *node_name == 0) {
goto Error;
}
as_strncpy(node_info->name, node_name, AS_NODE_NAME_SIZE);
node_info->fd = fd;
nv = as_vector_get(&values, 1);
char* features = nv->value;
if (features == 0) {
goto Error;
}
char* begin = features;
char* end = begin;
uint8_t has_batch_index = 0;
uint8_t has_replicas_all = 0;
uint8_t has_double = 0;
uint8_t has_geo = 0;
while (*begin && ! (has_batch_index &&
has_replicas_all &&
has_double &&
has_geo)) {
while (*end) {
if (*end == ';') {
*end++ = 0;
break;
}
end++;
}
if (strcmp(begin, "batch-index") == 0) {
has_batch_index = 1;
}
if (strcmp(begin, "replicas-all") == 0) {
has_replicas_all = 1;
}
if (strcmp(begin, "float") == 0) {
has_double = 1;
}
if (strcmp(begin, "geo") == 0) {
has_geo = 1;
}
begin = end;
}
node_info->has_batch_index = has_batch_index;
node_info->has_replicas_all = has_replicas_all;
node_info->has_double = has_double;
node_info->has_geo = has_geo;
cf_free(response);
return AEROSPIKE_OK;
Error: {
char addr_name[INET_ADDRSTRLEN];
as_socket_address_name(addr, addr_name);
as_error_update(err, status, "Invalid node info response from %s: %s", addr_name, response);
cf_free(response);
as_close(fd);
return AEROSPIKE_ERR_CLIENT;
}
}
as_status
as_command_execute(as_cluster* cluster, as_error * err, as_command_node* cn, uint8_t* command, size_t command_len,
uint32_t timeout_ms, uint32_t retry,
as_parse_results_fn parse_results_fn, void* parse_results_data
)
{
uint64_t deadline_ms = as_socket_deadline(timeout_ms);
uint32_t sleep_between_retries_ms = 0;
uint32_t failed_nodes = 0;
uint32_t failed_conns = 0;
uint32_t iterations = 0;
bool release_node;
// Execute command until successful, timed out or maximum iterations have been reached.
while (true) {
as_node* node;
if (cn->node) {
node = cn->node;
release_node = false;
}
else {
node = as_node_get(cluster, cn->ns, cn->digest, cn->write, cn->replica);
release_node = true;
}
if (!node) {
failed_nodes++;
sleep_between_retries_ms = 10;
goto Retry;
}
int fd;
as_status status = as_node_get_connection(err, node, deadline_ms, &fd);
if (status) {
if (release_node) {
as_node_release(node);
}
failed_conns++;
sleep_between_retries_ms = 1;
goto Retry;
}
// Send command.
status = as_socket_write_deadline(err, fd, command, command_len, deadline_ms);
if (status) {
// Socket errors are considered temporary anomalies. Retry.
// Close socket to flush out possible garbage. Do not put back in pool.
as_close(fd);
if (release_node) {
as_node_release(node);
}
sleep_between_retries_ms = 0;
goto Retry;
}
// Parse results returned by server.
status = parse_results_fn(err, fd, deadline_ms, parse_results_data);
if (status == AEROSPIKE_OK) {
// Reset error code if retry had occurred.
if (iterations > 0) {
as_error_reset(err);
}
}
else {
switch (status) {
// Retry on timeout.
case AEROSPIKE_ERR_TIMEOUT:
as_close(fd);
if (release_node) {
as_node_release(node);
}
sleep_between_retries_ms = 0;
goto Retry;
// Close socket on errors that can leave unread data in socket.
case AEROSPIKE_ERR_QUERY_ABORTED:
case AEROSPIKE_ERR_SCAN_ABORTED:
case AEROSPIKE_ERR_CLIENT_ABORT:
case AEROSPIKE_ERR_CLIENT:
as_close(fd);
if (release_node) {
as_node_release(node);
}
err->code = status;
return status;
default:
err->code = status;
break;
}
}
// Put connection back in pool.
as_node_put_connection(node, fd, cluster->conn_queue_size);
// Release resources.
if (release_node) {
as_node_release(node);
}
return status;
Retry:
// Check if max retries reached.
if (++iterations > retry) {
break;
}
// Check for client timeout.
if (deadline_ms > 0) {
int remaining_ms = (int)(deadline_ms - cf_getms() - sleep_between_retries_ms);
if (remaining_ms <= 0) {
break;
}
// Reset timeout in send buffer (destined for server).
*(uint32_t*)(command + 22) = cf_swap_to_be32(remaining_ms);
}
if (sleep_between_retries_ms > 0) {
// Sleep before trying again.
usleep(sleep_between_retries_ms * 1000);
}
}
return as_error_update(err, AEROSPIKE_ERR_TIMEOUT,
"Client timeout: timeout=%d iterations=%u failedNodes=%u failedConns=%u",
timeout_ms, iterations, failed_nodes, failed_conns);
}