static void tcpclient_write_event(struct ev_loop *loop, struct ev_io *watcher, int events) {
tcpclient_t *client = (tcpclient_t *)watcher->data;
buffer_t *sendq;
if (!(events & EV_WRITE)) {
return;
}
sendq = &client->send_queue;
ssize_t buf_len = buffer_datacount(sendq);
if (buf_len > 0) {
ssize_t send_len = send(client->sd, sendq->head, buf_len, 0);
stats_debug_log("tcpclient: sent %zd of %zd bytes to backend client %s via fd %d",
send_len, buf_len, client->name, client->sd);
if (send_len < 0) {
stats_error_log("tcpclient[%s]: Error from send: %s", client->name, strerror(errno));
ev_io_stop(client->loop, &client->write_watcher.watcher);
ev_io_stop(client->loop, &client->read_watcher.watcher);
client->last_error = time(NULL);
tcpclient_set_state(client, STATE_BACKOFF);
close(client->sd);
client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0);
return;
} else {
client->callback_sent(client, EVENT_SENT, client->callback_context, sendq->head, (size_t) send_len);
if (buffer_consume(sendq, send_len) != 0) {
stats_error_log("tcpclient[%s]: Unable to consume send queue", client->name);
return;
}
size_t qsize = buffer_datacount(&client->send_queue);
if (client->failing && qsize < client->config->max_send_queue) {
stats_log("tcpclient[%s]: client recovered from full queue, send queue is now %zd bytes",
client->name,
qsize);
client->failing = 0;
}
if (qsize == 0) {
ev_io_stop(client->loop, &client->write_watcher.watcher);
client->write_watcher.started = false;
}
}
} else {
// No data left in the client's buffer, stop waiting
// for write events.
ev_io_stop(client->loop, &client->write_watcher.watcher);
client->write_watcher.started = false;
}
}
static void tcpclient_read_event(struct ev_loop *loop, struct ev_io *watcher, int events) {
tcpclient_t *client = (tcpclient_t *)watcher->data;
ssize_t len;
char *buf;
if (!(events & EV_READ)) {
return;
}
buf = malloc(TCPCLIENT_RECV_BUFFER);
if (buf == NULL) {
stats_error_log("tcpclient[%s]: Unable to allocate memory for receive buffer", client->name);
return;
}
len = recv(client->sd, buf, TCPCLIENT_RECV_BUFFER, 0);
if (len < 0) {
stats_error_log("tcpclient[%s]: Error from recv: %s", client->name, strerror(errno));
if (client->read_watcher.started) {
ev_io_stop(client->loop, &client->read_watcher.watcher);
client->read_watcher.started = false;
}
if (client->write_watcher.started) {
ev_io_stop(client->loop, &client->write_watcher.watcher);
client->write_watcher.started = false;
}
close(client->sd);
free(buf);
tcpclient_set_state(client, STATE_BACKOFF);
client->last_error = time(NULL);
client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0);
return;
}
if (len == 0) {
stats_error_log("tcpclient[%s]: Server closed connection", client->name);
ev_io_stop(client->loop, &client->read_watcher.watcher);
ev_io_stop(client->loop, &client->write_watcher.watcher);
close(client->sd);
free(buf);
tcpclient_set_state(client, STATE_INIT);
client->last_error = time(NULL);
client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0);
return;
}
client->callback_recv(client, EVENT_RECV, client->callback_context, buf, len);
}
static int get_int_orelse(json_t* json, const char* key, int def) {
json_t* v = json_object_get(json, key);
if (v == NULL)
return def;
if (!json_is_number(v)) {
stats_error_log("Expected an integer value for '%s' - using default of '%d'", key, def);
}
return (int)json_integer_value(v);
}
int tcpclient_sendall(tcpclient_t *client, const char *buf, size_t len) {
buffer_t *sendq = &client->send_queue;
if (client->addr == NULL) {
stats_error_log("tcpclient[%s]: Cannot send before connect!", client->name);
return 1;
} else {
// Does nothing if we're already connected, triggers a
// reconnect if backoff has expired.
tcpclient_connect(client);
}
if (buffer_datacount(&client->send_queue) >= client->config->max_send_queue) {
if (client->failing == 0) {
stats_error_log("tcpclient[%s]: send queue for %s client is full (at %zd bytes, max is %" PRIu64 " bytes), dropping data",
client->name,
tcpclient_state_name[client->state],
buffer_datacount(&client->send_queue),
client->config->max_send_queue);
client->failing = 1;
}
return 2;
}
if (buffer_spacecount(sendq) < len) {
if (buffer_realign(sendq) != 0) {
stats_error_log("tcpclient[%s]: Unable to realign send queue", client->name);
return 3;
}
}
while (buffer_spacecount(sendq) < len) {
if (buffer_expand(sendq) != 0) {
stats_error_log("tcpclient[%s]: Unable to allocate additional memory for send queue, dropping data", client->name);
return 4;
}
}
memcpy(buffer_tail(sendq), buf, len);
buffer_produced(sendq, len);
if (client->state == STATE_CONNECTED) {
client->write_watcher.started = true;
ev_io_start(client->loop, &client->write_watcher.watcher);
}
return 0;
}
static bool set_boolean(const char *strval, bool *bool_val) {
if (strcmp(strval, "true") == 0) {
*bool_val = true;
} else if (strcmp(strval, "false") == 0) {
*bool_val = false;
} else {
stats_error_log("unexpected value \"%s\" for boolean field, "
"must be true/false", strval);
return false;
}
return true;
}
static void tcpclient_connect_timeout(struct ev_loop *loop, struct ev_timer *watcher, int events) {
tcpclient_t *client = (tcpclient_t *)watcher->data;
if (client->connect_watcher.started) {
ev_io_stop(loop, &client->connect_watcher.watcher);
client->connect_watcher.started = false;
}
close(client->sd);
stats_error_log("tcpclient[%s]: Connection timeout", client->name);
client->last_error = time(NULL);
tcpclient_set_state(client, STATE_BACKOFF);
client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0);
}
static void parse_server_list(const json_t* jshards, list_t ring) {
if (jshards == NULL) {
stats_error_log("no servers specified for routing");
return;
}
const json_t* jserver = NULL;
size_t index;
json_array_foreach(jshards, index, jserver) {
statsrelay_list_expand(ring);
char* serverline = strdup(json_string_value(jserver));
stats_log("adding server %s", serverline);
ring->data[ring->size - 1] = serverline;
}
static char* get_string(const json_t* json, const char* key) {
json_t* j = json_object_get(json, key);
if (key == NULL)
return NULL;
if (!json_is_string(j)) {
stats_error_log("Expected a string value for '%s' - ignoring config value", key);
return NULL;
}
const char* str = json_string_value(j);
if (str == NULL)
return NULL;
else
return strdup(str);
}
static void tcpclient_connected(struct ev_loop *loop, struct ev_io *watcher, int events) {
tcpclient_t *client = (tcpclient_t *)watcher->data;
int err;
socklen_t len = sizeof(err);
// Cancel timeout timer
ev_timer_stop(loop, &client->timeout_watcher);
ev_io_stop(loop, &client->connect_watcher.watcher);
if (getsockopt(client->sd, SOL_SOCKET, SO_ERROR, &err, &len) != 0) {
stats_error_log("tcpclient[%s]: Unable to get socket error state: %s", client->name, strerror(errno));
return;
}
if ((events & EV_ERROR) || err) {
stats_error_log("tcpclient[%s]: Connect failed: %s", client->name, strerror(err));
close(client->sd);
client->last_error = time(NULL);
tcpclient_set_state(client, STATE_BACKOFF);
return;
}
tcpclient_set_state(client, STATE_CONNECTED);
// Setup events for recv
client->read_watcher.started = true;
client->read_watcher.watcher.data = client;
ev_io_init(&client->read_watcher.watcher, tcpclient_read_event, client->sd, EV_READ);
ev_io_start(client->loop, &client->read_watcher.watcher);
client->write_watcher.started = true;
client->write_watcher.watcher.data = client;
ev_io_init(&client->write_watcher.watcher, tcpclient_write_event, client->sd, EV_WRITE);
ev_io_start(client->loop, &client->write_watcher.watcher);
client->callback_connect(client, EVENT_CONNECTED, client->callback_context, NULL, 0);
}
static void json_error_log(const char* msg, json_error_t* error) {
stats_error_log("JSON error %s: %s (%s) at line %d", msg, error->text, error->source, error->line);
}
struct config* parse_config(FILE *input) {
struct config *config = malloc(sizeof(struct config));
if (config == NULL) {
stats_error_log("malloc() error");
return NULL;
}
init_proto_config(&config->carbon_config);
if (config->carbon_config.ring == NULL) {
stats_error_log("failed to allocate ring");
free(config);
return NULL;
}
config->carbon_config.bind = strdup("127.0.0.1:2003");
init_proto_config(&config->statsd_config);
config->statsd_config.bind = strdup("127.0.0.1:8125");
yaml_parser_t parser;
yaml_event_t event;
if (!yaml_parser_initialize(&parser)) {
stats_log("failed to initialize yaml parser");
goto parse_err;
}
yaml_parser_set_input_file(&parser, input);
struct proto_config *protoc = NULL;
char *strval;
long numval;
int shard_count = -1;
int map_nesting = 0;
bool in_document = false;
bool keep_going = true;
bool is_key = false;
bool update_bind = false;
bool update_send_queue = false;
bool update_validate = false;
bool update_tcp_cork = false;
bool always_resolve_dns = false;
bool expect_shard_map = false;
while (keep_going) {
if (!yaml_parser_parse(&parser, &event)) {
goto parse_err;
}
switch(event.type) {
case YAML_NO_EVENT:
case YAML_STREAM_START_EVENT:
break; // nothing to do
case YAML_STREAM_END_EVENT:
keep_going = false;
break;
case YAML_DOCUMENT_START_EVENT:
if (in_document) {
stats_error_log("config should not have nested documents");
goto parse_err;
}
in_document = true;
break;
case YAML_DOCUMENT_END_EVENT:
in_document = false;
break;
case YAML_SEQUENCE_START_EVENT:
case YAML_SEQUENCE_END_EVENT:
stats_error_log("unexpectedly got sequence");
goto parse_err;
break;
case YAML_MAPPING_START_EVENT:
is_key = true;
map_nesting++;
break;
case YAML_MAPPING_END_EVENT:
map_nesting--;
break;
case YAML_ALIAS_EVENT:
stats_error_log("don't know how to handle yaml aliases");
goto parse_err;
break;
case YAML_SCALAR_EVENT:
strval = (char *) event.data.scalar.value;
switch (map_nesting) {
case 0:
stats_error_log("unexpectedly got scalar outside of a map");
goto parse_err;
break;
case 1:
if (strcmp(strval, "carbon") == 0) {
protoc = &config->carbon_config;
config->carbon_config.initialized = true;
} else if (strcmp(strval, "statsd") == 0) {
protoc = &config->statsd_config;
config->statsd_config.initialized = true;
} else {
stats_error_log("unexpectedly got map value: \"%s\"", strval);
goto parse_err;
}
break;
case 2:
if (is_key) {
if (strcmp(strval, "bind") == 0) {
update_bind = true;
} else if (strcmp(strval, "max_send_queue") == 0) {
update_send_queue = true;
} else if (strcmp(strval, "shard_map") == 0) {
shard_count = -1;
expect_shard_map = true;
} else if (strcmp(strval, "validate") == 0) {
update_validate = true;
} else if (strcmp(strval, "tcp_cork") == 0) {
update_tcp_cork = true;
} else if (strcmp(strval, "always_resolve_dns") == 0) {
always_resolve_dns = true;
}
} else {
if (update_bind) {
free(protoc->bind);
protoc->bind = strdup(strval);
update_bind = false;
} else if (update_send_queue) {
if (!convert_number(strval, &numval)) {
stats_error_log("max_send_queue was not a number: %s", strval);
}
protoc->max_send_queue = numval;
update_send_queue = false;
} else if (update_validate) {
if (!set_boolean(strval, &protoc->enable_validation)) {
goto parse_err;
}
update_validate = false;
} else if (update_tcp_cork) {
if (!set_boolean(strval, &protoc->enable_tcp_cork)) {
goto parse_err;
}
update_tcp_cork = false;
} else if (always_resolve_dns) {
if (!set_boolean(strval, &protoc->always_resolve_dns)) {
goto parse_err;
}
}
}
break;
case 3:
if (!expect_shard_map) {
stats_error_log("was not expecting shard map");
goto parse_err;
} else if (is_key) {
if (!convert_number(strval, &numval)) {
stats_error_log("shard key was not a number: \"%s\"", strval);
goto parse_err;
}
shard_count++;
if (numval != shard_count) {
stats_error_log("expected to see shard key %d, instead saw %d",
shard_count, numval);
goto parse_err;
}
} else {
if (statsrelay_list_expand(protoc->ring) == NULL) {
stats_error_log("unable to expand list");
goto parse_err;
}
if ((protoc->ring->data[protoc->ring->size - 1] = strdup(strval)) == NULL) {
stats_error_log("failed to copy string");
goto parse_err;
}
}
}
is_key = !is_key;
break;
default:
stats_error_log("unhandled yaml event");
goto parse_err;
}
yaml_event_delete(&event);
}
yaml_parser_delete(&parser);
return config;
parse_err:
destroy_config(config);
yaml_event_delete(&event);
yaml_parser_delete(&parser);
return NULL;
}
int tcpclient_init(tcpclient_t *client,
struct ev_loop *loop,
void *callback_context,
struct proto_config *config,
char *host,
char *port,
char *protocol) {
size_t len;
client->state = STATE_INIT;
client->loop = loop;
client->sd = -1;
client->addr = NULL;
client->last_error = 0;
client->failing = 0;
client->config = config;
client->socktype = SOCK_DGRAM;
if(host == NULL) {
stats_error_log("tcpclient_init: host is NULL\n");
return 1;
}
len = strlen(host);
if((client->host = calloc(1, len + 1)) == NULL) {
stats_error_log("tcpclient[%s]: unable to allocate memory for host\n", client->host);
return 1;
}
strncpy(client->host, host, len);
if(port == NULL) {
stats_error_log("tcpclient_init: port is NULL\n");
free(client->host);
return 1;
}
len = strlen(port);
if((client->port = calloc(1, len + 1)) == NULL) {
stats_error_log("tcpclient[%s]: unable to allocate memory for port\n", client->host);
free(client->host);
return 1;
}
strncpy(client->port, port, len);
if(protocol == NULL) {
// Default to TCP if protocol is not set
if((client->protocol = calloc(1, 4)) == NULL) {
stats_error_log("tcpclient[%s]: unable to allocate memory for protocol\n", client->host);
free(client->host);
free(client->port);
return 1;
}
strncpy(client->protocol, "tcp", 3);
}else{
len = strlen(protocol);
if((client->protocol = calloc(1, len + 1)) == NULL) {
stats_error_log("tcpclient[%s]: unable to allocate memory for protocol\n", client->host);
free(client->host);
free(client->port);
return 1;
}
strncpy(client->protocol, protocol, len);
}
strncpy(client->name, "UNRESOLVED", TCPCLIENT_NAME_LEN);
client->callback_connect = &tcpclient_default_callback;
client->callback_sent = &tcpclient_default_callback;
client->callback_recv = &tcpclient_default_callback;
client->callback_error = &tcpclient_default_callback;
client->callback_context = callback_context;
buffer_init(&client->send_queue);
buffer_newsize(&client->send_queue, DEFAULT_BUFFER_SIZE);
ev_timer_init(&client->timeout_watcher,
tcpclient_connect_timeout,
TCPCLIENT_CONNECT_TIMEOUT,
0);
client->connect_watcher.started = false;
client->read_watcher.started = false;
client->write_watcher.started = false;
return 0;
}
int tcpclient_connect(tcpclient_t *client) {
struct addrinfo hints;
struct addrinfo *addr;
int sd;
if (client->state == STATE_CONNECTED || client->state == STATE_CONNECTING) {
// Already connected, do nothing
return 1;
}
if (client->state == STATE_BACKOFF) {
// If backoff timer has expired, change to STATE_INIT and call recursively
if ((time(NULL) - client->last_error) > TCPCLIENT_RETRY_TIMEOUT) {
tcpclient_set_state(client, STATE_INIT);
return tcpclient_connect(client);
} else {
return 2;
}
}
if (client->state == STATE_INIT) {
// Resolve address, create socket, set nonblocking, setup callbacks, fire connect
if (client->config->always_resolve_dns == true && client->addr != NULL) {
freeaddrinfo(client->addr);
client->addr = NULL;
}
if (client->addr == NULL) {
// We only know about tcp and udp, so if we get something unexpected just
// default to tcp
if (strncmp(client->protocol, "udp", 3) == 0) {
client->socktype = SOCK_DGRAM;
} else {
client->socktype = SOCK_STREAM;
}
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = client->socktype;
hints.ai_flags = AI_PASSIVE;
if (getaddrinfo(client->host, client->port, &hints, &addr) != 0) {
stats_error_log("tcpclient: Error resolving backend address %s: %s", client->host, gai_strerror(errno));
client->last_error = time(NULL);
tcpclient_set_state(client, STATE_BACKOFF);
client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0);
return 3;
}
client->addr = addr;
snprintf(client->name, TCPCLIENT_NAME_LEN, "%s/%s/%s", client->host, client->port, client->protocol);
} else {
addr = client->addr;
}
if ((sd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol)) < 0) {
stats_error_log("tcpclient[%s]: Unable to create socket: %s", client->name, strerror(errno));
client->last_error = time(NULL);
tcpclient_set_state(client, STATE_BACKOFF);
client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0);
return 4;
}
#ifdef TCP_CORK
if (client->config->enable_tcp_cork &&
addr->ai_family == AF_INET &&
addr->ai_socktype == SOCK_STREAM &&
addr->ai_protocol == IPPROTO_TCP) {
int state = 1;
if (setsockopt(sd, IPPROTO_TCP, TCP_CORK, &state, sizeof(state))) {
stats_error_log("failed to set TCP_CORK");
}
}
#endif
client->sd = sd;
if (fcntl(sd, F_SETFL, (fcntl(sd, F_GETFL) | O_NONBLOCK)) != 0) {
stats_error_log("tcpclient[%s]: Unable to set socket to non-blocking: %s", client->name, strerror(errno));
client->last_error = time(NULL);
tcpclient_set_state(client, STATE_BACKOFF);
close(sd);
client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0);
return 5;
}
client->connect_watcher.started = true;
client->connect_watcher.watcher.data = client;
client->timeout_watcher.data = client;
ev_io_init(&client->connect_watcher.watcher, tcpclient_connected, sd, EV_WRITE);
ev_io_start(client->loop, &client->connect_watcher.watcher);
ev_timer_set(&client->timeout_watcher, TCPCLIENT_CONNECT_TIMEOUT, 0);
ev_timer_start(client->loop, &client->timeout_watcher);
if (connect(sd, addr->ai_addr, addr->ai_addrlen) != 0 && errno != EINPROGRESS) {
stats_error_log("tcpclient[%s]: Unable to connect: %s", client->name, strerror(errno));
client->last_error = time(NULL);
tcpclient_set_state(client, STATE_BACKOFF);
ev_timer_stop(client->loop, &client->timeout_watcher);
ev_io_stop(client->loop, &client->connect_watcher.watcher);
close(sd);
client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0);
return 6;
}
tcpclient_set_state(client, STATE_CONNECTING);
return 0;
}
stats_error_log("tcpclient[%s]: Connect with unknown state %i", client->name, client->state);
return 7;
}
